upgrade

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs

@@ -4,5 +4,4 @@ public class AgentTool
 {
     public const string FileAnalyzer = "file-analyzer";
     public const string ImageGenerator = "image-generator";
-    public const string HttpHandler = "http-handler";
 }

src/Infrastructure/BotSharp.Abstraction/Agents/IAgentToolHook.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Agents;
+
+public interface IAgentToolHook
+{
+    void AddTools(List<string> tools);
+}

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -29,4 +29,6 @@ public interface IBotSharpFileService
     /// <param name="data"></param>
     /// <returns></returns>
     (string, byte[]) GetFileInfoFromData(string data);
+
+    string GetFileContentType(string filePath);
 }

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs

@@ -57,11 +57,13 @@ public partial class AgentService : IAgentService
 
     public IEnumerable<string> GetAgentTools()
     {
-        var tools = typeof(AgentTool).GetFields(BindingFlags.Public | BindingFlags.Static)
+        var tools = new List<string>();
-                                     .Where(f => f.IsLiteral && f.FieldType == typeof(string))
-                                     .Select(x => x.GetRawConstantValue()?.ToString())
-                                     .ToList();
 
-        return tools;
+        var hooks = _services.GetServices<IAgentToolHook>();
+        foreach (var hook in hooks)
+        {
+            hook.AddTools(tools);
+        }
+        return tools.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().OrderBy(x => x).ToList();
     }
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -49,7 +49,8 @@
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\agent.json" />
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\instruction.liquid" />
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\functions.json" />
-    <None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment_prompt.liquid" />
+    <None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\handle_http_request.fn.liquid" />
+    <None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment.fn.liquid" />
     <None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
     <None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions.json" />
     <None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\instruction.liquid" />
@@ -159,7 +160,10 @@
     <Content Include="data\agents\00000000-0000-0000-0000-000000000000\functions.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment_prompt.liquid">
+    <Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\handle_http_request.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\plugins\config.json">

src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs

@@ -66,8 +66,7 @@ public partial class BotSharpFileService : IBotSharpFileService
         return (contentType, Convert.FromBase64String(base64Str));
     }
 
-    #region Private methods
+    public string GetFileContentType(string filePath)
-    private string GetFileContentType(string filePath)
     {
         string contentType;
         var provider = new FileExtensionContentTypeProvider();
@@ -79,6 +78,7 @@ public partial class BotSharpFileService : IBotSharpFileService
         return contentType;
     }
 
+    #region Private methods
     private bool ExistDirectory(string? dir)
     {
         return !string.IsNullOrEmpty(dir) && Directory.Exists(dir);

src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs

@@ -17,5 +17,6 @@ public class FilePlugin : IBotSharpPlugin
         services.AddScoped<IBotSharpFileService, BotSharpFileService>();
 
         services.AddScoped<IAgentHook, AttachmentProcessingHook>();
+        services.AddScoped<IAgentToolHook, FileAnalyzerToolHook>();
     }
 }

src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs

@@ -19,8 +19,8 @@ public class AttachmentProcessingHook : AgentHookBase
 
         if (isConvMode && isEnabled)
         {
-            var (prompt, loadAttachmentFn) = GetLoadAttachmentFn();
+            var (prompt, fn) = GetPromptAndFunction();
-            if (loadAttachmentFn != null)
+            if (fn != null)
             {
                 if (!string.IsNullOrWhiteSpace(prompt))
                 {
@@ -29,11 +29,11 @@ public class AttachmentProcessingHook : AgentHookBase
 
                 if (agent.Functions == null)
                 {
-                    agent.Functions = new List<FunctionDef> { loadAttachmentFn };
+                    agent.Functions = new List<FunctionDef> { fn };
                 }
                 else
                 {
-                    agent.Functions.Add(loadAttachmentFn);
+                    agent.Functions.Add(fn);
                 }
             }
         }
@@ -41,13 +41,13 @@ public class AttachmentProcessingHook : AgentHookBase
         base.OnAgentLoaded(agent);
     }
 
-    private (string, FunctionDef?) GetLoadAttachmentFn()
+    private (string, FunctionDef?) GetPromptAndFunction()
     {
-        var fnName = "load_attachment";
+        var fn = "load_attachment";
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var agent = db.GetAgent(TOOL_ASSISTANT);
-        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fnName}_prompt"))?.Content ?? string.Empty;
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fn}.fn"))?.Content ?? string.Empty;
-        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fnName));
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fn));
         return (prompt, loadAttachmentFn);
     }
 }

src/Infrastructure/BotSharp.Core/Files/Hooks/FileAnalyzerToolHook.cs

@@ -0,0 +1,10 @@
+
+namespace BotSharp.Core.Files.Hooks;
+
+public class FileAnalyzerToolHook : IAgentToolHook
+{
+    public void AddTools(List<string> tools)
+    {
+        tools.Add(AgentTool.FileAnalyzer);
+    }
+}

src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json

@@ -16,5 +16,28 @@
       },
       "required": [ "user_request", "file_types" ]
     }
+  },
+
+  {
+    "name": "handle_http_request",
+    "description": "If the user requests to send an http request, you need to capture the http method and request content, and then call this function to send the http request.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "request_url": {
+          "type": "string",
+          "description": "The http url that is requested. It can be an absolute url that starts with 'http' or 'https', or a relative url that starts with '/'"
+        },
+        "http_method": {
+          "type": "string",
+          "description": "The http method that is requested, e.g., GET, POST, PUT, and DELETE."
+        },
+        "request_content": {
+          "type": "string",
+          "description": "The http request content. It must be in json format.."
+        }
+      },
+      "required": [ "request_url", "http_method" ]
+    }
   }
 ]

src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_http_request.fn.liquid

@@ -0,0 +1 @@
+Please call handle_http_request if user wants to send an http request.

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -3,6 +3,7 @@ using BotSharp.Abstraction.Instructs;
 using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Core.Infrastructures;
 using BotSharp.OpenAPI.ViewModels.Instructs;
+using NetTopologySuite.IO;
 
 namespace BotSharp.OpenAPI.Controllers;
 

src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netstandard2.1</TargetFramework>
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.17" />
+    <PackageReference Include="Azure.AI.OpenAI" Version="2.0.0-beta.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -1,21 +1,4 @@
-using Azure.AI.OpenAI;
+using OpenAI.Chat;
-using BotSharp.Abstraction.Agents;
-using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.Agents.Models;
-using BotSharp.Abstraction.Conversations;
-using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Files;
-using BotSharp.Abstraction.Loggers;
-using BotSharp.Abstraction.MLTasks;
-using BotSharp.Abstraction.Utilities;
-using BotSharp.Plugin.AzureOpenAI.Settings;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
@@ -49,22 +32,23 @@ public class ChatCompletionProvider : IChatCompletion
         }
 
         var client = ProviderHelper.GetClient(Provider, _model, _services);
-        var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations);
+        var chatClient = client.GetChatClient(_model);
-        chatCompletionsOptions.DeploymentName = _model;
+        var (prompt, messages, options) = PrepareOptions(agent, conversations);
-        var response = client.GetChatCompletions(chatCompletionsOptions);
+
-        var choice = response.Value.Choices[0];
+        var response = chatClient.CompleteChat(messages, options);
-        var message = choice.Message;
+        var value = response.Value;
+        var reason = value.FinishReason;
+        var content = value.Content;
 
         RoleDialogModel responseMessage;
-
+        if (reason == ChatFinishReason.FunctionCall)
-        if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
         {
-            responseMessage = new RoleDialogModel(AgentRole.Function, message.Content)
+            responseMessage = new RoleDialogModel(AgentRole.Function, content[0].Text)
             {
                 CurrentAgentId = agent.Id,
                 MessageId = conversations.Last().MessageId,
-                FunctionName = message.FunctionCall.Name,
+                FunctionName = value.FunctionCall.FunctionName,
-                FunctionArgs = message.FunctionCall.Arguments
+                FunctionArgs = value.FunctionCall.FunctionArguments
             };
 
             // Somethings LLM will generate a function name with agent name.
@@ -73,28 +57,20 @@ public class ChatCompletionProvider : IChatCompletion
                 responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
             }
         }
-        else if (choice.FinishReason == CompletionsFinishReason.ToolCalls)
+        else if (reason == ChatFinishReason.ToolCalls)
         {
-            // Add the assistant message with tool calls to the conversation history
+            var toolCall = value.ToolCalls.FirstOrDefault();
-            // ChatRequestAssistantMessage toolCallHistoryMessage = new(message);
+            responseMessage = new RoleDialogModel(AgentRole.Function, content[0].Text)
-            // chatCompletionsOptions.Messages.Add(toolCallHistoryMessage);
-
-            // Add a new tool message for each tool call that is resolved
-            var toolCall = message.ToolCalls.First() as ChatCompletionsFunctionToolCall;
-            // var toolCallResponseMessage = GetToolCallResponseMessage(toolCall);
-            // Now make a new request with all the messages thus far, including the original
-
-            responseMessage = new RoleDialogModel(AgentRole.Function, message.Content)
             {
                 CurrentAgentId = agent.Id,
                 MessageId = conversations.Last().MessageId,
-                FunctionName = toolCall.Name,
+                FunctionName = toolCall?.FunctionName,
-                FunctionArgs = toolCall.Arguments
+                FunctionArgs = toolCall?.FunctionArguments
             };
         }
         else
         {
-            responseMessage = new RoleDialogModel(AgentRole.Assistant, message.Content)
+            responseMessage = new RoleDialogModel(AgentRole.Assistant, content[0].Text)
             {
                 CurrentAgentId = agent.Id,
                 MessageId = conversations.Last().MessageId
@@ -109,8 +85,8 @@ public class ChatCompletionProvider : IChatCompletion
                 Prompt = prompt,
                 Provider = Provider,
                 Model = _model,
-                PromptCount = response.Value.Usage.PromptTokens,
+                PromptCount = response.Value.Usage.InputTokens,
-                CompletionCount = response.Value.Usage.CompletionTokens
+                CompletionCount = response.Value.Usage.OutputTokens
             });
         }
 
@@ -131,14 +107,15 @@ public class ChatCompletionProvider : IChatCompletion
         }
 
         var client = ProviderHelper.GetClient(Provider, _model, _services);
-        var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations);
+        var chatClient = client.GetChatClient(_model);
+        var (prompt, messages, options) = PrepareOptions(agent, conversations);
 
-        chatCompletionsOptions.DeploymentName = _model;
+        var response = await chatClient.CompleteChatAsync(messages, options);
-        var response = await client.GetChatCompletionsAsync(chatCompletionsOptions);
+        var value = response.Value;
-        var choice = response.Value.Choices[0];
+        var reason = value.FinishReason;
-        var message = choice.Message;
+        var content = value.Content;
 
-        var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
+        var msg = new RoleDialogModel(AgentRole.Assistant, content[0].Text)
         {
             CurrentAgentId = agent.Id
         };
@@ -151,20 +128,20 @@ public class ChatCompletionProvider : IChatCompletion
                 Prompt = prompt,
                 Provider = Provider,
                 Model = _model,
-                PromptCount = response.Value.Usage.PromptTokens,
+                PromptCount = response.Value.Usage.InputTokens,
-                CompletionCount = response.Value.Usage.CompletionTokens
+                CompletionCount = response.Value.Usage.OutputTokens
             });
         }
 
-        if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
+        if (reason == ChatFinishReason.FunctionCall)
         {
-            _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name}({message.FunctionCall.Arguments})");
+            _logger.LogInformation($"[{agent.Name}]: {value.FunctionCall.FunctionName}({value.FunctionCall.FunctionArguments})");
 
-            var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content)
+            var funcContextIn = new RoleDialogModel(AgentRole.Function, content[0].Text)
             {
                 CurrentAgentId = agent.Id,
-                FunctionName = message.FunctionCall.Name,
+                FunctionName = value.FunctionCall.FunctionName,
-                FunctionArgs = message.FunctionCall.Arguments
+                FunctionArgs = value.FunctionCall.FunctionArguments
             };
 
             // Somethings LLM will generate a function name with agent name.
@@ -188,37 +165,33 @@ public class ChatCompletionProvider : IChatCompletion
     public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
     {
         var client = ProviderHelper.GetClient(Provider, _model, _services);
-        var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations);
+        var chatClient = client.GetChatClient(_model);
-        chatCompletionsOptions.DeploymentName = _model;
+        var (prompt, messages, options) = PrepareOptions(agent, conversations);
-        var response = await client.GetChatCompletionsStreamingAsync(chatCompletionsOptions);
+
+        var response = chatClient.CompleteChatStreamingAsync(messages, options);
 
-        string output = "";
         await foreach (var choice in response)
         {
-            if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
+            if (choice.FinishReason == ChatFinishReason.FunctionCall)
             {
-                Console.Write(choice.FunctionArgumentsUpdate);
+                Console.Write(choice.FunctionCallUpdate?.FunctionArgumentsUpdate);
                     
-                await onMessageReceived(new RoleDialogModel(ChatRole.Assistant.ToString(), choice.FunctionArgumentsUpdate));
+                await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, choice.FunctionCallUpdate?.FunctionArgumentsUpdate));
                 continue;
             }
 
-            if (choice.ContentUpdate == null)
+            if (choice.ContentUpdate.IsNullOrEmpty()) continue;
-                continue;
-            Console.Write(choice.ContentUpdate);
 
-            _logger.LogInformation(choice.ContentUpdate);
+            _logger.LogInformation(choice.ContentUpdate[0].Text);
 
-            await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate));
+            await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate[0].Text));
-            
-            output = "";
         }
 
         return true;
     }
 
 
-    protected (string, ChatCompletionsOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
+    protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var fileService = _services.GetRequiredService<IBotSharpFileService>();
@@ -227,75 +200,66 @@ public class ChatCompletionProvider : IChatCompletion
         var settings = settingsService.GetSetting(Provider, _model);
         var allowMultiModal = settings != null && settings.MultiModal;
 
-        var chatCompletionsOptions = new ChatCompletionsOptions();
+        var messages = new List<ChatMessage>();
-        
+
+        var temperature = float.Parse(state.GetState("temperature", "0.0"));
+        var maxTokens = int.Parse(state.GetState("max_tokens", "1024"));
+        var options = new ChatCompletionOptions()
+        {
+            Temperature = temperature,
+            MaxTokens = maxTokens
+        };
+
+        foreach (var function in agent.Functions)
+        {
+            if (!agentService.RenderFunction(agent, function)) continue;
+
+            var property = agentService.RenderFunctionProperty(agent, function);
+
+            options.Tools.Add(ChatTool.CreateFunctionTool(
+                functionName: function.Name,
+                functionDescription: function.Description,
+                functionParameters: BinaryData.FromObjectAsJson(property)));
+        }
+
         if (!string.IsNullOrEmpty(agent.Instruction))
         {
             var instruction = agentService.RenderedInstruction(agent);
-            chatCompletionsOptions.Messages.Add(new ChatRequestSystemMessage(instruction));
+            messages.Add(new SystemChatMessage(instruction));
         }
 
         if (!string.IsNullOrEmpty(agent.Knowledges))
         {
-            chatCompletionsOptions.Messages.Add(new ChatRequestSystemMessage(agent.Knowledges));
+            messages.Add(new SystemChatMessage(agent.Knowledges));
         }
 
         var samples = ProviderHelper.GetChatSamples(agent.Samples);
-        foreach (var message in samples)
+        foreach (var sample in samples)
         {
-            chatCompletionsOptions.Messages.Add(message.Role == AgentRole.User ? 
+            messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content));
-                new ChatRequestUserMessage(message.Content) :
-                new ChatRequestAssistantMessage(message.Content));
-        }
-
-        foreach (var function in agent.Functions)
-        {
-            if (agentService.RenderFunction(agent, function))
-            {
-                var property = agentService.RenderFunctionProperty(agent, function);
-
-                // legacy function call
-                /*chatCompletionsOptions.Functions.Add(new FunctionDefinition
-                {
-                    Name = function.Name,
-                    Description = function.Description,
-                    Parameters = BinaryData.FromObjectAsJson(property)
-                });*/
-
-                // new chat tool
-                chatCompletionsOptions.Tools.Add(new ChatCompletionsFunctionToolDefinition
-                {
-                    Name = function.Name,
-                    Description = function.Description,
-                    Parameters = BinaryData.FromObjectAsJson(property)
-                });
-            }
         }
 
         foreach (var message in conversations)
         {
-            if (message.Role == ChatRole.Function)
+            if (message.Role == AgentRole.Function)
             {
-                chatCompletionsOptions.Messages.Add(new ChatRequestAssistantMessage(string.Empty)
+                messages.Add(new AssistantChatMessage(string.Empty)
                 {
-                    FunctionCall = new FunctionCall(message.FunctionName, message.FunctionArgs ?? String.Empty),
+                    FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
                 });
 
-                chatCompletionsOptions.Messages.Add(new ChatRequestFunctionMessage(message.FunctionName, message.Content));
+                messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
-                // chatCompletionsOptions.Messages.Add(new ChatRequestToolMessage(message.Content, message.ToolCallId));
             }
-            else if (message.Role == ChatRole.User)
+            else if (message.Role == AgentRole.User)
             {
                 var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var chat = new UserChatMessage(text)
+                {
+                    ParticipantName = message.FunctionName
+                };
 
-                ChatRequestUserMessage userMessage = null;
                 if (allowMultiModal)
                 {
-                    var chatItems = new List<ChatMessageContentItem>()
-                    {
-                        new ChatMessageTextContentItem(text)
-                    };
-                
                     if (!message.Files.IsNullOrEmpty())
                     {
                         foreach (var file in message.Files)
@@ -303,127 +267,97 @@ public class ChatCompletionProvider : IChatCompletion
                             if (!string.IsNullOrEmpty(file.FileUrl))
                             {
                                 var uri = new Uri(file.FileUrl);
-                                chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
+                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
+                                chat = new UserChatMessage(contentPart, text);
                             }
                             else if (!string.IsNullOrEmpty(file.FileData))
                             {
                                 var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
-                                using var stream = new MemoryStream(bytes, 0, bytes.Length);
+                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
-                                chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low));
+                                chat = new UserChatMessage(contentPart, text);
                             }
                             else if (!string.IsNullOrEmpty(file.FileStorageUrl))
                             {
+                                var contentType = fileService.GetFileContentType(file.FileStorageUrl);
                                 using var stream = File.OpenRead(file.FileStorageUrl);
-                                chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low));
+                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
+                                chat = new UserChatMessage(contentPart, text);
                             }
                         }
                     }
-                
-                    //if (!string.IsNullOrEmpty(message.ImageUrl))
-                    //{
-                    //    var uri = new Uri(message.ImageUrl);
-                    //    userMessage.MultimodalContentItems.Add(
-                    //        new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
-                    //}
-                
-                    userMessage = new ChatRequestUserMessage(chatItems)
-                    {
-                        // To display Planner name in log
-                        Name = message.FunctionName,
-                    };
                 }
-                else
+                messages.Add(chat);
-                {
-                    userMessage = new ChatRequestUserMessage(text)
-                    {
-                        // To display Planner name in log
-                        Name = message.FunctionName,
-                    };
-                }
-
-                chatCompletionsOptions.Messages.Add(userMessage);
             }
-            else if (message.Role == ChatRole.Assistant)
+            else if (message.Role == AgentRole.Assistant)
             {
-                chatCompletionsOptions.Messages.Add(new ChatRequestAssistantMessage(message.Content));
+                messages.Add(new AssistantChatMessage(message.Content));
             }
         }
 
-        // https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683
+        var prompt = GetPrompt(messages, options);
-        //var state = _services.GetRequiredService<IConversationStateService>();
+        return (prompt, messages, options);
-        var temperature = float.Parse(state.GetState("temperature", "0.0"));
-        var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0"));
-        chatCompletionsOptions.Temperature = temperature;
-        chatCompletionsOptions.NucleusSamplingFactor = samplingFactor;
-        chatCompletionsOptions.MaxTokens = int.Parse(state.GetState("max_tokens", "1024"));
-        // chatCompletionsOptions.FrequencyPenalty = 0;
-        // chatCompletionsOptions.PresencePenalty = 0;
-
-        var prompt = GetPrompt(chatCompletionsOptions);
-
-        return (prompt, chatCompletionsOptions);
     }
 
-    private string GetPrompt(ChatCompletionsOptions chatCompletionsOptions)
+
+    private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
     {
         var prompt = string.Empty;
 
-        if (chatCompletionsOptions.Messages.Count > 0)
+        if (!messages.IsNullOrEmpty())
         {
             // System instruction
-            var verbose = string.Join("\r\n", chatCompletionsOptions.Messages
+            var verbose = string.Join("\r\n", messages
-                .Where(x => x.Role == AgentRole.System)
+                .Select(x => x as SystemChatMessage)
-                .Select(x => x as ChatRequestSystemMessage).Select(x =>
+                .Where(x => x != null)
+                .Select(x =>
                 {
-                    if (!string.IsNullOrEmpty(x.Name))
+                    if (!string.IsNullOrEmpty(x.ParticipantName))
                     {
                         // To display Agent name in log
-                        return $"[{x.Name}]: {x.Content}";
+                        return $"[{x.ParticipantName}]: {x.Content[0].Text}";
                     }
-                    return $"{x.Role}: {x.Content}";
+                    return $"{AgentRole.System}: {x.Content[0].Text}";
                 }));
             prompt += $"{verbose}\r\n";
 
             prompt += "\r\n[CONVERSATION]";
-            verbose = string.Join("\r\n", chatCompletionsOptions.Messages
+            verbose = string.Join("\r\n", messages
-                .Where(x => x.Role != AgentRole.System).Select(x =>
+                .Where(x => (x as SystemChatMessage) == null)
+                .Select(x =>
                 {
-                    if (x.Role == ChatRole.Function)
+                    var fnMessage = x as FunctionChatMessage;
+                    if (fnMessage != null)
                     {
-                        var m = x as ChatRequestFunctionMessage;
+                        return $"{AgentRole.Function}: {fnMessage.Content[0].Text}";
-                        return $"{m.Role}: {m.Content}";
                     }
-                    else if (x.Role == ChatRole.User)
+
+                    var userMessage = x as UserChatMessage;
+                    if (userMessage != null)
                     {
-                        var m = x as ChatRequestUserMessage;
+                        var content = x.Content[0].Text;
-                        var content = m.Content ?? string.Join(", ", m.MultimodalContentItems
+                        return !string.IsNullOrEmpty(userMessage.ParticipantName) && userMessage.ParticipantName != "route_to_agent" ?
-                            .Where(m => m is ChatMessageTextContentItem)
+                            $"{userMessage.ParticipantName}: {content}" :
-                            .Select(m => (m as ChatMessageTextContentItem)?.Text));
+                            $"{AgentRole.User}: {content}";
-                        return !string.IsNullOrEmpty(m.Name) && m.Name != "route_to_agent" ?
-                            $"{m.Name}: {content}" :
-                            $"{m.Role}: {content}";
                     }
-                    else if (x.Role == ChatRole.Assistant)
+
+                    var assistMessage = x as AssistantChatMessage;
+                    if (assistMessage != null)
                     {
-                        var m = x as ChatRequestAssistantMessage;
+                        return assistMessage.FunctionCall != null ?
-                        return m.FunctionCall != null ?
+                            $"{AgentRole.Assistant}: Call function {assistMessage.FunctionCall.FunctionName}({assistMessage.FunctionCall.FunctionArguments})" :
-                            $"{m.Role}: Call function {m.FunctionCall.Name}({m.FunctionCall.Arguments})" :
+                            $"{AgentRole.Assistant}: {assistMessage.Content[0].Text}";
-                            $"{m.Role}: {m.Content}";
-                    }
-                    else
-                    {
-                        throw new NotImplementedException("Not found role");
                     }
+
+                    return string.Empty;
                 }));
             prompt += $"\r\n{verbose}\r\n";
         }
 
-        if (chatCompletionsOptions.Tools.Count > 0)
+        if (!options.Tools.IsNullOrEmpty())
         {
-            var functions = string.Join("\r\n", chatCompletionsOptions.Tools.Select(x =>
+            var functions = string.Join("\r\n", options.Tools.Select(fn =>
             {
-                var fn = x as ChatCompletionsFunctionToolDefinition;
+                return $"\r\n{fn.FunctionName}: {fn.FunctionDescription}\r\n{fn.FunctionParameters}";
-                return $"\r\n{fn.Name}: {fn.Description}\r\n{fn.Parameters}";
             }));
             prompt += $"\r\n[FUNCTIONS]{functions}\r\n";
         }
@@ -435,15 +369,4 @@ public class ChatCompletionProvider : IChatCompletion
     {
         _model = model;
     }
-
-    ChatRequestToolMessage GetToolCallResponseMessage(ChatCompletionsToolCall toolCall)
-    {
-        var functionToolCall = toolCall as ChatCompletionsFunctionToolCall;
-        // Validate and process the JSON arguments for the function call
-        string unvalidatedArguments = functionToolCall.Arguments;
-        var functionResultData = (object)null; // GetYourFunctionResultData(unvalidatedArguments);
-                                               // Here, replacing with an example as if returned from "GetYourFunctionResultData"
-        functionResultData = "31 celsius";
-        return new ChatRequestToolMessage(functionResultData.ToString(), toolCall.Id);
-    }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs

@@ -1,17 +1,4 @@
-using Azure.AI.OpenAI;
+using OpenAI.Images;
-using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.Agents.Models;
-using BotSharp.Abstraction.Conversations;
-using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Loggers;
-using BotSharp.Abstraction.MLTasks;
-using BotSharp.Plugin.AzureOpenAI.Settings;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
@@ -47,21 +34,32 @@ public class ImageGenerationProvider : IImageGeneration
         }
 
         var client = ProviderHelper.GetClient(Provider, _model, _services);
-        var options = PrepareOptions(conversations);
+        var (prompt, options) = PrepareOptions(conversations);
-        var response = await client.GetImageGenerationsAsync(options);
+        var imageClient = client.GetImageClient(_model);
-        var image = response.Value.Data.First();
+
+        ImageGenerationOptions myoptions = new()
+        {
+            Quality = GeneratedImageQuality.High,
+            Size = GeneratedImageSize.W1792xH1024,
+            Style = GeneratedImageStyle.Vivid,
+            ResponseFormat = GeneratedImageFormat.Bytes
+        };
+
+        var response = imageClient.GenerateImage(prompt, myoptions);
+        var imageUri = response.Value.ImageUri;
+        var revisedPrompt = response.Value.RevisedPrompt;
 
         var content = string.Empty;
-        if (!string.IsNullOrEmpty(image.RevisedPrompt))
+        if (!string.IsNullOrEmpty(revisedPrompt))
         {
-            content = image.RevisedPrompt;
+            content = revisedPrompt;
         }
 
         var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
         {
             CurrentAgentId = agent.Id,
             MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
-            Data = image.Url.AbsoluteUri ?? image.Base64Data
+            Data = imageUri.AbsoluteUri
         };
 
         // After
@@ -69,10 +67,10 @@ public class ImageGenerationProvider : IImageGeneration
         {
             await hook.AfterGenerated(responseMessage, new TokenStatsModel
             {
-                Prompt = options.Prompt,
+                Prompt = prompt,
                 Provider = Provider,
                 Model = _model,
-                PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(),
+                PromptCount = prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(),
                 CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count()
             });
         }
@@ -80,25 +78,99 @@ public class ImageGenerationProvider : IImageGeneration
         return responseMessage;
     }
 
-    private ImageGenerationOptions PrepareOptions(List<RoleDialogModel> conversations)
+    private (string, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
     {
-        var state = _services.GetRequiredService<IConversationStateService>();
+        var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty;
 
-        var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024";
+        var state = _services.GetRequiredService<IConversationStateService>();
-        var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard";
+        var size = state.GetState("image_size");
+        var quality = state.GetState("image_quality");
+        var style = state.GetState("image_style");
 
         var options = new ImageGenerationOptions
         {
-            DeploymentName = _model,
+            //Size = GetImageSize(size),
-            Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty,
+            //Quality = GetImageQuality(quality),
-            Size = new ImageSize(sizeValue),
+            //Style = GetImageStyle(style),
-            Quality = new ImageGenerationQuality(qualityValue)
+            //ResponseFormat = GeneratedImageFormat.Uri
         };
-        return options;
+        return (prompt, options);
     }
 
     public void SetModelName(string model)
     {
         _model = model;
     }
+
+    private GeneratedImageSize GetImageSize(string size)
+    {
+        var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";
+
+        GeneratedImageSize retSize;
+        switch (value)
+        {
+            case "256x256":
+                retSize = GeneratedImageSize.W256xH256;
+                break;
+            case "512x512":
+                retSize = GeneratedImageSize.W512xH512;
+                break;
+            case "1024x1024":
+                retSize = GeneratedImageSize.W1024xH1024;
+                break;
+            case "1024x1792":
+                retSize = GeneratedImageSize.W1024xH1792;
+                break;
+            case "1792x1024":
+                retSize = GeneratedImageSize.W1792xH1024;
+                break;
+            default:
+                retSize = GeneratedImageSize.W1024xH1024;
+                break;
+        }
+
+        return retSize;
+    }
+
+    private GeneratedImageQuality GetImageQuality(string quality)
+    {
+        var value = !string.IsNullOrEmpty(quality) ? quality : "standard";
+
+        GeneratedImageQuality retQuality;
+        switch (value)
+        {
+            case "standard":
+                retQuality = GeneratedImageQuality.Standard;
+                break;
+            case "hd":
+                retQuality = GeneratedImageQuality.High;
+                break;
+            default:
+                retQuality = GeneratedImageQuality.Standard;
+                break;
+        }
+
+        return retQuality;
+    }
+
+    private GeneratedImageStyle GetImageStyle(string style)
+    {
+        var value = !string.IsNullOrEmpty(style) ? style : "natural";
+
+        GeneratedImageStyle retStyle;
+        switch (value)
+        {
+            case "standard":
+                retStyle = GeneratedImageStyle.Natural;
+                break;
+            case "vivid":
+                retStyle = GeneratedImageStyle.Vivid;
+                break;
+            default:
+                retStyle = GeneratedImageStyle.Natural;
+                break;
+        }
+
+        return retStyle;
+    }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiChatCompletionProvider.cs

@@ -1,7 +1,3 @@
-using BotSharp.Plugin.AzureOpenAI.Settings;
-using Microsoft.Extensions.Logging;
-using System;
-
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
 public class OpenAiChatCompletionProvider : ChatCompletionProvider

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs

@@ -1,7 +1,3 @@
-using BotSharp.Plugin.AzureOpenAI.Settings;
-using Microsoft.Extensions.Logging;
-using System;
-
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
 public class OpenAiImageGenerationProvider : ImageGenerationProvider

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs

@@ -1,10 +1,7 @@
 using Azure.AI.OpenAI;
 using Azure;
-using System;
+using OpenAI;
-using BotSharp.Abstraction.Conversations.Models;
+using System.ClientModel;
-using System.Collections.Generic;
-using Microsoft.Extensions.DependencyInjection;
-using BotSharp.Abstraction.MLTasks;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
@@ -15,8 +12,8 @@ public class ProviderHelper
         var settingsService = services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(provider, model);
         var client = provider == "openai" ?
-            new OpenAIClient($"{settings.ApiKey}") :
+            new OpenAIClient(new ApiKeyCredential(settings.ApiKey)) :
-            new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey));
+            new AzureOpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey));
         return client;
     }
 

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs

@@ -1,16 +1,4 @@
-using Azure.AI.OpenAI;
+using OpenAI.Chat;
-using BotSharp.Abstraction.MLTasks;
-using System;
-using System.Threading.Tasks;
-using BotSharp.Plugin.AzureOpenAI.Settings;
-using BotSharp.Abstraction.Conversations;
-using Microsoft.Extensions.DependencyInjection;
-using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Agents.Enums;
-using System.Linq;
-using System.Collections.Generic;
-using BotSharp.Abstraction.Agents.Models;
-using BotSharp.Abstraction.Loggers;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
@@ -51,28 +39,26 @@ public class TextCompletionProvider : ITextCompletion
                 })).ToArray());
 
         var client = ProviderHelper.GetClient(Provider, _model, _services);
+        var chatClient = client.GetChatClient(_model);
 
-        var completionsOptions = new CompletionsOptions()
+        var messages = new List<ChatMessage>()
         {
-            Prompts =
+            new UserChatMessage(text)
-            {
-                text
-            },
-            MaxTokens = 256,
         };
-        completionsOptions.StopSequences.Add($"{AgentRole.Assistant}:");
 
         var state = _services.GetRequiredService<IConversationStateService>();
         var temperature = float.Parse(state.GetState("temperature", "0.0"));
-        var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0"));
+        var completionOptions = new ChatCompletionOptions()
-        completionsOptions.Temperature = temperature;
+        {
-        completionsOptions.NucleusSamplingFactor = samplingFactor;
+            MaxTokens = 256,
-        completionsOptions.DeploymentName = _model;
+            Temperature = temperature
-        var response = await client.GetCompletionsAsync(completionsOptions);
+        };
+
+        var response = await chatClient.CompleteChatAsync(messages, completionOptions);
 
         // OpenAI
         var completion = "";
-        foreach (var t in response.Value.Choices)
+        foreach (var t in response.Value.Content)
         {
             completion += t.Text;
         };
@@ -89,8 +75,8 @@ public class TextCompletionProvider : ITextCompletion
                 Prompt = text,
                 Provider = Provider,
                 Model = _model,
-                PromptCount = response.Value.Usage.PromptTokens,
+                PromptCount = response.Value.Usage.InputTokens,
-                CompletionCount = response.Value.Usage.CompletionTokens
+                CompletionCount = response.Value.Usage.OutputTokens
             })).ToArray());
 
         return completion.Trim();

src/Plugins/BotSharp.Plugin.AzureOpenAI/Using.cs

@@ -0,0 +1,17 @@
+global using System;
+global using System.Collections.Generic;
+global using System.Linq;
+global using System.IO;
+global using System.Threading.Tasks;
+global using Microsoft.Extensions.DependencyInjection;
+global using Microsoft.Extensions.Logging;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Loggers;
+global using BotSharp.Abstraction.MLTasks;
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.Utilities;
+global using BotSharp.Plugin.AzureOpenAI.Settings;

src/Plugins/BotSharp.Plugin.HttpHandler/Enums/Tool.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Plugin.HttpHandler.Enums;
+
+public class Tool
+{
+    public const string HttpHandler = "http-handler";
+}

src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs

@@ -0,0 +1,59 @@
+using BotSharp.Abstraction.Agents;
+using BotSharp.Abstraction.Agents.Settings;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Repositories;
+using BotSharp.Plugin.HttpHandler.Enums;
+
+namespace BotSharp.Plugin.HttpHandler.Hooks;
+
+public class HttpHandlerHook : AgentHookBase
+{
+    private static string TOOL_ASSISTANT = Guid.Empty.ToString();
+
+    public override string SelfId => string.Empty;
+
+    public HttpHandlerHook(IServiceProvider services, AgentSettings settings)
+        : base(services, settings)
+    {
+    }
+
+    public override void OnAgentLoaded(Agent agent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var isConvMode = conv.IsConversationMode();
+        var isEnabled = !agent.Tools.IsNullOrEmpty() && agent.Tools.Contains(Tool.HttpHandler);
+
+        if (isConvMode && isEnabled)
+        {
+            var (prompt, fn) = GetPromptAndFunction();
+            if (fn != null)
+            {
+                if (!string.IsNullOrWhiteSpace(prompt))
+                {
+                    agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+                }
+
+                if (agent.Functions == null)
+                {
+                    agent.Functions = new List<FunctionDef> { fn };
+                }
+                else
+                {
+                    agent.Functions.Add(fn);
+                }
+            }
+        }
+
+        base.OnAgentLoaded(agent);
+    }
+
+    private (string, FunctionDef?) GetPromptAndFunction()
+    {
+        var fn = "handle_http_request";
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agent = db.GetAgent(TOOL_ASSISTANT);
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fn}.fn"))?.Content ?? string.Empty;
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fn));
+        return (prompt, loadAttachmentFn);
+    }
+}

src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerToolHook.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.Agents;
+using BotSharp.Plugin.HttpHandler.Enums;
+
+namespace BotSharp.Plugin.HttpHandler.Hooks;
+
+public class HttpHandlerToolHook : IAgentToolHook
+{
+    public void AddTools(List<string> tools)
+    {
+        tools.Add(Tool.HttpHandler);
+    }
+}

src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs

@@ -1,5 +1,6 @@
-using BotSharp.Abstraction.Http.Settings;
+using BotSharp.Abstraction.Agents;
 using BotSharp.Abstraction.Settings;
+using BotSharp.Plugin.HttpHandler.Hooks;
 using Microsoft.Extensions.Configuration;
 
 namespace BotSharp.Plugin.HttpHandler;
@@ -19,5 +20,8 @@ public class HttpHandlerPlugin : IBotSharpPlugin
             var settingService = provider.GetRequiredService<ISettingService>();
             return settingService.Bind<HttpSettings>("Http");
         });
+
+        services.AddScoped<IAgentHook, HttpHandlerHook>();
+        services.AddScoped<IAgentToolHook, HttpHandlerToolHook>();
     }
 }

src/WebStarter/appsettings.json

@@ -292,7 +292,8 @@
       "BotSharp.Plugin.WebDriver",
       "BotSharp.Plugin.LLamaSharp",
       "BotSharp.Plugin.SparkDesk",
-      "BotSharp.Plugin.MetaGLM"
+      "BotSharp.Plugin.MetaGLM",
+      "BotSharp.Plugin.HttpHandler"
     ]
   }
 }