Merge pull request #1064 from iceljc/test/google-realtime

add gemini file content

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs

@@ -1,7 +1,6 @@
 using BotSharp.Abstraction.Files.Converters;
 using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Instructs;
-using BotSharp.Abstraction.Infrastructures;
 
 namespace BotSharp.Core.Files.Services;
 
@@ -22,14 +21,24 @@ public partial class FileInstructService
 
         try
         {
+            var provider = options?.Provider ?? "openai";
             var pdfFiles = await DownloadFiles(sessionDir, files);
-            var images = await ConvertPdfToImages(pdfFiles);
+
-            if (images.IsNullOrEmpty()) return content;
+            var targetFiles = pdfFiles;
+            if (provider != "google-ai")
+            {
+                targetFiles = await ConvertPdfToImages(pdfFiles);
+            }
+
+            if (targetFiles.IsNullOrEmpty())
+            {
+                return content;
+            }
 
             var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
             var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
 
-            var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai",
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: provider,
                 model: options?.Model ?? "gpt-4o", multiModal: true);
             var message = await completion.GetChatCompletions(new Agent()
             {
@@ -39,7 +48,7 @@ public partial class FileInstructService
             {
                 new RoleDialogModel(AgentRole.User, text)
                 {
-                    Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
+                    Files = targetFiles.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
                 }
             });
 

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs

@@ -35,7 +35,7 @@ public class ReadImageFn : IFunctionCallback
         {
             Id = BuiltInAgentId.UtilityAssistant,
             Name = "Utility Agent",
-            Instruction = fromAgent?.Instruction ?? args.UserRequest ?? "Please describe the image(s).",
+            Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).",
             TemplateDict = new Dictionary<string, object>()
         };
 

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Routing;
+
 namespace BotSharp.Plugin.FileHandler.Functions;
 
 public class ReadPdfFn : IFunctionCallback
@@ -25,20 +27,31 @@ public class ReadPdfFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
         var conv = _services.GetRequiredService<IConversationService>();
+        var routingCtx = _services.GetRequiredService<IRoutingContext>();
         var agentService = _services.GetRequiredService<IAgentService>();
 
-        var wholeDialogs = conv.GetDialogHistory();
+        Agent? fromAgent = null;
-        var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
+        if (!string.IsNullOrEmpty(message.CurrentAgentId))
-        var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
-        var fileAgent = new Agent
         {
-            Id = agent?.Id ?? Guid.Empty.ToString(),
+            fromAgent = await agentService.LoadAgent(message.CurrentAgentId);
-            Name = agent?.Name ?? "Unkown",
+        }
-            Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the pdf file(s).",
+
+        var agent = new Agent
+        {
+            Id = BuiltInAgentId.UtilityAssistant,
+            Name = "Utility Agent",
+            Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).",
             TemplateDict = new Dictionary<string, object>()
         };
 
-        var response = await GetChatCompletion(fileAgent, dialogs);
+        var wholeDialogs = routingCtx.GetDialogs();
+        if (wholeDialogs.IsNullOrEmpty())
+        {
+            wholeDialogs = conv.GetDialogHistory();
+        }
+
+        var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
+        var response = await GetChatCompletion(agent, dialogs);
         message.Content = response;
         return true;
     }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs

@@ -1,9 +1,6 @@
-using System.Text.Json.Nodes;
+using BotSharp.Abstraction.Files;
-using BotSharp.Abstraction.Agents;
+using BotSharp.Abstraction.Files.Utilities;
-using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.Conversations;
 using BotSharp.Abstraction.Hooks;
-using BotSharp.Abstraction.Loggers;
 using GenerativeAI;
 using GenerativeAI.Core;
 using GenerativeAI.Types;
@@ -43,7 +40,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
         }
 
         var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
-        var aiModel = client.CreateGenerativeModel(_model);
+        var aiModel = client.CreateGenerativeModel(_model.ToModelId());
         var (prompt, request) = PrepareOptions(aiModel, agent, conversations);
 
         var response = await aiModel.GenerateContentAsync(request);
@@ -101,7 +98,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
         }
 
         var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
-        var chatClient = client.CreateGenerativeModel(_model);
+        var chatClient = client.CreateGenerativeModel(_model.ToModelId());
         var (prompt, messages) = PrepareOptions(chatClient, agent, conversations);
 
         var response = await chatClient.GenerateContentAsync(messages);
@@ -165,7 +162,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
     public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
     {
         var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
-        var chatClient = client.CreateGenerativeModel(_model);
+        var chatClient = client.CreateGenerativeModel(_model.ToModelId());
         var (prompt, messages) = PrepareOptions(chatClient,agent, conversations);
 
         var asyncEnumerable = chatClient.StreamContentAsync(messages);
@@ -207,6 +204,10 @@ public class GeminiChatCompletionProvider : IChatCompletion
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var googleSettings = _services.GetRequiredService<GoogleAiSettings>();
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var settings = settingsService.GetSetting(Provider, _model);
+        var allowMultiModal = settings != null && settings.MultiModal;
         renderedInstructions = [];
 
         // Add settings
@@ -298,7 +299,50 @@ public class GeminiChatCompletionProvider : IChatCompletion
             else if (message.Role == AgentRole.User)
             {
                 var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
-                contents.Add(new Content(text, AgentRole.User));
+                var contentParts = new List<Part> { new() { Text = text } };
+
+                if (allowMultiModal && !message.Files.IsNullOrEmpty())
+                {
+                    foreach (var file in message.Files)
+                    {
+                        if (!string.IsNullOrEmpty(file.FileData))
+                        {
+                            var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
+                            contentParts.Add(new Part()
+                            {
+                                InlineData = new()
+                                {
+                                    MimeType = contentType,
+                                    Data = Convert.ToBase64String(bytes)
+                                }
+                            });
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileStorageUrl))
+                        {
+                            var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
+                            var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
+                            contentParts.Add(new Part()
+                            {
+                                InlineData = new()
+                                {
+                                    MimeType = contentType,
+                                    Data = Convert.ToBase64String(bytes)
+                                }
+                            });
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileUrl))
+                        {
+                            contentParts.Add(new Part()
+                            {
+                                FileData = new()
+                                {
+                                    FileUri = file.FileUrl
+                                }
+                            });
+                        }
+                    }
+                }
+                contents.Add(new Content(contentParts, AgentRole.User));
                 convPrompts.Add($"{AgentRole.User}: {text}");
             }
             else if (message.Role == AgentRole.Assistant)

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs

@@ -7,7 +7,7 @@ public static class ProviderHelper
     public static GenerativeAI.GoogleAi GetGeminiClient(string provider, string model, IServiceProvider services)
     {
         var aiSettings = services.GetRequiredService<GoogleAiSettings>();
-        if (aiSettings == null || aiSettings.Gemini ==null || string.IsNullOrEmpty(aiSettings.Gemini.ApiKey))
+        if (string.IsNullOrEmpty(aiSettings?.Gemini?.ApiKey))
         {
             var settingsService = services.GetRequiredService<ILlmProviderService>();
             var settings = settingsService.GetSetting(provider, model);

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -218,7 +218,6 @@ public class ChatCompletionProvider : IChatCompletion
     protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
-        var state = _services.GetRequiredService<IConversationStateService>();
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var settingsService = _services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(Provider, _model);