temp save

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs

@@ -47,15 +47,19 @@ public class DialogElement
     public string Content { get; set; } = default!;
 
     [JsonPropertyName("secondary_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? SecondaryContent { get; set; }
 
     [JsonPropertyName("rich_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? RichContent { get; set; }
 
     [JsonPropertyName("secondary_rich_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? SecondaryRichContent { get; set; }
 
     [JsonPropertyName("payload")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? Payload { get; set; }
 
     public DialogElement()
@@ -95,8 +99,17 @@ public class DialogMetaData
     public string MessageType { get; set; } = default!;
 
     [JsonPropertyName("function_name")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? FunctionName { get; set; }
 
+    [JsonPropertyName("function_args")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FunctionArgs { get; set; }
+
+    [JsonPropertyName("tool_call_id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ToolCallId { get; set; }
+
     [JsonPropertyName("sender_id")]
     public string? SenderId { get; set; }
 

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -10,6 +10,7 @@ public class RealtimeHubConnection
     public string KeypadInputBuffer { get; set; } = string.Empty;
     public string CurrentAgentId { get; set; } = null!;
     public string ConversationId { get; set; } = null!;
+    public string? PrevSessionId { get; set; }
     public Func<string> OnModelReady { get; set; } = () => string.Empty;
     public Func<string, string> OnModelMessageReceived { get; set; } = null!;
     public Func<string> OnModelAudioResponseDone { get; set; } = null!;

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -65,9 +65,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
         else
         {
             // Update session for changed states
-
+            var instruction = await hub.Completer.UpdateSession(hub.HubConn);
-            // TO DO
-            //var instruction = await hub.Completer.UpdateSession(hub.HubConn);
             await hub.Completer.InsertConversationItem(message);
 
             if (string.IsNullOrEmpty(message.Content))
@@ -81,7 +79,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
             //}
             //else
             //{
-            //    await hub.Completer.TriggerModelInference();
+            //    await hub.Completer.TriggerModelInference(instruction);
             //}
         }
     }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -96,6 +96,8 @@ public class RealtimeHub : IRealtimeHub
                         }
 
                         await routing.InvokeFunction(message.FunctionName, message);
+                        dialogs.Add(message);
+                        storage.Append(_conn.ConversationId, message);
                     }
                     else
                     {

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs

@@ -40,6 +40,8 @@ public class ConversationStorage : IConversationStorage
                     MessageId = dialog.MessageId,
                     MessageType = dialog.MessageType,
                     FunctionName = dialog.FunctionName,
+                    FunctionArgs = dialog.FunctionArgs,
+                    ToolCallId = dialog.ToolCallId,
                     CreatedTime = dialog.CreatedAt
                 };
 
@@ -109,7 +111,6 @@ public class ConversationStorage : IConversationStorage
             var currentAgentId = meta.AgentId;
             var messageId = meta.MessageId;
             var messageType = meta.MessageType;
-            var function = meta.FunctionName;
             var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
             var createdAt = meta.CreatedTime;
             var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? 
@@ -124,7 +125,9 @@ public class ConversationStorage : IConversationStorage
                 MessageType = messageType,
                 CreatedAt = createdAt,
                 SenderId = senderId,
-                FunctionName = function,
+                FunctionName = meta.FunctionName,
+                FunctionArgs = meta.FunctionArgs,
+                ToolCallId = meta.ToolCallId,
                 RichContent = richContent,
                 SecondaryContent = secondaryContent,
                 SecondaryRichContent = secondaryRichContent,

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -32,6 +32,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement
     };
 
+    private RealtimeTranscriptionResponse _inputStream = new();
+    private RealtimeTranscriptionResponse _outputStream = new();
+
+
+    private RealtimeHubConnection _conn;
+    private Func<Task> _onModelReady;
+    private Func<string, string, Task> _onModelAudioDeltaReceived;
+    private Func<Task> _onModelAudioResponseDone;
+    private Func<string, Task> _onModelAudioTranscriptDone;
+    private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
+    private Func<string, Task> _onConversationItemCreated;
+    private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
+    private Func<Task> _onInterruptionDetected;
+
     public GoogleRealTimeProvider(
         IServiceProvider services,
         GoogleAiSettings settings,
@@ -58,17 +72,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
         Func<Task> onInterruptionDetected)
     {
+        _conn = conn;
+        _onModelReady = onModelReady;
+        _onModelAudioDeltaReceived = onModelAudioDeltaReceived;
+        _onModelAudioResponseDone = onModelAudioResponseDone;
+        _onModelAudioTranscriptDone = onModelAudioTranscriptDone;
+        _onModelResponseDone = onModelResponseDone;
+        _onConversationItemCreated = onConversationItemCreated;
+        _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
+        _onInterruptionDetected = onInterruptionDetected;
+
         var settingsService = _services.GetRequiredService<ILlmProviderService>();
         var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
 
         _model = realtimeModelSettings.Model;
         var modelSettings = settingsService.GetSetting(Provider, _model);
 
-        if (_session != null)
+        Reset();
-        {
-            _session.Dispose();
-        }
 
+        _inputStream = new();
+        _outputStream = new();
         _session = new LlmRealtimeSession(_services, new ChatSessionOptions
         {
             JsonOptions = _jsonOptions
@@ -79,32 +102,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
 
         await onModelReady();
 
-        _ = ReceiveMessage(
+        _ = ReceiveMessage();
-            conn,
-            onModelReady,
-            onModelAudioDeltaReceived,
-            onModelAudioResponseDone,
-            onModelAudioTranscriptDone,
-            onModelResponseDone,
-            onConversationItemCreated,
-            onInputAudioTranscriptionDone,
-            onInterruptionDetected);
     }
 
-    private async Task ReceiveMessage(
+    private async Task ReceiveMessage()
-        RealtimeHubConnection conn,
-        Func<Task> onModelReady,
-        Func<string, string, Task> onModelAudioDeltaReceived,
-        Func<Task> onModelAudioResponseDone,
-        Func<string, Task> onModelAudioTranscriptDone,
-        Func<List<RoleDialogModel>, Task> onModelResponseDone,
-        Func<string, Task> onConversationItemCreated,
-        Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
-        Func<Task> onInterruptionDetected)
     {
-        using var inputStream = new RealtimeTranscriptionResponse();
-        using var outputStream = new RealtimeTranscriptionResponse();
-
         await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
         {
             var receivedText = update?.RawResponse;
@@ -128,6 +130,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
                 else if (response.SessionResumptionUpdate != null)
                 {
                     _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");
+                    _conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle;
                 }
                 else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
                 {
@@ -137,20 +140,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
 
                     if (functionCall != null)
                     {
-                        var messages = OnFunctionCall(conn, functionCall);
+                        var messages = OnFunctionCall(_conn, functionCall);
-                        await onModelResponseDone(messages);
+                        await _onModelResponseDone(messages);
                     }
                 }
                 else if (response.ServerContent != null)
                 {
                     if (response.ServerContent.InputTranscription?.Text != null)
                     {
-                        inputStream.Collect(response.ServerContent.InputTranscription.Text);
+                        _inputStream.Collect(response.ServerContent.InputTranscription.Text);
                     }
 
                     if (response.ServerContent.OutputTranscription?.Text != null)
                     {
-                        outputStream.Collect(response.ServerContent.OutputTranscription.Text);
+                        _outputStream.Collect(response.ServerContent.OutputTranscription.Text);
                     }
 
                     if (response.ServerContent.ModelTurn != null)
@@ -158,13 +161,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
                         _logger.LogInformation($"Model audio delta received.");
 
                         // Handle input transcription
-                        var inputTranscription = inputStream.GetText();
+                        var inputTranscription = _inputStream.GetText();
                         if (!string.IsNullOrEmpty(inputTranscription))
                         {
-                            var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription);
+                            var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription);
-                            await onInputAudioTranscriptionDone(message);
+                            await _onInputAudioTranscriptionDone(message);
                         }
-                        inputStream.Clear();
+                        _inputStream.Clear();
 
                         var parts = response.ServerContent.ModelTurn.Parts;
                         if (!parts.IsNullOrEmpty())
@@ -173,7 +176,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
                             {
                                 if (!string.IsNullOrEmpty(part.InlineData?.Data))
                                 {
-                                    await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
+                                    await _onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
                                 }
                             }
                         }
@@ -187,14 +190,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
                         _logger.LogInformation($"Model turn completed.");
 
                         // Handle output transcription
-                        var outputTranscription = outputStream.GetText();
+                        var outputTranscription = _outputStream.GetText();
                         if (!string.IsNullOrEmpty(outputTranscription))
                         {
-                            var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);
+                            var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData);
-                            await onModelResponseDone(messages);
+                            await _onModelResponseDone(messages);
                         }
-                        inputStream.Clear();
+                        _inputStream.Clear();
-                        outputStream.Clear();
+                        _outputStream.Clear();
                     }
                 }
             }
@@ -205,6 +208,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
             }
         }
 
+        _inputStream.Dispose();
+        _outputStream.Dispose();
         _session.Dispose();
     }
 
@@ -213,7 +218,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
     {
         if (_session != null)
         {
+            _inputStream?.Dispose();
+            _outputStream?.Dispose();
             await _session.DisconnectAsync();
+            _session.Dispose();
         }
     }
 
@@ -242,8 +250,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
 
     public async Task TriggerModelInference(string? instructions = null)
     {
-        var content = new Content(instructions ?? "Please respond to user.", AgentRole.User);
+        if (string.IsNullOrWhiteSpace(instructions)) return;
 
+        var content = new Content(instructions, AgentRole.User);
         await SendEventToModel(new BidiClientPayload
         {
             ClientContent = new()
@@ -273,13 +282,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
 
     public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
     {
+        if (!isInit)
+        {
+            return null;
+        }
+
         var agentService = _services.GetRequiredService<IAgentService>();
         var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
 
         var agent = await agentService.LoadAgent(conn.CurrentAgentId);
         var (prompt, request) = PrepareOptions(agent, []);
 
-        var config = request.GenerationConfig;
+        var config = request.GenerationConfig ?? new();
         if (config != null)
         {
             //Output Modality can either be text or audio
@@ -317,7 +331,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
             });
         }
 
-        await SendEventToModel(new RealtimeClientPayload
+
+        var payload = new RealtimeClientPayload
         {
             Setup = new RealtimeGenerateContentSetup()
             {
@@ -328,8 +343,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
                 InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
                 OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
                 SessionResumption = new()
+                {
+                    Handle = _conn.PrevSessionId
+                }
             }
-        });
+        };
+
+        Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}");
+        await SendEventToModel(payload);
 
         return prompt;
     }
@@ -596,5 +617,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
     {
         return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}");
     }
+
+    private void Reset()
+    {
+        _inputStream?.Clear();
+        _outputStream?.Clear();
+
+        _inputStream?.Dispose();
+        _outputStream?.Dispose();
+        _session?.Dispose();
+    }
     #endregion
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs

@@ -47,6 +47,8 @@ public class DialogMetaDataMongoElement
     public string MessageId { get; set; } = default!;
     public string MessageType { get; set; } = default!;
     public string? FunctionName { get; set; }
+    public string? FunctionArgs { get; set; }
+    public string? ToolCallId { get; set; }
     public string? SenderId { get; set; }
     public DateTime CreateTime { get; set; }
 
@@ -59,6 +61,8 @@ public class DialogMetaDataMongoElement
             MessageId = meta.MessageId,
             MessageType = meta.MessageType,
             FunctionName = meta.FunctionName,
+            FunctionArgs = meta.FunctionArgs,
+            ToolCallId = meta.ToolCallId,
             SenderId = meta.SenderId,
             CreatedTime = meta.CreateTime,
         };
@@ -73,6 +77,8 @@ public class DialogMetaDataMongoElement
             MessageId = meta.MessageId,
             MessageType = meta.MessageType,
             FunctionName = meta.FunctionName,
+            FunctionArgs = meta.FunctionArgs,
+            ToolCallId = meta.ToolCallId,
             SenderId = meta.SenderId,
             CreateTime = meta.CreatedTime,
         };

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Repositories.Filters;
-using MongoDB.Driver;
 using System.Text.Json;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;