From a3157c24707e87686569e39396d827e6eabfca0c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 17 Jun 2025 13:17:28 -0500 Subject: [PATCH 01/28] init message hub and chathub observers --- Directory.Packages.props | 1 + .../BotSharp.Abstraction.csproj | 1 + .../Observables/Models/HubObserveData.cs | 6 + .../Observables/Models/ObserveDataBase.cs | 6 + .../Conversations/ConversationPlugin.cs | 3 + .../Observables/Queues/MessageHub.cs | 45 +++++++ .../BotSharp.Plugin.ChatHub/ChatHubPlugin.cs | 13 +- .../Observers/ChatHubObserver.cs | 119 ++++++++++++++++++ 8 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Observables/Models/ObserveDataBase.cs create mode 100644 src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs create mode 100644 src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index cec2ac88..38e68d3a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -26,6 +26,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 97e645d0..2008c6a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -36,6 +36,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs b/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs new file mode 100644 index 00000000..a6d10cce --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Observables.Models; + +public class HubObserveData : ObserveDataBase +{ + public RoleDialogModel Data { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Observables/Models/ObserveDataBase.cs b/src/Infrastructure/BotSharp.Abstraction/Observables/Models/ObserveDataBase.cs new file mode 100644 index 00000000..17773272 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Observables/Models/ObserveDataBase.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Observables.Models; + +public abstract class ObserveDataBase +{ + public IServiceProvider ServiceProvider { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 13ee1de6..e8e18eb8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -10,6 +10,7 @@ using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Reasoning; using BotSharp.Core.Templating; using BotSharp.Core.Translation; +using BotSharp.Core.Observables.Queues; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Conversations; @@ -41,6 +42,8 @@ public class ConversationPlugin : IBotSharpPlugin return settingService.Bind("GoogleApi"); }); + services.AddSingleton(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs new file mode 100644 index 00000000..ed6772b4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs @@ -0,0 +1,45 @@ +using BotSharp.Abstraction.Observables.Models; +using System.Reactive.Subjects; + +namespace BotSharp.Core.Observables.Queues; + +public class MessageHub +{ + private readonly ILogger _logger; + private readonly ISubject _observable = new Subject(); + public IObservable Events => _observable; + + public MessageHub(ILogger logger) + { + _logger = logger; + } + + /// + /// Push an item to the observers. + /// + /// + public void Push(HubObserveData item) + { + _logger.LogInformation($"Pushing item to observers: {item.Data.Content}"); + _observable.OnNext(item); + } + + /// + /// Send a complete notification to the observers. + /// This will stop the observers from receiving data. + /// + public void Complete() + { + _observable.OnCompleted(); + } + + /// + /// Send an error notification to the observers. + /// This will stop the observers from receiving data. + /// + /// + public void Error(Exception error) + { + _observable.OnError(error); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index 725655fc..b1fb144f 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -1,5 +1,8 @@ using BotSharp.Abstraction.Crontab; +using BotSharp.Core.Observables.Queues; using BotSharp.Plugin.ChatHub.Hooks; +using BotSharp.Plugin.ChatHub.Observers; +using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.ChatHub; @@ -7,7 +10,7 @@ namespace BotSharp.Plugin.ChatHub; /// /// The dialogue channel connects users, AI assistants and customer service representatives. /// -public class ChatHubPlugin : IBotSharpPlugin +public class ChatHubPlugin : IBotSharpPlugin, IBotSharpAppPlugin { public string Id => "6e52d42d-1e23-406b-8599-36af36c83209"; public string Name => "Chat Hub"; @@ -28,4 +31,12 @@ public class ChatHubPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); } + + public void Configure(IApplicationBuilder app) + { + var services = app.ApplicationServices; + var queue = services.GetRequiredService(); + var logger = services.GetRequiredService>(); + queue.Events.Subscribe(new ChatHubObserver(logger)); + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs new file mode 100644 index 00000000..5699945f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -0,0 +1,119 @@ +using BotSharp.Abstraction.Conversations.Dtos; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Abstraction.SideCar; +using BotSharp.Abstraction.Users.Dtos; +using BotSharp.Plugin.ChatHub.Hooks; +using Microsoft.AspNetCore.SignalR; + +namespace BotSharp.Plugin.ChatHub.Observers; + +public class ChatHubObserver : IObserver +{ + private readonly ILogger _logger; + private IServiceProvider _services; + private IUserIdentity _user; + + private const string RECEIVE_CLIENT_MESSAGE = "OnMessageReceivedFromClient"; + private const string GENERATE_SENDER_ACTION = "OnSenderActionGenerated"; + + public ChatHubObserver(ILogger logger) + { + _logger = logger; + } + + public void OnCompleted() + { + _logger.LogInformation($"{nameof(ChatHubObserver)} receives complete notification."); + } + + public void OnError(Exception error) + { + _logger.LogError(error, $"{nameof(ChatHubObserver)} receives error notification: {error.Message}"); + } + + public void OnNext(HubObserveData value) + { + _services = value.ServiceProvider; + _user = _services.GetRequiredService(); + + ReceiveMessage(value.Data).ConfigureAwait(false).GetAwaiter().GetResult(); + } + + private async Task ReceiveMessage(RoleDialogModel message) + { + if (!AllowSendingMessage()) return; + + var conv = _services.GetRequiredService(); + var userService = _services.GetRequiredService(); + var sender = await userService.GetMyProfile(); + + // Update console conversation UI for CSR + var model = new ChatResponseDto() + { + ConversationId = conv.ConversationId, + MessageId = message.MessageId, + Payload = message.Payload, + Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, + Sender = UserDto.FromUser(sender) + }; + await ReceiveClientMessage(conv.ConversationId, model); + + // Send typing-on to client + var action = new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOn + }; + + await GenerateSenderAction(conv.ConversationId, action); + } + + private async Task ReceiveClientMessage(string conversationId, ChatResponseDto model) + { + try + { + var settings = _services.GetRequiredService(); + var chatHub = _services.GetRequiredService>(); + + if (settings.EventDispatchBy == EventDispatchType.Group) + { + await chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } + else + { + await chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})"); + } + } + + private bool AllowSendingMessage() + { + var sidecar = _services.GetService(); + return sidecar == null || !sidecar.IsEnabled(); + } + + private async Task GenerateSenderAction(string conversationId, ConversationSenderActionModel action) + { + try + { + var settings = _services.GetRequiredService(); + var chatHub = _services.GetRequiredService>(); + if (settings.EventDispatchBy == EventDispatchType.Group) + { + await chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + } + else + { + await chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to generate sender action in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})"); + } + } +} From f4a74258c23edc595a81cabe185ea0252e8e3e8b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 17 Jun 2025 17:57:57 -0500 Subject: [PATCH 02/28] temp save --- .../Conversations/IConversationService.cs | 7 +- .../Observables/Models/HubObserveData.cs | 1 + .../Routing/IRoutingService.cs | 1 + .../Services/ConversationService.Stream.cs | 86 ++++++++++++ .../Observables/Queues/MessageHub.cs | 1 - .../Routing/RoutingService.InstructStream.cs | 49 +++++++ .../Controllers/ConversationController.cs | 24 ++++ .../Observers/ChatHubObserver.cs | 125 ++++++++++++++---- .../Providers/Chat/ChatCompletionProvider.cs | 53 +++++++- 9 files changed, 309 insertions(+), 38 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 656b397d..fccab0fa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -36,10 +36,15 @@ public interface IConversationService /// Received the response from AI Agent /// Task SendMessage(string agentId, - RoleDialogModel lastDialog, + RoleDialogModel message, PostbackMessageModel? replyMessage, Func onResponseReceived); + + Task StreamMessage(string agentId, + RoleDialogModel lastDialog, + PostbackMessageModel? replyMessage); + List GetDialogHistory(int lastCount = 100, bool fromBreakpoint = true, IEnumerable? includeMessageTypes = null); Task CleanHistory(string agentId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs b/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs index a6d10cce..bf771cb4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Observables/Models/HubObserveData.cs @@ -2,5 +2,6 @@ namespace BotSharp.Abstraction.Observables.Models; public class HubObserveData : ObserveDataBase { + public string EventName { get; set; } = null!; public RoleDialogModel Data { get; set; } = null!; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index 97be8874..5dbf8d3f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -41,6 +41,7 @@ public interface IRoutingService /// /// Task InstructDirect(Agent agent, RoleDialogModel message, List dialogs); + Task InstructStream(Agent agent, RoleDialogModel message, List dialogs); Task GetConversationContent(List dialogs, int maxDialogCount = 100); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs new file mode 100644 index 00000000..f650a28f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs @@ -0,0 +1,86 @@ +using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Routing.Enums; +using BotSharp.Abstraction.Routing.Settings; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task StreamMessage(string agentId, + RoleDialogModel message, + PostbackMessageModel? replyMessage) + { + var conversation = await GetConversationRecordOrCreateNew(agentId); + var agentService = _services.GetRequiredService(); + Agent agent = await agentService.LoadAgent(agentId); + + var content = $"Received [{agent.Name}] {message.Role}: {message.Content}"; + _logger.LogInformation(content); + + message.CurrentAgentId = agent.Id; + if (string.IsNullOrEmpty(message.SenderId)) + { + message.SenderId = _user.Id; + } + + var conv = _services.GetRequiredService(); + var dialogs = conv.GetDialogHistory(); + + var statistics = _services.GetRequiredService(); + + RoleDialogModel response = message; + bool stopCompletion = false; + + // Enqueue receiving agent first in case it stop completion by OnMessageReceived + var routing = _services.GetRequiredService(); + routing.Context.SetMessageId(_conversationId, message.MessageId); + + // Save payload in order to assign the payload before hook is invoked + if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) + { + message.Payload = replyMessage.Payload; + } + + var hooks = _services.GetHooksOrderByPriority(message.CurrentAgentId); + foreach (var hook in hooks) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + if (replyMessage == null || string.IsNullOrEmpty(replyMessage.FunctionName)) + { + await hook.OnMessageReceived(message); + } + else + { + await hook.OnPostbackMessageReceived(message, replyMessage); + } + + // Interrupted by hook + if (message.StopCompletion) + { + stopCompletion = true; + routing.Context.Pop(); + break; + } + } + + if (!stopCompletion) + { + // Routing with reasoning + var settings = _services.GetRequiredService(); + + // reload agent in case it has been changed by hook + if (message.CurrentAgentId != agent.Id) + { + agent = await agentService.LoadAgent(message.CurrentAgentId); + } + + await routing.InstructStream(agent, message, dialogs); + routing.Context.ResetRecursiveCounter(); + } + + return true; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs index ed6772b4..9950a613 100644 --- a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs +++ b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs @@ -20,7 +20,6 @@ public class MessageHub /// public void Push(HubObserveData item) { - _logger.LogInformation($"Pushing item to observers: {item.Data.Content}"); _observable.OnNext(item); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs new file mode 100644 index 00000000..08c72be1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs @@ -0,0 +1,49 @@ +namespace BotSharp.Core.Routing; + +public partial class RoutingService +{ + public async Task InstructStream(Agent agent, RoleDialogModel message, List dialogs) + { + var conv = _services.GetRequiredService(); + var storage = _services.GetRequiredService(); + storage.Append(conv.ConversationId, message); + + dialogs.Add(message); + Context.SetDialogs(dialogs); + + var routing = _services.GetRequiredService(); + routing.Context.Push(agent.Id, "instruct directly"); + var agentId = routing.Context.GetCurrentAgentId(); + + // Update next action agent's name + var agentService = _services.GetRequiredService(); + + if (agent.Disabled) + { + var content = $"This agent ({agent.Name}) is disabled, please install the corresponding plugin ({agent.Plugin.Name}) to activate this agent."; + + message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: content); + dialogs.Add(message); + } + else + { + var provider = agent.LlmConfig.Provider; + var model = agent.LlmConfig.Model; + + if (provider == null || model == null) + { + var agentSettings = _services.GetRequiredService(); + provider = agentSettings.LlmConfig.Provider; + model = agentSettings.LlmConfig.Model; + } + + var chatCompletion = CompletionProvider.GetChatCompletion(_services, + provider: provider, + model: model); + + await chatCompletion.GetChatCompletionsStreamingAsync(agent, dialogs, async data => { }); + } + + return true; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index d3da550f..a3cedc1e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -377,6 +377,30 @@ public class ConversationController : ControllerBase return response; } + + [HttpPost("/conversation/{agentId}/{conversationId}/stream")] + public async Task StreamMessage( + [FromRoute] string agentId, + [FromRoute] string conversationId, + [FromBody] NewMessageModel input) + { + var conv = _services.GetRequiredService(); + var inputMsg = new RoleDialogModel(AgentRole.User, input.Text) + { + MessageId = !string.IsNullOrWhiteSpace(input.InputMessageId) ? input.InputMessageId : Guid.NewGuid().ToString(), + CreatedAt = DateTime.UtcNow + }; + + var routing = _services.GetRequiredService(); + routing.Context.SetMessageId(conversationId, inputMsg.MessageId); + + conv.SetConversationId(conversationId, input.States); + SetStates(conv, input); + + await conv.StreamMessage(agentId, inputMsg, replyMessage: input.Postback); + } + + [HttpPost("/conversation/{agentId}/{conversationId}/sse")] public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) { diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs index 5699945f..dcf332b0 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Conversations.Dtos; using BotSharp.Abstraction.Observables.Models; using BotSharp.Abstraction.SideCar; -using BotSharp.Abstraction.Users.Dtos; using BotSharp.Plugin.ChatHub.Hooks; using Microsoft.AspNetCore.SignalR; @@ -11,9 +10,10 @@ public class ChatHubObserver : IObserver { private readonly ILogger _logger; private IServiceProvider _services; - private IUserIdentity _user; - private const string RECEIVE_CLIENT_MESSAGE = "OnMessageReceivedFromClient"; + private const string BEFORE_RECEIVE_LLM_STREAM_MESSAGE = "BeforeReceiveLlmStreamMessage"; + private const string ON_RECEIVE_LLM_STREAM_MESSAGE = "OnReceiveLlmStreamMessage"; + private const string AFTER_RECEIVE_LLM_STREAM_MESSAGE = "AfterReceiveLlmStreamMessage"; private const string GENERATE_SENDER_ACTION = "OnSenderActionGenerated"; public ChatHubObserver(ILogger logger) @@ -34,41 +34,106 @@ public class ChatHubObserver : IObserver public void OnNext(HubObserveData value) { _services = value.ServiceProvider; - _user = _services.GetRequiredService(); - - ReceiveMessage(value.Data).ConfigureAwait(false).GetAwaiter().GetResult(); + + var message = value.Data; + var model = new ChatResponseDto(); + if (value.EventName == BEFORE_RECEIVE_LLM_STREAM_MESSAGE + || value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE) + { + var conv = _services.GetRequiredService(); + model = new ChatResponseDto() + { + ConversationId = conv.ConversationId, + MessageId = message.MessageId, + Text = string.Empty, + Sender = new() + { + FirstName = "AI", + LastName = "Assistant", + Role = AgentRole.Assistant + } + }; + + var action = new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = value.EventName == BEFORE_RECEIVE_LLM_STREAM_MESSAGE ? SenderActionEnum.TypingOn : SenderActionEnum.TypingOff + }; + + GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); + } + else if (value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE) + { + //var conv = _services.GetRequiredService(); + //model = new ChatResponseDto() + //{ + // ConversationId = conv.ConversationId, + // MessageId = message.MessageId, + // Text = string.Empty, + // Sender = new() + // { + // FirstName = "AI", + // LastName = "Assistant", + // Role = AgentRole.Assistant + // } + //}; + + //var action = new ConversationSenderActionModel + //{ + // ConversationId = conv.ConversationId, + // SenderAction = SenderActionEnum.TypingOff + //}; + + //GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); + + //var storage = _services.GetRequiredService(); + //storage.Append(conv.ConversationId, message); + } + else if (value.EventName == ON_RECEIVE_LLM_STREAM_MESSAGE) + { + var conv = _services.GetRequiredService(); + model = new ChatResponseDto() + { + ConversationId = conv.ConversationId, + MessageId = message.MessageId, + Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, + Function = message.FunctionName, + RichContent = message.SecondaryRichContent ?? message.RichContent, + Data = message.Data, + Sender = new() + { + FirstName = "AI", + LastName = "Assistant", + Role = AgentRole.Assistant + } + }; + } + + OnReceiveAssistantMessage(value.EventName, model.ConversationId, model).ConfigureAwait(false).GetAwaiter().GetResult(); } - private async Task ReceiveMessage(RoleDialogModel message) + private async Task ReceiveLlmStreamResponse(RoleDialogModel message) { - if (!AllowSendingMessage()) return; - var conv = _services.GetRequiredService(); - var userService = _services.GetRequiredService(); - var sender = await userService.GetMyProfile(); - - // Update console conversation UI for CSR var model = new ChatResponseDto() { ConversationId = conv.ConversationId, MessageId = message.MessageId, - Payload = message.Payload, Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, - Sender = UserDto.FromUser(sender) + Function = message.FunctionName, + RichContent = message.SecondaryRichContent ?? message.RichContent, + Data = message.Data, + Sender = new() + { + FirstName = "AI", + LastName = "Assistant", + Role = AgentRole.Assistant + } }; - await ReceiveClientMessage(conv.ConversationId, model); - - // Send typing-on to client - var action = new ConversationSenderActionModel - { - ConversationId = conv.ConversationId, - SenderAction = SenderActionEnum.TypingOn - }; - - await GenerateSenderAction(conv.ConversationId, action); + await OnReceiveAssistantMessage(ON_RECEIVE_LLM_STREAM_MESSAGE, conv.ConversationId, model); } - private async Task ReceiveClientMessage(string conversationId, ChatResponseDto model) + private async Task OnReceiveAssistantMessage(string @event, string conversationId, ChatResponseDto model) { try { @@ -77,11 +142,12 @@ public class ChatHubObserver : IObserver if (settings.EventDispatchBy == EventDispatchType.Group) { - await chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + await chatHub.Clients.Group(conversationId).SendAsync(@event, model); } else { - await chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + var user = _services.GetRequiredService(); + await chatHub.Clients.User(user.Id).SendAsync(@event, model); } } catch (Exception ex) @@ -108,7 +174,8 @@ public class ChatHubObserver : IObserver } else { - await chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + var user = _services.GetRequiredService(); + await chatHub.Clients.User(user.Id).SendAsync(GENERATE_SENDER_ACTION, action); } } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 17ee1012..83a96266 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Hooks; +using BotSharp.Core.Observables.Queues; +using ModelContextProtocol.Protocol.Types; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -185,7 +187,20 @@ public class ChatCompletionProvider : IChatCompletion var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); + var hub = _services.GetRequiredService(); var response = chatClient.CompleteChatStreamingAsync(messages, options); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); await foreach (var choice in response) { @@ -194,23 +209,47 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; _logger.LogInformation(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); + //await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + //{ + // //RenderedInstruction = string.Join("\r\n", renderedInstructions) + //}); continue; } if (choice.ContentUpdate.IsNullOrEmpty()) continue; - _logger.LogInformation(choice.ContentUpdate[0]?.Text); + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + _logger.LogInformation(text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + var content = new RoleDialogModel(AgentRole.Assistant, text) { - RenderedInstruction = string.Join("\r\n", renderedInstructions) + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content }); + + //await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + //{ + // RenderedInstruction = string.Join("\r\n", renderedInstructions) + //}); } + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + return true; } From eeb4f06f639638c2f0ef641282f332d541a6de8a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 17 Jun 2025 22:23:43 -0500 Subject: [PATCH 03/28] refine stream --- .../Observers/ChatHubObserver.cs | 72 +++++++------------ .../Providers/Chat/ChatCompletionProvider.cs | 4 +- 2 files changed, 29 insertions(+), 47 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs index dcf332b0..c7a852fa 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -23,7 +23,7 @@ public class ChatHubObserver : IObserver public void OnCompleted() { - _logger.LogInformation($"{nameof(ChatHubObserver)} receives complete notification."); + _logger.LogWarning($"{nameof(ChatHubObserver)} receives complete notification."); } public void OnError(Exception error) @@ -35,10 +35,11 @@ public class ChatHubObserver : IObserver { _services = value.ServiceProvider; + if (!AllowSendingMessage()) return; + var message = value.Data; var model = new ChatResponseDto(); - if (value.EventName == BEFORE_RECEIVE_LLM_STREAM_MESSAGE - || value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE) + if (value.EventName == BEFORE_RECEIVE_LLM_STREAM_MESSAGE) { var conv = _services.GetRequiredService(); model = new ChatResponseDto() @@ -57,37 +58,37 @@ public class ChatHubObserver : IObserver var action = new ConversationSenderActionModel { ConversationId = conv.ConversationId, - SenderAction = value.EventName == BEFORE_RECEIVE_LLM_STREAM_MESSAGE ? SenderActionEnum.TypingOn : SenderActionEnum.TypingOff + SenderAction = SenderActionEnum.TypingOn }; GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); } else if (value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE) { - //var conv = _services.GetRequiredService(); - //model = new ChatResponseDto() - //{ - // ConversationId = conv.ConversationId, - // MessageId = message.MessageId, - // Text = string.Empty, - // Sender = new() - // { - // FirstName = "AI", - // LastName = "Assistant", - // Role = AgentRole.Assistant - // } - //}; + var conv = _services.GetRequiredService(); + model = new ChatResponseDto() + { + ConversationId = conv.ConversationId, + MessageId = message.MessageId, + Text = message.Content, + Sender = new() + { + FirstName = "AI", + LastName = "Assistant", + Role = AgentRole.Assistant + } + }; - //var action = new ConversationSenderActionModel - //{ - // ConversationId = conv.ConversationId, - // SenderAction = SenderActionEnum.TypingOff - //}; + var action = new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOff + }; - //GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); + GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); - //var storage = _services.GetRequiredService(); - //storage.Append(conv.ConversationId, message); + var storage = _services.GetRequiredService(); + storage.Append(conv.ConversationId, message); } else if (value.EventName == ON_RECEIVE_LLM_STREAM_MESSAGE) { @@ -112,27 +113,6 @@ public class ChatHubObserver : IObserver OnReceiveAssistantMessage(value.EventName, model.ConversationId, model).ConfigureAwait(false).GetAwaiter().GetResult(); } - private async Task ReceiveLlmStreamResponse(RoleDialogModel message) - { - var conv = _services.GetRequiredService(); - var model = new ChatResponseDto() - { - ConversationId = conv.ConversationId, - MessageId = message.MessageId, - Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, - Function = message.FunctionName, - RichContent = message.SecondaryRichContent ?? message.RichContent, - Data = message.Data, - Sender = new() - { - FirstName = "AI", - LastName = "Assistant", - Role = AgentRole.Assistant - } - }; - await OnReceiveAssistantMessage(ON_RECEIVE_LLM_STREAM_MESSAGE, conv.ConversationId, model); - } - private async Task OnReceiveAssistantMessage(string @event, string conversationId, ChatResponseDto model) { try diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 83a96266..67ac7c66 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -190,6 +190,7 @@ public class ChatCompletionProvider : IChatCompletion var hub = _services.GetRequiredService(); var response = chatClient.CompleteChatStreamingAsync(messages, options); var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + var allText = string.Empty; hub.Push(new() { @@ -219,6 +220,7 @@ public class ChatCompletionProvider : IChatCompletion if (choice.ContentUpdate.IsNullOrEmpty()) continue; var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + allText += text; _logger.LogInformation(text); var content = new RoleDialogModel(AgentRole.Assistant, text) @@ -243,7 +245,7 @@ public class ChatCompletionProvider : IChatCompletion { ServiceProvider = _services, EventName = "AfterReceiveLlmStreamMessage", - Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + Data = new RoleDialogModel(AgentRole.Assistant, allText) { CurrentAgentId = agent.Id, MessageId = messageId From 1b32cb5c68ada29c2c1952344da70054abdb0875 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 18 Jun 2025 15:08:52 -0500 Subject: [PATCH 04/28] temp save --- .../Providers/Chat/ChatCompletionProvider.cs | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 67ac7c66..e3ea89fe 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -208,37 +208,36 @@ public class ChatCompletionProvider : IChatCompletion if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) { var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; - _logger.LogInformation(update); + _logger.LogCritical($"Tool Call (reason: {choice.FinishReason}): {update}"); //await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) //{ // //RenderedInstruction = string.Join("\r\n", renderedInstructions) //}); - continue; } - - if (choice.ContentUpdate.IsNullOrEmpty()) continue; - - var text = choice.ContentUpdate[0]?.Text ?? string.Empty; - allText += text; - _logger.LogInformation(text); - - var content = new RoleDialogModel(AgentRole.Assistant, text) + else if (!choice.ContentUpdate.IsNullOrEmpty()) { - CurrentAgentId = agent.Id, - MessageId = messageId - }; - hub.Push(new() - { - ServiceProvider = _services, - EventName = "OnReceiveLlmStreamMessage", - Data = content - }); + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + allText += text; + _logger.LogCritical($"Content update (reason: {choice.FinishReason}) {text}"); - //await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) - //{ - // RenderedInstruction = string.Join("\r\n", renderedInstructions) - //}); + var content = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); + + //await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + //{ + // RenderedInstruction = string.Join("\r\n", renderedInstructions) + //}); + } } hub.Push(new() From 5541500ed22e3d20304c6722f52f17916362c7a9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 18 Jun 2025 16:42:58 -0500 Subject: [PATCH 05/28] init tool calling --- .../Streams/RealtimeTextStream.cs} | 15 +++++-- .../Realtime/RealTimeCompletionProvider.cs | 7 +-- .../Providers/Chat/ChatCompletionProvider.cs | 44 ++++++++++++------- 3 files changed, 42 insertions(+), 24 deletions(-) rename src/{Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs => Infrastructure/BotSharp.Core/Infrastructures/Streams/RealtimeTextStream.cs} (81%) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Streams/RealtimeTextStream.cs similarity index 81% rename from src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs rename to src/Infrastructure/BotSharp.Core/Infrastructures/Streams/RealtimeTextStream.cs index 0a383c80..e041a53b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Streams/RealtimeTextStream.cs @@ -1,12 +1,12 @@ using System.IO; -namespace BotSharp.Plugin.GoogleAI.Models.Realtime; +namespace BotSharp.Core.Infrastructures.Streams; -internal class RealtimeTranscriptionResponse : IDisposable +public class RealtimeTextStream : IDisposable { - public RealtimeTranscriptionResponse() + public RealtimeTextStream() { - + } private bool _disposed = false; @@ -20,6 +20,13 @@ internal class RealtimeTranscriptionResponse : IDisposable } } + public long Length => _contentStream.Length; + + public bool IsNullOrEmpty() + { + return _contentStream == null || Length == 0; + } + public void Collect(string text) { if (_disposed) return; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 2e95cfa2..f262efd7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,11 +1,12 @@ -using System.Threading; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Realtime.Models.Session; +using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Session; using BotSharp.Plugin.GoogleAI.Models.Realtime; using GenerativeAI; using GenerativeAI.Types; using GenerativeAI.Types.Converters; +using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -33,8 +34,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement }; - private RealtimeTranscriptionResponse _inputStream = new(); - private RealtimeTranscriptionResponse _outputStream = new(); + private RealtimeTextStream _inputStream = new(); + private RealtimeTextStream _outputStream = new(); private bool _isBlocking = false; private RealtimeHubConnection _conn; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index e3ea89fe..8643ebc2 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,7 +1,10 @@ using BotSharp.Abstraction.Hooks; +using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Observables.Queues; +using EntityFrameworkCore.BootKit; using ModelContextProtocol.Protocol.Types; using OpenAI.Chat; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -188,9 +191,7 @@ public class ChatCompletionProvider : IChatCompletion var (prompt, messages, options) = PrepareOptions(agent, conversations); var hub = _services.GetRequiredService(); - var response = chatClient.CompleteChatStreamingAsync(messages, options); var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; - var allText = string.Empty; hub.Push(new() { @@ -203,23 +204,30 @@ public class ChatCompletionProvider : IChatCompletion } }); - await foreach (var choice in response) + using var textStream = new RealtimeTextStream(); + var toolCalls = new List(); + + await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) { + if (choice.ToolCallUpdates != null) + { + toolCalls.AddRange(choice.ToolCallUpdates); + } + if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) { - var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; - _logger.LogCritical($"Tool Call (reason: {choice.FinishReason}): {update}"); + var functionName = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName))?.FunctionName; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); - //await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - //{ - // //RenderedInstruction = string.Join("\r\n", renderedInstructions) - //}); + _logger.LogCritical($"Tool Call: {functionName}({functionArgument})"); } else if (!choice.ContentUpdate.IsNullOrEmpty()) { var text = choice.ContentUpdate[0]?.Text ?? string.Empty; - allText += text; - _logger.LogCritical($"Content update (reason: {choice.FinishReason}) {text}"); + textStream.Collect(text); + + _logger.LogCritical($"Content update: {text}"); var content = new RoleDialogModel(AgentRole.Assistant, text) { @@ -232,11 +240,6 @@ public class ChatCompletionProvider : IChatCompletion EventName = "OnReceiveLlmStreamMessage", Data = content }); - - //await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) - //{ - // RenderedInstruction = string.Join("\r\n", renderedInstructions) - //}); } } @@ -244,7 +247,7 @@ public class ChatCompletionProvider : IChatCompletion { ServiceProvider = _services, EventName = "AfterReceiveLlmStreamMessage", - Data = new RoleDialogModel(AgentRole.Assistant, allText) + Data = new RoleDialogModel(AgentRole.Assistant, textStream.GetText()) { CurrentAgentId = agent.Id, MessageId = messageId @@ -452,4 +455,11 @@ public class ChatCompletionProvider : IChatCompletion { _model = model; } +} + + +class ToolCallData +{ + public ChatFinishReason? Reason { get; set; } + public List ToolCalls { get; set; } = []; } \ No newline at end of file From 0ef474aea5dc8c430596b796c1097686f9af5e7b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 18 Jun 2025 16:45:42 -0500 Subject: [PATCH 06/28] get toolcall id --- .../Providers/Chat/ChatCompletionProvider.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 8643ebc2..f9b7df13 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -216,11 +216,13 @@ public class ChatCompletionProvider : IChatCompletion if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) { - var functionName = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName))?.FunctionName; + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); var functionArgument = string.Join(string.Empty, args); - _logger.LogCritical($"Tool Call: {functionName}({functionArgument})"); + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); } else if (!choice.ContentUpdate.IsNullOrEmpty()) { From d62340732dff245e85897730b4851020ec5ae7de Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 19 Jun 2025 12:20:12 -0500 Subject: [PATCH 07/28] refine streaming --- .../Providers/Chat/ChatCompletionProvider.cs | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index f9b7df13..af207e12 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -2,9 +2,12 @@ using BotSharp.Abstraction.Hooks; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Observables.Queues; using EntityFrameworkCore.BootKit; +using Fluid; using ModelContextProtocol.Protocol.Types; using OpenAI.Chat; +using System.Xml; using static Microsoft.EntityFrameworkCore.DbLoggerCategory; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -206,25 +209,20 @@ public class ChatCompletionProvider : IChatCompletion using var textStream = new RealtimeTextStream(); var toolCalls = new List(); + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) { - if (choice.ToolCallUpdates != null) + if (!choice.ToolCallUpdates.IsNullOrEmpty()) { toolCalls.AddRange(choice.ToolCallUpdates); } - if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) - { - var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); - var functionName = meta?.FunctionName; - var toolCallId = meta?.ToolCallId; - var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); - var functionArgument = string.Join(string.Empty, args); - - _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); - } - else if (!choice.ContentUpdate.IsNullOrEmpty()) + if (!choice.ContentUpdate.IsNullOrEmpty()) { var text = choice.ContentUpdate[0]?.Text ?? string.Empty; textStream.Collect(text); @@ -243,17 +241,45 @@ public class ChatCompletionProvider : IChatCompletion Data = content }); } + + if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) + { + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); + + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); + + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = toolCallId, + FunctionName = functionName, + FunctionArgs = functionArgument + }; + + } + else if (choice.FinishReason.HasValue) + { + var allText = textStream.GetText(); + _logger.LogCritical($"Text Content: {allText}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + } } hub.Push(new() { ServiceProvider = _services, EventName = "AfterReceiveLlmStreamMessage", - Data = new RoleDialogModel(AgentRole.Assistant, textStream.GetText()) - { - CurrentAgentId = agent.Id, - MessageId = messageId - } + Data = responseMessage }); return true; From 5551561a07f8b3b1953e185723c09f23bd639bdd Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 20 Jun 2025 14:57:16 -0500 Subject: [PATCH 08/28] change method name --- .../BotSharp.OpenAPI/Controllers/ConversationController.cs | 2 +- .../Providers/Chat/ChatCompletionProvider.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index a3cedc1e..0552b19f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -379,7 +379,7 @@ public class ConversationController : ControllerBase [HttpPost("/conversation/{agentId}/{conversationId}/stream")] - public async Task StreamMessage( + public async Task SendMessageStream( [FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index af207e12..38d1455e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -242,7 +242,7 @@ public class ChatCompletionProvider : IChatCompletion }); } - if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) + if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) { var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); var functionName = meta?.FunctionName; From 7008667a1ae7042eb919406480b1f4c97ccdde7e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 23 Jun 2025 21:43:25 -0500 Subject: [PATCH 09/28] support invoke function --- .../Conversations/IConversationService.cs | 5 -- .../Conversations/Models/RoleDialogModel.cs | 6 +- .../Settings/ConversationSetting.cs | 1 + .../MLTasks/IChatCompletion.cs | 5 +- .../Routing/IRoutingService.cs | 3 +- .../Services/ConversationService.Stream.cs | 86 ------------------- .../Demo/Functions/GetWeatherFn.cs | 2 +- .../Routing/Reasoning/InstructExecutor.cs | 3 +- .../Routing/RoutingService.InstructStream.cs | 49 ----------- .../Routing/RoutingService.InvokeAgent.cs | 20 +++-- .../BotSharp.Core/Routing/RoutingService.cs | 3 +- .../Controllers/ConversationController.cs | 23 ----- .../Hooks/ChatHubConversationHook.cs | 2 +- .../Observers/ChatHubObserver.cs | 40 ++++----- .../Providers/Chat/ChatCompletionProvider.cs | 17 ++-- .../BotSharp.LLM.Tests/ChatCompletionTests.cs | 14 +-- 16 files changed, 66 insertions(+), 213 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index fccab0fa..322d86b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -40,11 +40,6 @@ public interface IConversationService PostbackMessageModel? replyMessage, Func onResponseReceived); - - Task StreamMessage(string agentId, - RoleDialogModel lastDialog, - PostbackMessageModel? replyMessage); - List GetDialogHistory(int lastCount = 100, bool fromBreakpoint = true, IEnumerable? includeMessageTypes = null); Task CleanHistory(string agentId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index de269759..6a8af2d6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -117,6 +117,9 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string RenderedInstruction { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool IsStreaming { get; set; } + private RoleDialogModel() { } @@ -159,7 +162,8 @@ public class RoleDialogModel : ITrackableMessage Payload = source.Payload, StopCompletion = source.StopCompletion, Instruction = source.Instruction, - Data = source.Data + Data = source.Data, + IsStreaming = source.IsStreaming }; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index c4f131dc..980569dc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -12,6 +12,7 @@ public class ConversationSetting public bool EnableContentLog { get; set; } public bool EnableStateLog { get; set; } public bool EnableTranslationMemory { get; set; } + public bool EnableStreaming { get; set; } public CleanConversationSetting CleanSetting { get; set; } = new(); public RateLimitSetting RateLimit { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 7cf52fe4..4d93f307 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -23,7 +23,6 @@ public interface IChatCompletion Func onMessageReceived, Func onFunctionExecuting); - Task GetChatCompletionsStreamingAsync(Agent agent, - List conversations, - Func onMessageReceived); + Task GetChatCompletionsStreamingAsync(Agent agent, + List conversations) => Task.FromResult(new RoleDialogModel(AgentRole.Assistant, string.Empty)); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index 5dbf8d3f..fb542d5f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -30,7 +30,7 @@ public interface IRoutingService //int GetRecursiveCounter(); //void SetRecursiveCounter(int counter); - Task InvokeAgent(string agentId, List dialogs); + Task InvokeAgent(string agentId, List dialogs, bool useStream = false); Task InvokeFunction(string name, RoleDialogModel messages); Task InstructLoop(Agent agent, RoleDialogModel message, List dialogs); @@ -41,7 +41,6 @@ public interface IRoutingService /// /// Task InstructDirect(Agent agent, RoleDialogModel message, List dialogs); - Task InstructStream(Agent agent, RoleDialogModel message, List dialogs); Task GetConversationContent(List dialogs, int maxDialogCount = 100); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs deleted file mode 100644 index f650a28f..00000000 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Stream.cs +++ /dev/null @@ -1,86 +0,0 @@ -using BotSharp.Abstraction.Hooks; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Abstraction.Routing.Enums; -using BotSharp.Abstraction.Routing.Settings; - -namespace BotSharp.Core.Conversations.Services; - -public partial class ConversationService -{ - public async Task StreamMessage(string agentId, - RoleDialogModel message, - PostbackMessageModel? replyMessage) - { - var conversation = await GetConversationRecordOrCreateNew(agentId); - var agentService = _services.GetRequiredService(); - Agent agent = await agentService.LoadAgent(agentId); - - var content = $"Received [{agent.Name}] {message.Role}: {message.Content}"; - _logger.LogInformation(content); - - message.CurrentAgentId = agent.Id; - if (string.IsNullOrEmpty(message.SenderId)) - { - message.SenderId = _user.Id; - } - - var conv = _services.GetRequiredService(); - var dialogs = conv.GetDialogHistory(); - - var statistics = _services.GetRequiredService(); - - RoleDialogModel response = message; - bool stopCompletion = false; - - // Enqueue receiving agent first in case it stop completion by OnMessageReceived - var routing = _services.GetRequiredService(); - routing.Context.SetMessageId(_conversationId, message.MessageId); - - // Save payload in order to assign the payload before hook is invoked - if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) - { - message.Payload = replyMessage.Payload; - } - - var hooks = _services.GetHooksOrderByPriority(message.CurrentAgentId); - foreach (var hook in hooks) - { - hook.SetAgent(agent) - .SetConversation(conversation); - - if (replyMessage == null || string.IsNullOrEmpty(replyMessage.FunctionName)) - { - await hook.OnMessageReceived(message); - } - else - { - await hook.OnPostbackMessageReceived(message, replyMessage); - } - - // Interrupted by hook - if (message.StopCompletion) - { - stopCompletion = true; - routing.Context.Pop(); - break; - } - } - - if (!stopCompletion) - { - // Routing with reasoning - var settings = _services.GetRequiredService(); - - // reload agent in case it has been changed by hook - if (message.CurrentAgentId != agent.Id) - { - agent = await agentService.LoadAgent(message.CurrentAgentId); - } - - await routing.InstructStream(agent, message, dialogs); - routing.Context.ResetRecursiveCounter(); - } - - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs index a78ad4ec..cb6bcd88 100644 --- a/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs @@ -17,7 +17,7 @@ public class GetWeatherFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { message.Content = $"It is a sunny day!"; - //message.StopCompletion = true; + message.StopCompletion = false; return true; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs index c93e133c..42a3d652 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs @@ -57,7 +57,8 @@ public class InstructExecutor : IExecutor } else { - var ret = await routing.InvokeAgent(agentId, dialogs); + var convSettings = _services.GetRequiredService(); + var ret = await routing.InvokeAgent(agentId, dialogs, convSettings.EnableStreaming); } var response = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs deleted file mode 100644 index 08c72be1..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructStream.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace BotSharp.Core.Routing; - -public partial class RoutingService -{ - public async Task InstructStream(Agent agent, RoleDialogModel message, List dialogs) - { - var conv = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - storage.Append(conv.ConversationId, message); - - dialogs.Add(message); - Context.SetDialogs(dialogs); - - var routing = _services.GetRequiredService(); - routing.Context.Push(agent.Id, "instruct directly"); - var agentId = routing.Context.GetCurrentAgentId(); - - // Update next action agent's name - var agentService = _services.GetRequiredService(); - - if (agent.Disabled) - { - var content = $"This agent ({agent.Name}) is disabled, please install the corresponding plugin ({agent.Plugin.Name}) to activate this agent."; - - message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: content); - dialogs.Add(message); - } - else - { - var provider = agent.LlmConfig.Provider; - var model = agent.LlmConfig.Model; - - if (provider == null || model == null) - { - var agentSettings = _services.GetRequiredService(); - provider = agentSettings.LlmConfig.Provider; - model = agentSettings.LlmConfig.Model; - } - - var chatCompletion = CompletionProvider.GetChatCompletion(_services, - provider: provider, - model: model); - - await chatCompletion.GetChatCompletionsStreamingAsync(agent, dialogs, async data => { }); - } - - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 7bfd9352..77505718 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Routing; public partial class RoutingService { - public async Task InvokeAgent(string agentId, List dialogs) + public async Task InvokeAgent(string agentId, List dialogs, bool useStream = false) { var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(agentId); @@ -30,8 +30,16 @@ public partial class RoutingService provider: provider, model: model); + RoleDialogModel response; var message = dialogs.Last(); - var response = await chatCompletion.GetChatCompletions(agent, dialogs); + if (useStream) + { + response = await chatCompletion.GetChatCompletionsStreamingAsync(agent, dialogs); + } + else + { + response = await chatCompletion.GetChatCompletions(agent, dialogs); + } if (response.Role == AgentRole.Function) { @@ -45,8 +53,9 @@ public partial class RoutingService message.FunctionArgs = response.FunctionArgs; message.Indication = response.Indication; message.CurrentAgentId = agent.Id; + message.IsStreaming = response.IsStreaming; - await InvokeFunction(message, dialogs); + await InvokeFunction(message, dialogs, useStream); } else { @@ -59,6 +68,7 @@ public partial class RoutingService message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content); message.CurrentAgentId = agent.Id; + message.IsStreaming = response.IsStreaming; dialogs.Add(message); Context.SetDialogs(dialogs); } @@ -66,7 +76,7 @@ public partial class RoutingService return true; } - private async Task InvokeFunction(RoleDialogModel message, List dialogs) + private async Task InvokeFunction(RoleDialogModel message, List dialogs, bool useStream = false) { // execute function // Save states @@ -102,7 +112,7 @@ public partial class RoutingService // Send to Next LLM var curAgentId = routing.Context.GetCurrentAgentId(); - await InvokeAgent(curAgentId, dialogs); + await InvokeAgent(curAgentId, dialogs, useStream); } } else diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 3e45fd63..bc9e1da2 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -51,7 +51,8 @@ public partial class RoutingService : IRoutingService } else { - var ret = await routing.InvokeAgent(agentId, dialogs); + var convSettings = _services.GetRequiredService(); + var ret = await routing.InvokeAgent(agentId, dialogs, convSettings.EnableStreaming); } var response = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 0552b19f..bd131ca1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -378,29 +378,6 @@ public class ConversationController : ControllerBase } - [HttpPost("/conversation/{agentId}/{conversationId}/stream")] - public async Task SendMessageStream( - [FromRoute] string agentId, - [FromRoute] string conversationId, - [FromBody] NewMessageModel input) - { - var conv = _services.GetRequiredService(); - var inputMsg = new RoleDialogModel(AgentRole.User, input.Text) - { - MessageId = !string.IsNullOrWhiteSpace(input.InputMessageId) ? input.InputMessageId : Guid.NewGuid().ToString(), - CreatedAt = DateTime.UtcNow - }; - - var routing = _services.GetRequiredService(); - routing.Context.SetMessageId(conversationId, inputMsg.MessageId); - - conv.SetConversationId(conversationId, input.States); - SetStates(conv, input); - - await conv.StreamMessage(agentId, inputMsg, replyMessage: input.Postback); - } - - [HttpPost("/conversation/{agentId}/{conversationId}/sse")] public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) { diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index a41abe87..79604c98 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -105,7 +105,7 @@ public class ChatHubConversationHook : ConversationHookBase public override async Task OnResponseGenerated(RoleDialogModel message) { - if (!AllowSendingMessage()) return; + if (!AllowSendingMessage() || message.IsStreaming) return; var conv = _services.GetRequiredService(); var state = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs index c7a852fa..7da61c76 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -65,30 +65,30 @@ public class ChatHubObserver : IObserver } else if (value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE) { - var conv = _services.GetRequiredService(); - model = new ChatResponseDto() + if (message.IsStreaming) { - ConversationId = conv.ConversationId, - MessageId = message.MessageId, - Text = message.Content, - Sender = new() + var conv = _services.GetRequiredService(); + model = new ChatResponseDto() { - FirstName = "AI", - LastName = "Assistant", - Role = AgentRole.Assistant - } - }; + ConversationId = conv.ConversationId, + MessageId = message.MessageId, + Text = message.Content, + Sender = new() + { + FirstName = "AI", + LastName = "Assistant", + Role = AgentRole.Assistant + } + }; - var action = new ConversationSenderActionModel - { - ConversationId = conv.ConversationId, - SenderAction = SenderActionEnum.TypingOff - }; + var action = new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOff + }; - GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); - - var storage = _services.GetRequiredService(); - storage.Append(conv.ConversationId, message); + GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); + } } else if (value.EventName == ON_RECEIVE_LLM_STREAM_MESSAGE) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 38d1455e..bb8c484a 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,13 +1,7 @@ using BotSharp.Abstraction.Hooks; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Observables.Queues; -using EntityFrameworkCore.BootKit; -using Fluid; -using ModelContextProtocol.Protocol.Types; using OpenAI.Chat; -using System.Xml; -using static Microsoft.EntityFrameworkCore.DbLoggerCategory; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -187,7 +181,7 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); @@ -227,7 +221,9 @@ public class ChatCompletionProvider : IChatCompletion var text = choice.ContentUpdate[0]?.Text ?? string.Empty; textStream.Collect(text); +#if DEBUG _logger.LogCritical($"Content update: {text}"); +#endif var content = new RoleDialogModel(AgentRole.Assistant, text) { @@ -250,7 +246,9 @@ public class ChatCompletionProvider : IChatCompletion var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); var functionArgument = string.Join(string.Empty, args); +#if DEBUG _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); +#endif responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) { @@ -270,7 +268,8 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) { CurrentAgentId = agent.Id, - MessageId = messageId + MessageId = messageId, + IsStreaming = true }; } } @@ -282,7 +281,7 @@ public class ChatCompletionProvider : IChatCompletion Data = responseMessage }); - return true; + return responseMessage; } diff --git a/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs b/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs index ee3e132a..f6c06dc4 100644 --- a/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs +++ b/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.MLTasks; @@ -96,13 +96,15 @@ namespace BotSharp.Plugin.Google.Core public async Task GetChatCompletionsStreamingAsync_Test(IChatCompletion chatCompletion, Agent agent, string modelName) { chatCompletion.SetModelName(modelName); - var conversation = new List([new RoleDialogModel(AgentRole.User, "write a poem about stars")]); + RoleDialogModel reply = null; - var result = await chatCompletion.GetChatCompletionsStreamingAsync(agent,conversation, async (received) => + var messages = new List { - reply = received; - }); - result.ShouldBeTrue(); + new RoleDialogModel(AgentRole.User, "write a poem about stars") + }; + var result = await chatCompletion.GetChatCompletionsStreamingAsync(agent, messages); + + result.ShouldNotBeNull(); reply.ShouldNotBeNull(); reply.Content.ShouldNotBeNullOrEmpty(); } From 3d57d149b47c5f68ba6d6f7b4186cb9b3162f946 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 23 Jun 2025 21:47:58 -0500 Subject: [PATCH 10/28] clean code --- .../BotSharp.Abstraction/Routing/IRoutingService.cs | 4 ---- .../BotSharp.Core/Routing/RoutingService.InvokeAgent.cs | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index fb542d5f..33803a98 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -26,10 +26,6 @@ public interface IRoutingService /// RoutingRule[] GetRulesByAgentId(string id); - //void ResetRecursiveCounter(); - //int GetRecursiveCounter(); - //void SetRecursiveCounter(int counter); - Task InvokeAgent(string agentId, List dialogs, bool useStream = false); Task InvokeFunction(string name, RoleDialogModel messages); Task InstructLoop(Agent agent, RoleDialogModel message, List dialogs); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 77505718..eb60e925 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -76,7 +76,7 @@ public partial class RoutingService return true; } - private async Task InvokeFunction(RoleDialogModel message, List dialogs, bool useStream = false) + private async Task InvokeFunction(RoleDialogModel message, List dialogs, bool useStream) { // execute function // Save states From 5f211d6aa5c92a5e872dc68dc184044f5f258bfb Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 24 Jun 2025 08:34:38 -0500 Subject: [PATCH 11/28] minor change --- .../Observers/ChatHubObserver.cs | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs index 7da61c76..97cb89a4 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -63,32 +63,29 @@ public class ChatHubObserver : IObserver GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); } - else if (value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE) + else if (value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE && message.IsStreaming) { - if (message.IsStreaming) + var conv = _services.GetRequiredService(); + model = new ChatResponseDto() { - var conv = _services.GetRequiredService(); - model = new ChatResponseDto() + ConversationId = conv.ConversationId, + MessageId = message.MessageId, + Text = message.Content, + Sender = new() { - ConversationId = conv.ConversationId, - MessageId = message.MessageId, - Text = message.Content, - Sender = new() - { - FirstName = "AI", - LastName = "Assistant", - Role = AgentRole.Assistant - } - }; + FirstName = "AI", + LastName = "Assistant", + Role = AgentRole.Assistant + } + }; - var action = new ConversationSenderActionModel - { - ConversationId = conv.ConversationId, - SenderAction = SenderActionEnum.TypingOff - }; + var action = new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOff + }; - GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); - } + GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); } else if (value.EventName == ON_RECEIVE_LLM_STREAM_MESSAGE) { From 2b465722df0a2cec34a52c84580947c12f859430 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 24 Jun 2025 22:09:13 -0500 Subject: [PATCH 12/28] use stream message from state and add token usage --- .../Conversations/Dtos/ChatResponseDto.cs | 3 ++ .../Settings/ConversationSetting.cs | 1 - .../Routing/Reasoning/InstructExecutor.cs | 5 +-- .../BotSharp.Core/Routing/RoutingService.cs | 5 +-- .../Hooks/ChatHubConversationHook.cs | 9 ++++-- .../Providers/Chat/ChatCompletionProvider.cs | 31 +++++++++++++++++++ 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs index b391bc7b..19b5e8b2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs @@ -35,6 +35,9 @@ public class ChatResponseDto : InstructResult [JsonPropertyName("has_message_files")] public bool HasMessageFiles { get; set; } + [JsonPropertyName("is_streaming")] + public bool IsStreaming { get; set; } + [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index 980569dc..c4f131dc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -12,7 +12,6 @@ public class ConversationSetting public bool EnableContentLog { get; set; } public bool EnableStateLog { get; set; } public bool EnableTranslationMemory { get; set; } - public bool EnableStreaming { get; set; } public CleanConversationSetting CleanSetting { get; set; } = new(); public RateLimitSetting RateLimit { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs index 42a3d652..18d87cb9 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs @@ -57,8 +57,9 @@ public class InstructExecutor : IExecutor } else { - var convSettings = _services.GetRequiredService(); - var ret = await routing.InvokeAgent(agentId, dialogs, convSettings.EnableStreaming); + var state = _services.GetRequiredService(); + var useStreamMsg = state.GetState("use_stream_message"); + var ret = await routing.InvokeAgent(agentId, dialogs, bool.TryParse(useStreamMsg, out var useStream) && useStream); } var response = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index bc9e1da2..e4764b92 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -51,8 +51,9 @@ public partial class RoutingService : IRoutingService } else { - var convSettings = _services.GetRequiredService(); - var ret = await routing.InvokeAgent(agentId, dialogs, convSettings.EnableStreaming); + var state = _services.GetRequiredService(); + var useStreamMsg = state.GetState("use_stream_message"); + var ret = await routing.InvokeAgent(agentId, dialogs, bool.TryParse(useStreamMsg, out var useStream) && useStream); } var response = dialogs.Last(); diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 79604c98..7bc4600a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -105,7 +105,7 @@ public class ChatHubConversationHook : ConversationHookBase public override async Task OnResponseGenerated(RoleDialogModel message) { - if (!AllowSendingMessage() || message.IsStreaming) return; + if (!AllowSendingMessage()) return; var conv = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -118,6 +118,7 @@ public class ChatHubConversationHook : ConversationHookBase RichContent = message.SecondaryRichContent ?? message.RichContent, Data = message.Data, States = state.GetStates(), + IsStreaming = message.IsStreaming, Sender = new() { FirstName = "AI", @@ -133,7 +134,11 @@ public class ChatHubConversationHook : ConversationHookBase SenderAction = SenderActionEnum.TypingOff }; - await GenerateSenderAction(conv.ConversationId, action); + if (!message.IsStreaming) + { + await GenerateSenderAction(conv.ConversationId, action); + } + await ReceiveAssistantMessage(conv.ConversationId, json); await base.OnResponseGenerated(message); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index bb8c484a..16e56b25 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,6 +1,9 @@ +using Azure; using BotSharp.Abstraction.Hooks; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Observables.Queues; +using BotSharp.Plugin.OpenAI.Models.Realtime; +using Fluid; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -190,6 +193,13 @@ public class ChatCompletionProvider : IChatCompletion var hub = _services.GetRequiredService(); var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + var contentHooks = _services.GetHooks(agent.Id); + // Before chat completion hook + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + hub.Push(new() { ServiceProvider = _services, @@ -201,8 +211,11 @@ public class ChatCompletionProvider : IChatCompletion } }); + using var textStream = new RealtimeTextStream(); var toolCalls = new List(); + ChatTokenUsage? tokenUsage = null; + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) { CurrentAgentId = agent.Id, @@ -211,6 +224,8 @@ public class ChatCompletionProvider : IChatCompletion await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) { + tokenUsage = choice.Usage; + if (!choice.ToolCallUpdates.IsNullOrEmpty()) { toolCalls.AddRange(choice.ToolCallUpdates); @@ -281,6 +296,22 @@ public class ChatCompletionProvider : IChatCompletion Data = responseMessage }); + + var inputTokenDetails = tokenUsage?.InputTokenDetails; + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 + }); + } + return responseMessage; } From 0aff924dd9a15a8ee1e33793b78c18a38f739656 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 27 Jun 2025 11:48:13 -0500 Subject: [PATCH 13/28] minor change --- src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 720f3fd0..b499438e 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -74,7 +74,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR var log = $"{GetMessageContent(message)}"; var replyContent = JsonSerializer.Serialize(replyMsg, _options.JsonSerializerOptions); - log += $"\r\n```json\r\n{replyContent}\r\n```"; + log += $"\r\n\r\n```json\r\n{replyContent}\r\n```"; var input = new ContentLogInputModel(conversationId, message) { @@ -233,7 +233,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR if (message.RichContent != null || message.SecondaryRichContent != null) { var richContent = JsonSerializer.Serialize(message.SecondaryRichContent ?? message.RichContent, _localJsonOptions); - log += $"\r\n```json\r\n{richContent}\r\n```"; + log += $"\r\n\r\n```json\r\n{richContent}\r\n```"; } var input = new ContentLogInputModel(conv.ConversationId, message) From aac09ca3bdab6786729a8475c0646347fc2116fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 27 Jun 2025 13:49:44 -0500 Subject: [PATCH 14/28] implement streaming chat interface --- .../MLTasks/IChatCompletion.cs | 2 +- .../Conversations/ConversationPlugin.cs | 3 +- .../Observables/Queues/MessageHub.cs | 13 +- .../Providers/ChatCompletionProvider.cs | 3 +- .../BotSharp.Plugin.AzureOpenAI.csproj | 2 +- .../Providers/Chat/ChatCompletionProvider.cs | 135 ++++++++++++++--- .../BotSharp.Plugin.ChatHub/ChatHubPlugin.cs | 5 +- .../BotSharp.Plugin.DeepSeekAI.csproj | 2 +- .../Providers/Chat/ChatCompletionProvider.cs | 141 +++++++++++++++--- .../Chat/GeminiChatCompletionProvider.cs | 35 +---- .../Chat/PalmChatCompletionProvider.cs | 2 +- .../Providers/ChatCompletionProvider.cs | 4 +- .../BotSharp.Plugin.LLamaSharp.csproj | 2 +- .../Providers/ChatCompletionProvider.cs | 64 +++++++- .../Providers/ChatCompletionProvider.cs | 2 +- .../Providers/ChatCompletionProvider.cs | 2 +- ...osoftExtensionsAIChatCompletionProvider.cs | 4 +- .../Providers/Chat/ChatCompletionProvider.cs | 5 +- .../SemanticKernelChatCompletionProvider.cs | 2 +- .../BotSharp.Plugin.SparkDesk.csproj | 2 +- .../Providers/ChatCompletionProvider.cs | 79 ++++++++-- 21 files changed, 385 insertions(+), 124 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 4d93f307..51f89375 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -24,5 +24,5 @@ public interface IChatCompletion Func onFunctionExecuting); Task GetChatCompletionsStreamingAsync(Agent agent, - List conversations) => Task.FromResult(new RoleDialogModel(AgentRole.Assistant, string.Empty)); + List conversations); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index e8e18eb8..a559877f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -12,6 +12,7 @@ using BotSharp.Core.Templating; using BotSharp.Core.Translation; using BotSharp.Core.Observables.Queues; using Microsoft.Extensions.Configuration; +using BotSharp.Abstraction.Observables.Models; namespace BotSharp.Core.Conversations; @@ -42,7 +43,7 @@ public class ConversationPlugin : IBotSharpPlugin return settingService.Bind("GoogleApi"); }); - services.AddSingleton(); + services.AddSingleton>(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs index 9950a613..affb142d 100644 --- a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs +++ b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs @@ -1,15 +1,14 @@ -using BotSharp.Abstraction.Observables.Models; using System.Reactive.Subjects; namespace BotSharp.Core.Observables.Queues; -public class MessageHub +public class MessageHub where T : class { - private readonly ILogger _logger; - private readonly ISubject _observable = new Subject(); - public IObservable Events => _observable; + private readonly ILogger> _logger; + private readonly ISubject _observable = new Subject(); + public IObservable Events => _observable; - public MessageHub(ILogger logger) + public MessageHub(ILogger> logger) { _logger = logger; } @@ -18,7 +17,7 @@ public class MessageHub /// Push an item to the observers. /// /// - public void Push(HubObserveData item) + public void Push(T item) { _observable.OnNext(item); } diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 285e8abd..50d67a1b 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -96,8 +96,7 @@ public class ChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, - Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 494326fa..372fb3de 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index ab7135ef..605bceb8 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,6 +1,9 @@ using Azure; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; using OpenAI.Chat; using System.ClientModel; @@ -203,39 +206,133 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var response = chatClient.CompleteChatStreamingAsync(messages, options); + var hub = _services.GetRequiredService>(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; - await foreach (var choice in response) + var contentHooks = _services.GetHooks(agent.Id); + // Before chat completion hook + foreach (var hook in contentHooks) { - if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) - { - var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; - Console.Write(update); + await hook.BeforeGenerating(agent, conversations); + } - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var toolCalls = new List(); + ChatTokenUsage? tokenUsage = null; + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) + { + tokenUsage = choice.Usage; + + if (!choice.ToolCallUpdates.IsNullOrEmpty()) + { + toolCalls.AddRange(choice.ToolCallUpdates); } - if (choice.ContentUpdate.IsNullOrEmpty()) continue; - - _logger.LogInformation(choice.ContentUpdate[0]?.Text); - - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + if (!choice.ContentUpdate.IsNullOrEmpty()) { - RenderedInstruction = string.Join("\r\n", renderedInstructions) + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + textStream.Collect(text); + +#if DEBUG + _logger.LogCritical($"Content update: {text}"); +#endif + + var content = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); + } + + if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) + { + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); + +#if DEBUG + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); +#endif + + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = toolCallId, + FunctionName = functionName, + FunctionArgs = functionArgument + }; + } + else if (choice.FinishReason.HasValue) + { + var allText = textStream.GetText(); + _logger.LogCritical($"Text Content: {allText}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + } + } + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + + var inputTokenDetails = tokenUsage?.InputTokenDetails; + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } - return true; + return responseMessage; } protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index b1fb144f..d3c3f2ac 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Observables.Models; using BotSharp.Core.Observables.Queues; using BotSharp.Plugin.ChatHub.Hooks; using BotSharp.Plugin.ChatHub.Observers; @@ -35,8 +36,8 @@ public class ChatHubPlugin : IBotSharpPlugin, IBotSharpAppPlugin public void Configure(IApplicationBuilder app) { var services = app.ApplicationServices; - var queue = services.GetRequiredService(); - var logger = services.GetRequiredService>(); + var queue = services.GetRequiredService>(); + var logger = services.GetRequiredService>>(); queue.Events.Subscribe(new ChatHubObserver(logger)); } } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 2f7e326a..3f9a26ce 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 42ce1ac9..bcbcc2a4 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; +using BotSharp.Plugin.DeepSeek.Providers; using Microsoft.Extensions.Logging; using OpenAI.Chat; -using BotSharp.Abstraction.Files; -using BotSharp.Plugin.DeepSeek.Providers; -using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.DeepSeekAI.Providers.Chat; @@ -170,39 +173,133 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var response = chatClient.CompleteChatStreamingAsync(messages, options); + var hub = _services.GetRequiredService>(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; - await foreach (var choice in response) + var contentHooks = _services.GetHooks(agent.Id); + // Before chat completion hook + foreach (var hook in contentHooks) { - if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) - { - var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; - _logger.LogInformation(update); + await hook.BeforeGenerating(agent, conversations); + } - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var toolCalls = new List(); + ChatTokenUsage? tokenUsage = null; + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) + { + tokenUsage = choice.Usage; + + if (!choice.ToolCallUpdates.IsNullOrEmpty()) + { + toolCalls.AddRange(choice.ToolCallUpdates); } - if (choice.ContentUpdate.IsNullOrEmpty()) continue; - - _logger.LogInformation(choice.ContentUpdate[0]?.Text); - - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + if (!choice.ContentUpdate.IsNullOrEmpty()) { - RenderedInstruction = string.Join("\r\n", renderedInstructions) + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + textStream.Collect(text); + +#if DEBUG + _logger.LogCritical($"Content update: {text}"); +#endif + + var content = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); + } + + if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) + { + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); + +#if DEBUG + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); +#endif + + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = toolCallId, + FunctionName = functionName, + FunctionArgs = functionArgument + }; + } + else if (choice.FinishReason.HasValue) + { + var allText = textStream.GetText(); + _logger.LogCritical($"Text Content: {allText}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + } + } + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + + var inputTokenDetails = tokenUsage?.InputTokenDetails; + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } - return true; + return responseMessage; } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 1ead28e6..1d67eac8 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -159,40 +159,9 @@ public class GeminiChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { - var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - var chatClient = client.CreateGenerativeModel(_model.ToModelId()); - var (prompt, messages) = PrepareOptions(chatClient,agent, conversations); - - var asyncEnumerable = chatClient.StreamContentAsync(messages); - - await foreach (var response in asyncEnumerable) - { - if (response.GetFunction() != null) - { - var func = response.GetFunction(); - var update = func?.Args?.ToJsonString().ToString() ?? string.Empty; - _logger.LogInformation(update); - - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; - } - - if (response.Text().IsNullOrEmpty()) continue; - - _logger.LogInformation(response.Text()); - - await onMessageReceived(new RoleDialogModel(response.Candidates?.LastOrDefault()?.Content?.Role?.ToString() ?? AgentRole.Assistant.ToString(), response.Text() ?? string.Empty) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - } - - return true; + throw new NotImplementedException(); } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index 72a47adc..e992fd60 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -145,7 +145,7 @@ public class PalmChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 460677ba..5a38b0a9 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -76,9 +76,9 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { - return true; + throw new NotImplementedException(); } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index af7a6237..80807a26 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 45f41e95..321b9aee 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -1,5 +1,12 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; +using Microsoft.AspNetCore.SignalR; +using static LLama.Common.ChatHistory; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.LLamaSharp.Providers; @@ -159,12 +166,8 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { - string totalResponse = ""; - var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); - content += $"\r\n{AgentRole.Assistant}: "; - var state = _services.GetRequiredService(); var model = state.GetState("model", "llama-2-7b-chat.Q8_0"); @@ -180,13 +183,60 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(agent.Instruction); } + var hub = _services.GetRequiredService>(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + await foreach (var response in executor.InferAsync(agent.Instruction, inferenceParams)) { Console.Write(response); - totalResponse += response; + textStream.Collect(response); + + var content = new RoleDialogModel(AgentRole.Assistant, response) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); } - return true; + responseMessage = new RoleDialogModel(AgentRole.Assistant, textStream.GetText()) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + return responseMessage; } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs index aa6b1175..9e2aa1d4 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs @@ -65,7 +65,7 @@ namespace BotSharp.Plugin.VertexAI.Providers throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index c1d4bef1..8b3ecea3 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -235,7 +235,7 @@ public class ChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index d95a25ae..e510a9a2 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -169,8 +169,10 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio throw new NotImplementedException(); /// - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) => + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) + { throw new NotImplementedException(); + } private sealed class NopAIFunction(string name, string description, JsonElement schema) : AIFunction { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 16e56b25..314697c8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using Azure; using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Observables.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Observables.Queues; using BotSharp.Plugin.OpenAI.Models.Realtime; @@ -190,7 +191,7 @@ public class ChatCompletionProvider : IChatCompletion var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var hub = _services.GetRequiredService(); + var hub = _services.GetRequiredService>(); var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; var contentHooks = _services.GetHooks(agent.Id); @@ -210,7 +211,6 @@ public class ChatCompletionProvider : IChatCompletion MessageId = messageId } }); - using var textStream = new RealtimeTextStream(); var toolCalls = new List(); @@ -273,7 +273,6 @@ public class ChatCompletionProvider : IChatCompletion FunctionName = functionName, FunctionArgs = functionArgument }; - } else if (choice.FinishReason.HasValue) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index 3f742777..57277399 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -94,7 +94,7 @@ namespace BotSharp.Plugin.SemanticKernel throw new NotImplementedException(); } /// - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj index 51f26f87..d49d41df 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index ba0aa220..035e03e0 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -1,6 +1,10 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; +using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.SparkDesk.Providers; @@ -143,34 +147,77 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = new SparkDeskClient(appId: _settings.AppId, apiKey: _settings.ApiKey, apiSecret: _settings.ApiSecret); var (prompt, messages, funcall) = PrepareOptions(agent, conversations); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + var hub = _services.GetRequiredService>(); + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + using var textStream = new RealtimeTextStream(); await foreach (StreamedChatResponse response in client.ChatAsStreamAsync(modelVersion: _settings.ModelVersion, messages, functions: funcall.Length == 0 ? null : funcall)) { - if (response.FunctionCall !=null) + if (response.FunctionCall != null) { - await onMessageReceived(new RoleDialogModel(AgentRole.Function, response.Text) - { + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = response.FunctionCall.Name, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments, - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; + FunctionArgs = response.FunctionCall.Arguments + }; } - - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, response.Text) + else { - CurrentAgentId = agent.Id, - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - - } + textStream.Collect(response.Text); + responseMessage = new RoleDialogModel(AgentRole.Assistant, response.Text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; - return true; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = responseMessage + }); + } + } + + if (responseMessage.Role == AgentRole.Assistant) + { + responseMessage.Content = textStream.GetText(); + responseMessage.IsStreaming = true; + } + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + return responseMessage; } public void SetModelName(string model) From e52eac5854a9098f85914ba1007757ce5e5deb14 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 15 Jul 2025 16:52:13 -0500 Subject: [PATCH 15/28] add invoke source --- .../Conversations/ConversationHookBase.cs | 4 ++-- .../Conversations/IConversationHook.cs | 6 ++++-- .../Routing/Enums/InvokeSource.cs | 19 +++++++++++++++++++ .../Routing/IRoutingService.cs | 4 ++-- .../BotSharp.Abstraction/Using.cs | 1 + .../Hooks/RealtimeConversationHook.cs | 7 ++++--- .../Services/RealtimeHub.cs | 3 ++- .../ConversationService.SendMessage.cs | 1 - .../Evaluations/EvaluationConversationHook.cs | 4 ++-- .../Routing/Reasoning/InstructExecutor.cs | 4 ++-- .../Routing/RoutingService.InvokeAgent.cs | 10 +++++----- .../Routing/RoutingService.InvokeFunction.cs | 6 +++--- .../BotSharp.Core/Routing/RoutingService.cs | 2 +- src/Infrastructure/BotSharp.Core/Using.cs | 1 + .../Controllers/RealtimeController.cs | 3 ++- .../Hooks/ChatHubConversationHook.cs | 5 +++-- .../Hooks/StreamingLogHook.cs | 5 +++-- .../Hooks/TwilioConversationHook.cs | 9 +++++---- 18 files changed, 61 insertions(+), 33 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/Enums/InvokeSource.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index da87646e..0a5244f8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -49,10 +49,10 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnHumanInterventionNeeded(RoleDialogModel message) => Task.CompletedTask; - public virtual Task OnFunctionExecuting(RoleDialogModel message) + public virtual Task OnFunctionExecuting(RoleDialogModel message, string from = InvokeSource.Manual) => Task.CompletedTask; - public virtual Task OnFunctionExecuted(RoleDialogModel message) + public virtual Task OnFunctionExecuted(RoleDialogModel message, string from = InvokeSource.Manual) => Task.CompletedTask; public virtual Task OnMessageReceived(RoleDialogModel message) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index 6d967391..d1b6f83c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -59,15 +59,17 @@ public interface IConversationHook : IHookBase /// Triggered before LLM calls function. /// /// + /// /// - Task OnFunctionExecuting(RoleDialogModel message); + Task OnFunctionExecuting(RoleDialogModel message, string from = InvokeSource.Manual); /// /// Triggered when the function calling completed. /// /// + /// /// - Task OnFunctionExecuted(RoleDialogModel message); + Task OnFunctionExecuted(RoleDialogModel message, string from = InvokeSource.Manual); Task OnResponseGenerated(RoleDialogModel message); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Enums/InvokeSource.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Enums/InvokeSource.cs new file mode 100644 index 00000000..b3460bb6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Enums/InvokeSource.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Abstraction.Routing.Enums; + +public static class InvokeSource +{ + /// + /// Invoke manually + /// + public const string Manual = "manual"; + + /// + /// Invoke by LLM directly + /// + public const string Llm = "llm"; + + /// + /// Invoke by agent routing + /// + public const string Routing = "routing"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index 97be8874..384a606f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -30,8 +30,8 @@ public interface IRoutingService //int GetRecursiveCounter(); //void SetRecursiveCounter(int counter); - Task InvokeAgent(string agentId, List dialogs); - Task InvokeFunction(string name, RoleDialogModel messages); + Task InvokeAgent(string agentId, List dialogs, string from = InvokeSource.Manual); + Task InvokeFunction(string name, RoleDialogModel messages, string from = InvokeSource.Manual); Task InstructLoop(Agent agent, RoleDialogModel message, List dialogs); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 8a9d30cb..38a6dca0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -12,6 +12,7 @@ global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Infrastructures.Enums; global using BotSharp.Abstraction.Models; global using BotSharp.Abstraction.Routing.Models; +global using BotSharp.Abstraction.Routing.Enums; global using BotSharp.Abstraction.Templating; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 4eb4c64e..095ff2b8 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.Utilities; namespace BotSharp.Core.Realtime.Hooks; @@ -10,7 +11,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook _services = services; } - public async Task OnFunctionExecuting(RoleDialogModel message) + public async Task OnFunctionExecuting(RoleDialogModel message, string from = InvokeSource.Manual) { var hub = _services.GetRequiredService(); if (hub.HubConn == null) @@ -31,10 +32,10 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook } } - public async Task OnFunctionExecuted(RoleDialogModel message) + public async Task OnFunctionExecuted(RoleDialogModel message, string from = InvokeSource.Manual) { var hub = _services.GetRequiredService(); - if (hub.HubConn == null) + if (from != InvokeSource.Llm || hub.HubConn == null) { return; } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 32b0a112..fe544b4d 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing.Enums; using BotSharp.Core.Infrastructures; namespace BotSharp.Core.Realtime.Services; @@ -98,7 +99,7 @@ public class RealtimeHub : IRealtimeHub agent.Id); } - await routing.InvokeFunction(message.FunctionName, message); + await routing.InvokeFunction(message.FunctionName, message, from: InvokeSource.Llm); } else { diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index a28ccbc7..1155dc38 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -2,7 +2,6 @@ using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.Routing.Settings; namespace BotSharp.Core.Conversations.Services; diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs index b315b683..b1b4541e 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs @@ -22,13 +22,13 @@ public class EvaluationConversationHook : ConversationHookBase return base.OnMessageReceived(message); } - public override Task OnFunctionExecuted(RoleDialogModel message) + public override Task OnFunctionExecuted(RoleDialogModel message, string from = InvokeSource.Manual) { if (Conversation != null && _convSettings.EnableExecutionLog) { _logger.Append(Conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.FunctionName}({message.FunctionArgs}) => {message.Content}"); } - return base.OnFunctionExecuted(message); + return base.OnFunctionExecuted(message, from: from); } public override Task OnResponseGenerated(RoleDialogModel message) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs index c93e133c..30806451 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs @@ -33,7 +33,7 @@ public class InstructExecutor : IExecutor if (message.FunctionName != null) { var msg = RoleDialogModel.From(message, role: AgentRole.Function); - await routing.InvokeFunction(message.FunctionName, msg); + await routing.InvokeFunction(message.FunctionName, msg, from: InvokeSource.Llm); } var agentId = routing.Context.GetCurrentAgentId(); @@ -57,7 +57,7 @@ public class InstructExecutor : IExecutor } else { - var ret = await routing.InvokeAgent(agentId, dialogs); + var ret = await routing.InvokeAgent(agentId, dialogs, from: InvokeSource.Routing); } var response = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 7bfd9352..8e625a36 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Routing; public partial class RoutingService { - public async Task InvokeAgent(string agentId, List dialogs) + public async Task InvokeAgent(string agentId, List dialogs, string from = InvokeSource.Manual) { var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(agentId); @@ -46,7 +46,7 @@ public partial class RoutingService message.Indication = response.Indication; message.CurrentAgentId = agent.Id; - await InvokeFunction(message, dialogs); + await InvokeFunction(message, dialogs, from: from); } else { @@ -66,7 +66,7 @@ public partial class RoutingService return true; } - private async Task InvokeFunction(RoleDialogModel message, List dialogs) + private async Task InvokeFunction(RoleDialogModel message, List dialogs, string from) { // execute function // Save states @@ -75,7 +75,7 @@ public partial class RoutingService var routing = _services.GetRequiredService(); // Call functions - await routing.InvokeFunction(message.FunctionName, message); + await routing.InvokeFunction(message.FunctionName, message, from: from); // Pass execution result to LLM to get response if (!message.StopCompletion) @@ -102,7 +102,7 @@ public partial class RoutingService // Send to Next LLM var curAgentId = routing.Context.GetCurrentAgentId(); - await InvokeAgent(curAgentId, dialogs); + await InvokeAgent(curAgentId, dialogs, from); } } else diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 75754939..1001dede 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -5,7 +5,7 @@ namespace BotSharp.Core.Routing; public partial class RoutingService { - public async Task InvokeFunction(string name, RoleDialogModel message) + public async Task InvokeFunction(string name, RoleDialogModel message, string from = InvokeSource.Manual) { var currentAgentId = message.CurrentAgentId; var agentService = _services.GetRequiredService(); @@ -36,7 +36,7 @@ public partial class RoutingService foreach (var hook in hooks) { hook.SetAgent(agent); - await hook.OnFunctionExecuting(clonedMessage); + await hook.OnFunctionExecuting(clonedMessage, from: from); } bool result = false; @@ -48,7 +48,7 @@ public partial class RoutingService // After functions have been executed foreach (var hook in hooks) { - await hook.OnFunctionExecuted(clonedMessage); + await hook.OnFunctionExecuted(clonedMessage, from: from); } // Set result to original message diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 3e45fd63..262c3001 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -51,7 +51,7 @@ public partial class RoutingService : IRoutingService } else { - var ret = await routing.InvokeAgent(agentId, dialogs); + var ret = await routing.InvokeAgent(agentId, dialogs, from: InvokeSource.Routing); } var response = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index aa5a4595..0e8e2762 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -22,6 +22,7 @@ global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Repositories.Filters; global using BotSharp.Abstraction.Roles; global using BotSharp.Abstraction.Roles.Models; +global using BotSharp.Abstraction.Routing.Enums; global using BotSharp.Abstraction.Routing; global using BotSharp.Abstraction.SideCar.Attributes; global using BotSharp.Abstraction.Statistics.Enums; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs index f8851cb3..61ae77a7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Routing; +using BotSharp.Abstraction.Routing.Enums; namespace BotSharp.OpenAPI.Controllers; @@ -25,7 +26,7 @@ public class RealtimeController : ControllerBase FunctionName = functionName, FunctionArgs = JsonSerializer.Serialize(args) }; - await routing.InvokeFunction(functionName, message); + await routing.InvokeFunction(functionName, message, from: InvokeSource.Llm); return message.Content; } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 3c7a675d..74735bd8 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Conversations.Dtos; +using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.SideCar; using BotSharp.Abstraction.Users.Dtos; using Microsoft.AspNetCore.SignalR; @@ -84,7 +85,7 @@ public class ChatHubConversationHook : ConversationHookBase await base.OnMessageReceived(message); } - public override async Task OnFunctionExecuting(RoleDialogModel message) + public override async Task OnFunctionExecuting(RoleDialogModel message, string from = InvokeSource.Manual) { var conv = _services.GetRequiredService(); var action = new ConversationSenderActionModel @@ -95,7 +96,7 @@ public class ChatHubConversationHook : ConversationHookBase }; await GenerateSenderAction(conv.ConversationId, action); - await base.OnFunctionExecuting(message); + await base.OnFunctionExecuting(message, from: from); } public override async Task OnPostbackMessageReceived(RoleDialogModel message, PostbackMessageModel replyMsg) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 720f3fd0..991a64e5 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Routing.Enums; using Microsoft.AspNetCore.SignalR; using System.Text.Encodings.Web; using System.Text.Unicode; @@ -141,7 +142,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR if (!_convSettings.ShowVerboseLog) return; } - public override async Task OnFunctionExecuting(RoleDialogModel message) + public override async Task OnFunctionExecuting(RoleDialogModel message, string from = InvokeSource.Manual) { var conversationId = _state.GetConversationId(); if (string.IsNullOrEmpty(conversationId)) return; @@ -164,7 +165,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR await SendContentLog(conversationId, input); } - public override async Task OnFunctionExecuted(RoleDialogModel message) + public override async Task OnFunctionExecuted(RoleDialogModel message, string from = InvokeSource.Manual) { var conversationId = _state.GetConversationId(); if (string.IsNullOrEmpty(conversationId)) return; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs index bc647140..11bd8d36 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs @@ -1,9 +1,10 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Routing; -using Task = System.Threading.Tasks.Task; -using Twilio.Rest.Api.V2010.Account; +using BotSharp.Abstraction.Routing.Enums; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; -using BotSharp.Abstraction.Hooks; +using Twilio.Rest.Api.V2010.Account; +using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Hooks; @@ -22,7 +23,7 @@ public class TwilioConversationHook : ConversationHookBase, IConversationHook _logger = logger; } - public override async Task OnFunctionExecuted(RoleDialogModel message) + public override async Task OnFunctionExecuted(RoleDialogModel message, string from = InvokeSource.Manual) { var hooks = _services.GetHooks(message.CurrentAgentId); From 95fd02be200a42ac881906c99455bbe538f7bc0d Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Mon, 21 Jul 2025 16:02:40 -0500 Subject: [PATCH 16/28] add channel to the instruct controller --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index e8c87c2c..de67fcab 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -30,7 +30,8 @@ public class InstructModeController : ControllerBase .SetState("model_id", input.ModelId, source: StateSource.External) .SetState("instruction", input.Instruction, source: StateSource.External) .SetState("input_text", input.Text, source: StateSource.External) - .SetState("template_name", input.Template, source: StateSource.External); + .SetState("template_name", input.Template, source: StateSource.External) + .SetState("channel", input.Channel, source: StateSource.External); var instructor = _services.GetRequiredService(); var result = await instructor.Execute(agentId, From 061ce95e353d05622da935a3a57b6bdff57a2d0b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 21 Jul 2025 23:01:53 -0500 Subject: [PATCH 17/28] allow using template across agents --- .../FileRepository/FileRepository.Agent.cs | 2 +- .../Templating/TemplateRender.cs | 46 +++++++++++++++---- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index f7e70ca7..c5947708 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -475,7 +475,7 @@ namespace BotSharp.Core.Repository if (!filter.AgentNames.IsNullOrEmpty()) { - query = query.Where(x => filter.AgentNames.Contains(x.Name)); + query = query.Where(x => filter.AgentNames.Contains(x.Name, StringComparer.OrdinalIgnoreCase)); } if (!string.IsNullOrEmpty(filter.SimilarName)) diff --git a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs index f7e5ecde..5b65465d 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs @@ -8,6 +8,7 @@ using System.Collections; using System.IO; using System.Reflection; using System.Text.Encodings.Web; +using System.Text.RegularExpressions; namespace BotSharp.Core.Templating; @@ -36,9 +37,9 @@ public class TemplateRender : ITemplateRender _options.MemberAccessStrategy.Register(); _options.MemberAccessStrategy.Register(); - _parser.RegisterIdentifierTag("link", (string identifier, TextWriter writer, TextEncoder encoder, TemplateContext context) => + _parser.RegisterExpressionTag("link", (Expression expression, TextWriter writer, TextEncoder encoder, TemplateContext context) => { - return RenderIdentifierTag("link", identifier, writer, encoder, context); + return RenderTag("link", expression, writer, encoder, context, services); }); } @@ -82,20 +83,47 @@ public class TemplateRender : ITemplateRender #region Private methods - private static async ValueTask RenderIdentifierTag(string tag, string identifier, TextWriter writer, TextEncoder encoder, TemplateContext context) + private static async ValueTask RenderTag( + string tag, + Expression expression, + TextWriter writer, + TextEncoder encoder, + TemplateContext context, + IServiceProvider services) { + try { - var value = await context.Model.GetValueAsync(TemplateRenderConstant.RENDER_AGENT, context); - var agent = value?.ToObjectValue() as Agent; - var found = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(identifier)); - var key = $"{agent?.Id} | {tag} | {identifier}"; + var value = await expression.EvaluateAsync(context); + var expStr = value?.ToStringValue() ?? string.Empty; - if (found == null || (context.AmbientValues.TryGetValue(key, out var visited) && (bool)visited)) + value = await context.Model.GetValueAsync(TemplateRenderConstant.RENDER_AGENT, context); + var agent = value?.ToObjectValue() as Agent; + + var splited = Regex.Split(expStr, @"\s*from\s*", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => x.Trim()) + .ToArray(); + + var templateName = splited.ElementAtOrDefault(0); + var agentName = splited.ElementAtOrDefault(1); + + if (splited.Length > 1 && !agentName.IsEqualTo(agent?.Name)) + { + using var scope = services.CreateScope(); + var agentService = scope.ServiceProvider.GetRequiredService(); + var result = await agentService.GetAgents(new() { SimilarName = agentName }); + agent = result?.Items?.FirstOrDefault(); + } + + var template = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(templateName)); + var key = $"{tag} | {agent?.Id} | {templateName}"; + + if (template == null || (context.AmbientValues.TryGetValue(key, out var visited) && (bool)visited)) { writer.Write(string.Empty); } - else if (_parser.TryParse(found.Content, out var t, out _)) + else if (_parser.TryParse(template.Content, out var t, out _)) { context.AmbientValues[key] = true; var rendered = t.Render(context); From 1cabd91ec083fdce331a562e58c3a07db94ce180 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 21 Jul 2025 23:02:44 -0500 Subject: [PATCH 18/28] minor change --- src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs index 5b65465d..e2a0d029 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs @@ -91,7 +91,6 @@ public class TemplateRender : ITemplateRender TemplateContext context, IServiceProvider services) { - try { var value = await expression.EvaluateAsync(context); From b1eb7543d411025e287e02ee77d33eaf36326eaf Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 22 Jul 2025 10:26:00 -0500 Subject: [PATCH 19/28] add from agent filter --- .../BotSharp.Core/Templating/TemplateRender.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs index e2a0d029..2124da16 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.Templating; using BotSharp.Abstraction.Translation.Models; using Fluid; using Fluid.Ast; +using Fluid.Values; using System.Collections; using System.IO; using System.Reflection; @@ -37,6 +38,8 @@ public class TemplateRender : ITemplateRender _options.MemberAccessStrategy.Register(); _options.MemberAccessStrategy.Register(); + _options.Filters.AddFilter("from_agent", FromAgentFilter); + _parser.RegisterExpressionTag("link", (Expression expression, TextWriter writer, TextEncoder encoder, TemplateContext context) => { return RenderTag("link", expression, writer, encoder, context, services); @@ -99,7 +102,7 @@ public class TemplateRender : ITemplateRender value = await context.Model.GetValueAsync(TemplateRenderConstant.RENDER_AGENT, context); var agent = value?.ToObjectValue() as Agent; - var splited = Regex.Split(expStr, @"\s*from\s*", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant) + var splited = Regex.Split(expStr, @"\s*from_agent\s*", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant) .Where(x => !string.IsNullOrWhiteSpace(x)) .Select(x => x.Trim()) .ToArray(); @@ -142,6 +145,16 @@ public class TemplateRender : ITemplateRender return Completion.Normal; } + private static ValueTask FromAgentFilter( + FluidValue input, + FilterArguments arguments, + TemplateContext context) + { + var inputStr = input?.ToStringValue() ?? string.Empty; + var fromAgent = arguments.At(0).ToStringValue(); + return new StringValue($"{inputStr} from_agent {fromAgent}"); + } + private static bool IsStringType(Type type) { return type == typeof(string); From ee491369b0126f77f1e04eaeb5606756b60e8305 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 23 Jul 2025 15:29:44 -0500 Subject: [PATCH 20/28] update router instruction --- .../instructions/instruction.liquid | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index 775de66c..341557bb 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -5,7 +5,8 @@ Follow these steps to handle user request: 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. -5. If user is greeting, you can call function response_to_user with a greeting message. +5. Call function route_to_agent if user have specific requests, let agent handle and fill the required args, do not ask user to provide any args by yourself. +6. If user is greeting or do not have specific request, you can call function response_to_user with a greeting message. {% if routing_requirements and routing_requirements != empty %} [REQUIREMENTS] From 27dc42df8a058c23807afd1a8b348a6bfccbcc37 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 23 Jul 2025 15:40:18 -0500 Subject: [PATCH 21/28] update router instruction --- .../instructions/instruction.liquid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index 341557bb..1ada1850 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -5,8 +5,8 @@ Follow these steps to handle user request: 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. -5. Call function route_to_agent if user have specific requests, let agent handle and fill the required args, do not ask user to provide any args by yourself. -6. If user is greeting or do not have specific request, you can call function response_to_user with a greeting message. +5. Call function route_to_agent if user have specific requests and available agent to proceed. Do not ask user to provide any required args by yourself. The requested agent will handle and fill the required args internally. +6. If user is greeting or do not have specific request, then you can call function response_to_user with a greeting message. {% if routing_requirements and routing_requirements != empty %} [REQUIREMENTS] From 957a3e51a8db317291b062fc3201e676cbdf0d85 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 24 Jul 2025 15:15:53 -0500 Subject: [PATCH 22/28] resolve conflict --- .../BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs | 2 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs index 97cb89a4..e4cb28af 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -136,7 +136,7 @@ public class ChatHubObserver : IObserver private bool AllowSendingMessage() { var sidecar = _services.GetService(); - return sidecar == null || !sidecar.IsEnabled(); + return sidecar == null || !sidecar.IsEnabled; } private async Task GenerateSenderAction(string conversationId, ConversationSenderActionModel action) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 314697c8..0d54e635 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -259,10 +259,10 @@ public class ChatCompletionProvider : IChatCompletion var functionName = meta?.FunctionName; var toolCallId = meta?.ToolCallId; var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); - var functionArgument = string.Join(string.Empty, args); + var functionArguments = string.Join(string.Empty, args); #if DEBUG - _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArguments})"); #endif responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) @@ -271,7 +271,7 @@ public class ChatCompletionProvider : IChatCompletion MessageId = messageId, ToolCallId = toolCallId, FunctionName = functionName, - FunctionArgs = functionArgument + FunctionArgs = functionArguments }; } else if (choice.FinishReason.HasValue) From 74a00b8c38a9c64fe9bc6f011b6fa415253868f8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 24 Jul 2025 16:00:37 -0500 Subject: [PATCH 23/28] refine --- .../Observers/ChatHubObserver.cs | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs index e4cb28af..c15419ed 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs @@ -61,7 +61,7 @@ public class ChatHubObserver : IObserver SenderAction = SenderActionEnum.TypingOn }; - GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); + GenerateSenderAction(conv.ConversationId, action); } else if (value.EventName == AFTER_RECEIVE_LLM_STREAM_MESSAGE && message.IsStreaming) { @@ -85,7 +85,7 @@ public class ChatHubObserver : IObserver SenderAction = SenderActionEnum.TypingOff }; - GenerateSenderAction(conv.ConversationId, action).ConfigureAwait(false).GetAwaiter().GetResult(); + GenerateSenderAction(conv.ConversationId, action); } else if (value.EventName == ON_RECEIVE_LLM_STREAM_MESSAGE) { @@ -107,30 +107,7 @@ public class ChatHubObserver : IObserver }; } - OnReceiveAssistantMessage(value.EventName, model.ConversationId, model).ConfigureAwait(false).GetAwaiter().GetResult(); - } - - private async Task OnReceiveAssistantMessage(string @event, string conversationId, ChatResponseDto model) - { - try - { - var settings = _services.GetRequiredService(); - var chatHub = _services.GetRequiredService>(); - - if (settings.EventDispatchBy == EventDispatchType.Group) - { - await chatHub.Clients.Group(conversationId).SendAsync(@event, model); - } - else - { - var user = _services.GetRequiredService(); - await chatHub.Clients.User(user.Id).SendAsync(@event, model); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, $"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})"); - } + OnReceiveAssistantMessage(value.EventName, model.ConversationId, model); } private bool AllowSendingMessage() @@ -139,7 +116,30 @@ public class ChatHubObserver : IObserver return sidecar == null || !sidecar.IsEnabled; } - private async Task GenerateSenderAction(string conversationId, ConversationSenderActionModel action) + private void OnReceiveAssistantMessage(string @event, string conversationId, ChatResponseDto model) + { + try + { + var settings = _services.GetRequiredService(); + var chatHub = _services.GetRequiredService>(); + + if (settings.EventDispatchBy == EventDispatchType.Group) + { + chatHub.Clients.Group(conversationId).SendAsync(@event, model).ConfigureAwait(false).GetAwaiter().GetResult(); + } + else + { + var user = _services.GetRequiredService(); + chatHub.Clients.User(user.Id).SendAsync(@event, model).ConfigureAwait(false).GetAwaiter().GetResult(); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})"); + } + } + + private void GenerateSenderAction(string conversationId, ConversationSenderActionModel action) { try { @@ -147,12 +147,12 @@ public class ChatHubObserver : IObserver var chatHub = _services.GetRequiredService>(); if (settings.EventDispatchBy == EventDispatchType.Group) { - await chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action).ConfigureAwait(false).GetAwaiter().GetResult(); } else { var user = _services.GetRequiredService(); - await chatHub.Clients.User(user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + chatHub.Clients.User(user.Id).SendAsync(GENERATE_SENDER_ACTION, action).ConfigureAwait(false).GetAwaiter().GetResult(); } } catch (Exception ex) From b5dc4ea20e373303baed746782601d78c00e26f1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Jul 2025 10:28:27 -0500 Subject: [PATCH 24/28] upgrade openai --- Directory.Packages.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index cec2ac88..de12d90c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -44,8 +44,8 @@ - - + + @@ -147,4 +147,4 @@ - + \ No newline at end of file From 77ac9d22cf9ddb02c0dc35839808ea4187b63546 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 25 Jul 2025 12:54:58 -0500 Subject: [PATCH 25/28] gpt-image-1 --- .../Infrastructures/CompletionProvider.cs | 2 +- .../Image/ImageCompletionProvider.Generation.cs | 6 ++---- .../Providers/Image/ImageCompletionProvider.cs | 10 +++++----- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 4bf45d86..4409c8eb 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -213,7 +213,7 @@ public class CompletionProvider model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model; if (state.ContainsState("model")) { - model = state.GetState("model", model ?? "dall-e-3"); + model = state.GetState("model", model ?? "gpt-image-1"); } else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 2f55d917..5747df2b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -38,10 +38,8 @@ public partial class ImageCompletionProvider var options = new ImageGenerationOptions { - Size = size, - Quality = quality, - Style = style, - ResponseFormat = format + Quality = new GeneratedImageQuality("medium"), + Size = GeneratedImageSize.W1024xH1024 }; return (prompt, count, options); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 7cfda0f5..eaa03bb0 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -44,7 +44,7 @@ public partial class ImageCompletionProvider : IImageCompletion { generatedImage.ImageUrl = image?.ImageUri?.AbsoluteUri ?? string.Empty; } - else if (format == GeneratedImageFormat.Bytes) + else { var base64Str = string.Empty; var bytes = image?.ImageBytes?.ToArray(); @@ -125,16 +125,16 @@ public partial class ImageCompletionProvider : IImageCompletion private GeneratedImageFormat GetImageFormat(string format) { - var value = !string.IsNullOrEmpty(format) ? format : "uri"; + var value = !string.IsNullOrEmpty(format) ? format : "bytes"; GeneratedImageFormat retFormat; switch (value) { - case "bytes": - retFormat = GeneratedImageFormat.Bytes; + case "url": + retFormat = GeneratedImageFormat.Uri; break; default: - retFormat = GeneratedImageFormat.Uri; + retFormat = GeneratedImageFormat.Bytes; break; } From b15f580930ae9ba0a8337974b290e196463d51fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Jul 2025 15:13:52 -0500 Subject: [PATCH 26/28] add logging in twilio stream --- .../TwilioStreamMiddleware.cs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index ea4d5c50..3bcaf18b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.Routing; +using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; @@ -88,10 +89,15 @@ public class TwilioStreamMiddleware continue; } - var (eventType, data) = MapEvents(conn, receivedText); + var (eventType, data) = MapEvents(conn, receivedText, conversationId); if (eventType == "user_connected") { +#if DEBUG + _logger.LogCritical($"Start twilio stream connection for conversation ({conversationId})"); +#else + _logger.LogInformation($"Start twilio stream connection for conversation ({conversationId})"); +#endif // Connect to model await ConnectToModel(hub, webSocket); } @@ -115,6 +121,11 @@ public class TwilioStreamMiddleware } else if (eventType == "user_disconnected") { +#if DEBUG + _logger.LogCritical($"Disconnecting twilio stream connection for conversation ({conversationId})"); +#else + _logger.LogInformation($"Disconnecting twilio stream connection for conversation ({conversationId})"); +#endif await hub.Completer.Disconnect(); await HandleUserDisconnected(); } @@ -131,9 +142,19 @@ public class TwilioStreamMiddleware }); } - private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText) + private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText, string conversationId) { - var response = JsonSerializer.Deserialize(receivedText); + StreamEventResponse? response = new(); + + try + { + response = JsonSerializer.Deserialize(receivedText); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when deserializing twilio stream event response for conversation ({conversationId}) (response: {receivedText?.SubstringMax(30)})"); + } + conn.StreamId = response.StreamSid; string eventType = response.Event; string data = string.Empty; From ffd9823ccd5d59670bf6b7e5a5fbd73146378a0d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Jul 2025 16:25:59 -0500 Subject: [PATCH 27/28] minor change --- .../Providers/Chat/ChatCompletionProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 0d54e635..57b44252 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -237,7 +237,7 @@ public class ChatCompletionProvider : IChatCompletion textStream.Collect(text); #if DEBUG - _logger.LogCritical($"Content update: {text}"); + _logger.LogCritical($"Stream Content update: {text}"); #endif var content = new RoleDialogModel(AgentRole.Assistant, text) @@ -277,7 +277,7 @@ public class ChatCompletionProvider : IChatCompletion else if (choice.FinishReason.HasValue) { var allText = textStream.GetText(); - _logger.LogCritical($"Text Content: {allText}"); + _logger.LogInformation($"Stream text Content: {allText}"); responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) { From 031eb8730f8518e119517fdf5606ada967351de4 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Jul 2025 17:03:09 -0500 Subject: [PATCH 28/28] add default return --- .../BotSharp.Abstraction/MLTasks/IChatCompletion.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 51f89375..4ac62841 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -16,13 +16,13 @@ public interface IChatCompletion void SetModelName(string model); Task GetChatCompletions(Agent agent, - List conversations); + List conversations) => throw new NotImplementedException(); Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, - Func onFunctionExecuting); + Func onFunctionExecuting) => throw new NotImplementedException(); Task GetChatCompletionsStreamingAsync(Agent agent, - List conversations); + List conversations) => throw new NotImplementedException(); }