diff --git a/BotSharp.sln b/BotSharp.sln index d4890be1..b176747f 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -127,6 +127,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Rules", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.DeepSeekAI", "src\Plugins\BotSharp.Plugin.DeepSeekAI\BotSharp.Plugin.DeepSeekAI.csproj", "{AF329442-B48E-4B48-A18A-1C869D1BA6F5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "src\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj", "{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -519,6 +521,14 @@ Global {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|Any CPU.Build.0 = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.ActiveCfg = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.Build.0 = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.Build.0 = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.Build.0 = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.ActiveCfg = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -580,6 +590,7 @@ Global {F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AFD64412-4D6A-452E-82A2-79E5D8842E29} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA} + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/Directory.Build.props b/Directory.Build.props index 6716a3c7..3cb973b5 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ net8.0 12.0 - 4.0.0 + 4.1.0 true false diff --git a/Directory.Packages.props b/Directory.Packages.props index a031cec3..aa76cf61 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,24 +1,26 @@ 8.0.0 + 2.3.0 + true - - - - - - - + + + + + + + + - + - - - + + @@ -27,14 +29,96 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 2a2705db..68673f7d 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -10,8 +10,8 @@ - - + + diff --git a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj index 53f9559f..d19f2eea 100644 --- a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj +++ b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj @@ -9,17 +9,17 @@ - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index cf49da42..7c2466d9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -12,7 +12,7 @@ public interface IAgentService Task CreateAgent(Agent agent); Task RefreshAgents(); Task> GetAgents(AgentFilter filter); - Task> GetAgentOptions(); + Task> GetAgentOptions(List? agentIds = null); /// /// Load agent configurations and trigger hooks diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index f189d323..97e645d0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -8,7 +8,6 @@ Icon.png $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index fe575462..1abf7994 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -16,7 +16,10 @@ public class PageActionArgs /// /// This value has to be set to true if you want to get the page XHR/ Fetch responses /// - public bool OpenNewTab { get; set; } = false; + [JsonPropertyName("open_new_tab")] + public bool OpenNewTab { get; set; } = true; + [JsonPropertyName("open_blank_page")] + public bool OpenBlankPage { get; set; } = true; public bool EnableResponseCallback { get; set; } = false; @@ -47,4 +50,6 @@ public class PageActionArgs public int WaitTime { get; set; } public bool ReadInnerHTMLAsBody { get; set; } = false; + [JsonPropertyName("keep_browser_open")] + public bool KeepBrowserOpen { get; set; } = false; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 25149fe0..f876c449 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -3,4 +3,10 @@ namespace BotSharp.Abstraction.Browsing.Settings; public class WebBrowsingSettings { public string Driver { get; set; } = "Playwright"; + public bool Headless { get; set; } + // Default timeout in milliseconds + public float DefaultTimeout { get; set; } = 30000; + public bool IsEnableScreenshot { get; set; } + // Default wait time in seconds after page is opened + public int DefaultWaitTime { get; set; } = 5; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs index c13f26a8..e59622bb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs @@ -6,4 +6,5 @@ public static class MessageTypeName public const string Notification = "notification"; public const string FunctionCall = "function"; public const string Audio = "audio"; + public const string Error = "error"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 6d84103e..15f7e89e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -13,7 +13,7 @@ public interface IConversationService Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); Task UpdateConversationTitleAlias(string id, string titleAlias); - Task UpdateConversationTags(string conversationId, List tags); + Task UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags); Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request); Task> GetLastConversations(); Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds); @@ -27,20 +27,17 @@ public interface IConversationService /// If not null, delete messages while input a new message; otherwise delete messages only /// Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null); - Task> GetConversationContentLogs(string conversationId); - Task> GetConversationStateLogs(string conversationId); /// /// Send message to LLM /// /// - /// + /// + /// /// Received the response from AI Agent - /// This delegate is useful when you want to report progress on UI - /// This delegate is useful when you want to report progress on UI /// Task SendMessage(string agentId, - RoleDialogModel lastDalog, + RoleDialogModel lastDialog, PostbackMessageModel? replyMessage, Func onResponseReceived); @@ -64,12 +61,7 @@ public interface IConversationService void SaveStates(); - /// - /// Get conversation keys for searching - /// - /// search query - /// conversation limit - /// if pre-loading, then keys are not filter by the search query - /// - Task> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false); + Task> GetConversationStateSearhKeys(ConversationStateKeysFilter filter); + + Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index 07258402..ac1efb04 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -58,17 +58,13 @@ public class DialogElement [JsonPropertyName("payload")] public string? Payload { get; set; } - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("data")] - public object? Data { get; set; } - public DialogElement() { } public DialogElement(DialogMetaData meta, string content, string? richContent = null, - string? secondaryContent = null, string? secondaryRichContent = null, string? payload = null, object? data = null) + string? secondaryContent = null, string? secondaryRichContent = null, string? payload = null) { MetaData = meta; Content = content; @@ -76,7 +72,6 @@ public class DialogElement SecondaryContent = secondaryContent; SecondaryRichContent = secondaryRichContent; Payload = payload; - Data = data; } public override string ToString() diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 585112a2..6f96eec5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -108,7 +108,8 @@ public class RoleDialogModel : ITrackableMessage [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); - + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public string RenderedInstruction { get; set; } = string.Empty; private RoleDialogModel() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index bf2644dd..db9ccb65 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -47,6 +47,17 @@ public static class FileUtility return $"data:{contentType};base64,{base64}"; } + public static BinaryData BuildBinaryDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var binary = BinaryData.FromStream(stream); + stream.Close(); + + return binary; + } + public static string GetFileContentType(string fileName) { string contentType; diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..9db7c440 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -8,8 +8,12 @@ public class StateConst public const string NEXT_ACTION_REASON = "next_action_reason"; public const string USER_GOAL_AGENT = "user_goal_agent"; public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; + // lazy or eager + public const string ROUTING_MODE = "routing_mode"; + public const string LAZY_ROUTING_AGENT_ID = "lazy_routing_agent_id"; public const string LANGUAGE = "language"; public const string SUB_CONVERSATION_ID = "sub_conversation_id"; + public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs index 8eaee99b..c9fa1cd7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs @@ -7,4 +7,5 @@ public interface IInstructHook string SelfId { get; } Task BeforeCompletion(Agent agent, RoleDialogModel message); Task AfterCompletion(Agent agent, InstructResult result); + Task OnResponseGenerated(InstructResponseModel response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs index b59e4640..f19cca5c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs @@ -15,4 +15,9 @@ public class InstructHookBase : IInstructHook { return; } + + public virtual async Task OnResponseGenerated(InstructResponseModel response) + { + return; + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs new file mode 100644 index 00000000..62752bf4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructResponseModel +{ + public string? AgentId { get; set; } + public string Provider { get; set; } = default!; + public string Model { get; set; } = default!; + public string? TemplateName { get; set; } + public string UserMessage { get; set; } = default!; + public string? SystemInstruction { get; set; } + public string CompletionText { get; set; } = default!; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs new file mode 100644 index 00000000..d1f51348 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Instructs.Settings; + +public class InstructionSettings +{ + public bool EnableLog { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 3fc90816..fb368b0b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -60,6 +60,14 @@ public interface IKnowledgeService Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion + #region Snapshot + Task> GetVectorCollectionSnapshots(string collectionName); + Task CreateVectorCollectionSnapshot(string collectionName); + Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName); + Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData); + Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); + #endregion + #region Common Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs index 9d0662db..c4f63450 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Abstraction.Loggers; /// @@ -37,4 +39,13 @@ public interface IContentGeneratingHook /// /// Task OnRenderingTemplate(Agent agent, string name, string content) => Task.CompletedTask; + + /// + /// Realtime session updated + /// + /// + /// + /// + /// + Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs new file mode 100644 index 00000000..de843023 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -0,0 +1,53 @@ +using System.Text.Json; + +namespace BotSharp.Abstraction.Loggers.Models; + +public class InstructionLogModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } = default!; + + [JsonPropertyName("agent_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentId { get; set; } + + [JsonPropertyName("agent_name")] + [JsonIgnore] + public string? AgentName { get; set; } + + [JsonPropertyName("provider")] + public string Provider { get; set; } = default!; + + [JsonPropertyName("model")] + public string Model { get; set; } = default!; + + [JsonPropertyName("template_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TemplateName { get; set; } + + [JsonPropertyName("user_message")] + public string UserMessage { get; set; } = string.Empty; + + [JsonPropertyName("system_instruction")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SystemInstruction { get; set; } + + [JsonPropertyName("completion_text")] + public string CompletionText { get; set; } = string.Empty; + + [JsonPropertyName("user_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserId { get; set; } + + [JsonIgnore] + public string? UserName { get; set; } + + [JsonIgnore] + public Dictionary States { get; set; } = []; + + [JsonPropertyName("states")] + public Dictionary InnerStates { get; set; } = []; + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs new file mode 100644 index 00000000..564fcf37 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -0,0 +1,17 @@ +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories.Filters; + +namespace BotSharp.Abstraction.Loggers.Services; + +public interface ILoggerService +{ + #region Conversation + Task> GetConversationContentLogs(string conversationId, ConversationLogFilter filter); + Task> GetConversationStateLogs(string conversationId, ConversationLogFilter filter); + #endregion + + #region Instruction + Task> GetInstructionLogs(InstructLogFilter filter); + Task> GetInstructionLogSearchKeys(InstructLogKeysFilter filter); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs index 54ed2d7d..85fc84f2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -6,6 +6,8 @@ public interface IAudioCompletion { string Provider { get; } + string Model { get; } + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); Task GenerateAudioFromTextAsync(string text); diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 306e72e3..7cf52fe4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -7,6 +7,8 @@ public interface IChatCompletion /// string Provider { get; } + string Model { get; } + /// /// Set model name, one provider can consume different model or version(s) /// diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs index 323fe643..2b74a337 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs @@ -9,6 +9,8 @@ public interface IImageCompletion /// string Provider { get; } + string Model { get; } + /// /// Set model name, one provider can consume different model or version(s) /// diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 8e1d11d5..68a96dfa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -6,12 +6,11 @@ public interface IRealTimeCompletion { string Provider { get; } string Model { get; } - void SetModelName(string model); Task Connect(RealtimeHubConnection conn, Action onModelReady, - Action onModelAudioDeltaReceived, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, Action> onModelResponseDone, @@ -24,9 +23,11 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateInitialSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true); Task InsertConversationItem(RoleDialogModel message); + Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); + Task CancelModelResponse(); Task> OnResponsedDone(RealtimeHubConnection conn, string response); Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs index 63a15940..8fc549f7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs @@ -6,6 +6,7 @@ public interface ITextCompletion /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI /// string Provider { get; } + string Model { get; } /// /// Set model name, one provider can consume different model or version(s) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs index 79b3aff8..a96dfec9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs @@ -6,9 +6,14 @@ public interface ITextEmbedding /// The Embedding provider like Microsoft Azure, OpenAI, ClaudAI /// string Provider { get; } + string Model { get; } + + void SetModelName(string model); + + Task GetVectorAsync(string text); Task> GetVectorsAsync(List texts); - void SetModelName(string model); + void SetDimension(int dimension); int GetDimension(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs index 81286489..3cb9c73f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -3,10 +3,24 @@ namespace BotSharp.Abstraction.Models; public class KeyValue { [JsonPropertyName("key")] - public string Key { get; set; } + public string Key { get; set; } = string.Empty; [JsonPropertyName("value")] - public string Value { get; set; } + public string? Value { get; set; } + + public override string ToString() + { + return $"Key: {Key}, Value: {Value}"; + } +} + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } = string.Empty; + + [JsonPropertyName("value")] + public T? Value { get; set; } public override string ToString() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs index 90168872..b8609422 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Options; public class BotSharpOptions { - private readonly static JsonSerializerOptions defaultJsonOptions = new JsonSerializerOptions() + public readonly static JsonSerializerOptions defaultJsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs index c387d1ca..f347fb84 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs @@ -13,6 +13,9 @@ public class PluginMenuDef(string label, string? link = null, string? icon = nul [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Link { get; set; } = link; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public EmbeddingData? EmbeddingInfo { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? IsHeader { get; set; } @@ -30,3 +33,35 @@ public class PluginMenuDef(string label, string? link = null, string? icon = nul return $"{Label} {Link} {Weight}"; } } + +public class EmbeddingData +{ + /// + /// Embedding source, e.g., tableau + /// + public string Source { get; set; } = default!; + + /// + /// Embedding url + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Url { get; set; } + + /// + /// Html tag + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? HtmlTag { get; set; } + + /// + /// Javascript script src + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScriptSrc { get; set; } + + /// + /// Javascript script type, e.g., module + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScriptType { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs new file mode 100644 index 00000000..2ad8f1df --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Realtime; + +public interface IRealtimeHook +{ + string[] OnModelTranscriptPrompt(Agent agent); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 67d0f18c..23ecb654 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime.Models; using System.Net.WebSockets; @@ -8,5 +9,11 @@ namespace BotSharp.Abstraction.Realtime; /// public interface IRealtimeHub { - Task Listen(WebSocket userWebSocket, Func onUserMessageReceived); + RealtimeHubConnection HubConn { get; } + RealtimeHubConnection SetHubConnection(string conversationId); + + IRealTimeCompletion Completer { get; } + IRealTimeCompletion SetCompleter(string provider); + + Task Listen(WebSocket userWebSocket, Action onUserMessageReceived); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs new file mode 100644 index 00000000..a65a8985 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class ModelTurnDetection +{ + public int PrefixPadding { get; set; } = 300; + + public int SilenceDuration { get; set; } = 800; + + public float Threshold { get; set; } = 0.8f; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 60fec1dc..9fa3ee3b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -1,14 +1,34 @@ +using System.Collections.Concurrent; + namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; - public string EntryAgentId { get; set; } = null!; + public string? LastAssistantItemId { get; set; } = null!; + public long LatestMediaTimestamp { get; set; } + public long? ResponseStartTimestamp { get; set; } + public string KeypadInputBuffer { get; set; } = string.Empty; + public ConcurrentQueue MarkQueue { get; set; } = new(); + public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; public string Model { get; set; } = null!; public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; public Func OnModelUserInterrupted { get; set; } = null!; + + public void ResetResponseState() + { + MarkQueue.Clear(); + LastAssistantItemId = null; + ResponseStartTimestamp = null; + } + + public void ResetStreamState() + { + ResponseStartTimestamp = null; + LatestMediaTimestamp = 0; + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs new file mode 100644 index 00000000..30b6cfb8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeModelSettings +{ + public float Temperature { get; set; } = 0.8f; + public int MaxResponseOutputTokens { get; set; } = 512; + public ModelTurnDetection TurnDetection { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs index 23543049..d05c49c6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs @@ -23,9 +23,9 @@ public class ConversationFilter /// /// Check whether each key in the list is in the conversation states and its value equals to target value if not empty /// - public IEnumerable? States { get; set; } = []; + public List? States { get; set; } - public IEnumerable? Tags { get; set; } = []; + public List? Tags { get; set; } public static ConversationFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs new file mode 100644 index 00000000..59cf9d27 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class ConversationLogFilter +{ + public int Size { get; set; } = 20; + public DateTime StartTime { get; set; } = DateTime.UtcNow; + + public ConversationLogFilter() + { + + } + + public static ConversationLogFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs new file mode 100644 index 00000000..bff4b43c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs @@ -0,0 +1,21 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class ConversationStateKeysFilter +{ + public string? Query { get; set; } + public int KeyLimit { get; set; } = 10; + public int ConvLimit { get; set; } = 100; + public bool PreLoad { get; set; } + public List? AgentIds { get; set; } + public List? UserIds { get; set; } + + public ConversationStateKeysFilter() + { + + } + + public static ConversationStateKeysFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs new file mode 100644 index 00000000..c1040f94 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class InstructLogFilter : Pagination +{ + public List? AgentIds { get; set; } + public List? Providers { get; set; } + public List? Models { get; set; } + public List? TemplateNames { get; set; } + public List? UserIds { get; set; } + public List? States { get; set; } + + public static InstructLogFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs new file mode 100644 index 00000000..c3eade89 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs @@ -0,0 +1,21 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class InstructLogKeysFilter +{ + public string? Query { get; set; } + public int KeyLimit { get; set; } = 10; + public int LogLimit { get; set; } = 100; + public bool PreLoad { get; set; } + public List? AgentIds { get; set; } + public List? UserIds { get; set; } + + public InstructLogKeysFilter() + { + + } + + public static InstructLogKeysFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 235c449b..332ca340 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -132,7 +132,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); void UpdateConversationTitleAlias(string conversationId, string titleAlias) => throw new NotImplementedException(); - bool UpdateConversationTags(string conversationId, List tags) + bool UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) => throw new NotImplementedException(); bool AppendConversationTags(string conversationId, List tags) => throw new NotImplementedException(); @@ -148,7 +148,11 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); - List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) + List GetConversationStateSearchKeys(ConversationStateKeysFilter filter) + => throw new NotImplementedException(); + List GetConversationsToMigrate(int batchSize = 100) + => throw new NotImplementedException(); + bool MigrateConvsersationLatestStates(string conversationId) => throw new NotImplementedException(); #endregion @@ -160,17 +164,28 @@ public interface IBotSharpRepository : IHaveServiceProvider #region Conversation Content Log void SaveConversationContentLog(ContentLogOutputModel log) => throw new NotImplementedException(); - List GetConversationContentLogs(string conversationId) + DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) => throw new NotImplementedException(); #endregion #region Conversation State Log void SaveConversationStateLog(ConversationStateLogModel log) => throw new NotImplementedException(); - List GetConversationStateLogs(string conversationId) + DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) => throw new NotImplementedException(); #endregion + #region Instruction Log + bool SaveInstructionLogs(IEnumerable logs) + => throw new NotImplementedException(); + + PagedItems GetInstructionLogs(InstructLogFilter filter) + => throw new NotImplementedException(); + + List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + => throw new NotImplementedException(); + #endregion + #region Statistics BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs index da7d5c01..337d63c6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs @@ -13,10 +13,10 @@ public interface IRoutingContext bool IsEmpty { get; } string IntentName { get; set; } int AgentCount { get; } - void Push(string agentId, string? reason = null); - void Pop(string? reason = null); - void PopTo(string agentId, string reason); - void Replace(string agentId, string? reason = null); + void Push(string agentId, string? reason = null, bool updateLazyRouting = true); + void Pop(string? reason = null, bool updateLazyRouting = true); + void PopTo(string agentId, string reason, bool updateLazyRouting = true); + void Replace(string agentId, string? reason = null, bool updateLazyRouting = true); void Empty(string? reason = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs new file mode 100644 index 00000000..6fc2ce03 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Routing.Models; + +public class FallbackArgs +{ + [JsonPropertyName("fallback_reason")] + public string Reason { get; set; } = null!; + + [JsonPropertyName("user_question")] + public string Question { get; set; } = null; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index 5b1c6a2a..bc9a6fa8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -12,10 +12,6 @@ public class RoutingArgs [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? NextActionReason { get; set; } - [JsonPropertyName("reason")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Reason { get; set; } - [JsonPropertyName("conversation_end")] public bool ConversationEnd { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs index e4852307..f5e379c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs @@ -2,10 +2,13 @@ namespace BotSharp.Abstraction.Tasks.Models; public class AgentTask { + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string Id { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; public string? Description { get; set; } public bool Enabled { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string Content { get; set; } = string.Empty; [JsonIgnore(Condition = JsonIgnoreCondition.Always)] @@ -19,9 +22,15 @@ public class AgentTask /// public string Status { get; set; } = TaskStatus.New; - public DateTime? LastExecutedDateTime { get; set; } - public DateTime? NextExecutionDateTime { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTime? LastExecutionTime { get; set; } - public DateTime CreatedDateTime { get; set; } - public DateTime UpdatedDateTime { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTime? NextExecutionTime { get; set; } + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } + + [JsonPropertyName("updated_time")] + public DateTime UpdatedTime { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 15b04f28..1499ea17 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -10,7 +10,7 @@ public interface IUserService Task> GetUsers(UserFilter filter); Task> SearchLoginUsers(User filter); Task GetUserDetails(string userId, bool includeAgent = false); - Task IsAdminUser(string userId); + Task<(bool, User?)> IsAdminUser(string userId); Task GetUserAuthorizations(IEnumerable? agentIds = null); Task UpdateUser(User user, bool isUpdateUserAgents = false); Task CreateUser(User user); diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs new file mode 100644 index 00000000..fc56e16e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Utilities; + +public class DateTimePagination : PagedItems +{ + public DateTime? NextTime { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index 65f7b78e..af222d0d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -92,4 +92,20 @@ public static class StringExtensions return JsonSerializer.Deserialize(text, options); } + + public static bool IsPrimitiveValue(this string value) + { + return int.TryParse(value, out _) || + long.TryParse(value, out _) || + double.TryParse(value, out _) || + float.TryParse(value, out _) || + bool.TryParse(value, out _) || + char.TryParse(value, out _) || + byte.TryParse(value, out _) || + sbyte.TryParse(value, out _) || + short.TryParse(value, out _) || + ushort.TryParse(value, out _) || + uint.TryParse(value, out _) || + ulong.TryParse(value, out _); + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 66f7727a..ff090da6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,14 +6,36 @@ public interface IVectorDb { string Provider { get; } - Task DoesCollectionExist(string collectionName); - Task> GetCollections(); - Task> GetPagedCollectionData(string collectionName, VectorFilter filter); - Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dimension); - Task DeleteCollection(string collectionName); - Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); - Task DeleteCollectionData(string collectionName, List ids); - Task DeleteCollectionAllData(string collectionName); + Task DoesCollectionExist(string collectionName) + => throw new NotImplementedException(); + Task> GetCollections() + => throw new NotImplementedException(); + Task> GetPagedCollectionData(string collectionName, VectorFilter filter) + => throw new NotImplementedException(); + Task> GetCollectionData(string collectionName, IEnumerable ids, + bool withPayload = false, bool withVector = false) + => throw new NotImplementedException(); + Task CreateCollection(string collectionName, int dimension) + => throw new NotImplementedException(); + Task DeleteCollection(string collectionName) + => throw new NotImplementedException(); + Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) + => throw new NotImplementedException(); + Task> Search(string collectionName, float[] vector, IEnumerable? fields, + int limit = 5, float confidence = 0.5f, bool withVector = false) + => throw new NotImplementedException(); + Task DeleteCollectionData(string collectionName, List ids) + => throw new NotImplementedException(); + Task DeleteCollectionAllData(string collectionName) + => throw new NotImplementedException(); + Task> GetCollectionSnapshots(string collectionName) + => throw new NotImplementedException(); + Task CreateCollectionShapshot(string collectionName) + => throw new NotImplementedException(); + Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName) + => throw new NotImplementedException(); + Task RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) + => throw new NotImplementedException(); + Task DeleteCollectionShapshot(string collectionName, string snapshotName) + => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs new file mode 100644 index 00000000..a8405a80 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionSnapshot +{ + public string Name { get; set; } = default!; + public long Size { get; set; } + public DateTime CreatedTime { get; set; } + public string? CheckSum { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj index ae4c9fef..ff6c2368 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj +++ b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj @@ -29,7 +29,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 5cbf9028..cf4e1846 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -97,7 +97,7 @@ public class CrontabService : ICrontabService, ITaskFeeder Status = TaskStatus.Scheduled, Enabled = !x.Disabled, Description = cron.Description, - LastExecutedDateTime = cron.LastExecutionTime + LastExecutionTime = cron.LastExecutionTime })); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj new file mode 100644 index 00000000..25218b08 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -0,0 +1,18 @@ + + + + $(TargetFramework) + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(SolutionDir)packages + enable + enable + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs new file mode 100644 index 00000000..3d45ecfd --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -0,0 +1,64 @@ +using BotSharp.Abstraction.Utilities; + +namespace BotSharp.Core.Realtime.Hooks; + +public class RealtimeConversationHook : ConversationHookBase, IConversationHook +{ + private readonly IServiceProvider _services; + public RealtimeConversationHook(IServiceProvider services) + { + _services = services; + } + + public async Task OnFunctionExecuting(RoleDialogModel message) + { + var hub = _services.GetRequiredService(); + if (hub.HubConn == null) + { + return; + } + // Save states + var states = _services.GetRequiredService(); + states.SaveStateByArgs(message.FunctionArgs?.JsonContent() ?? JsonDocument.Parse("{}")); + } + + public async Task OnFunctionExecuted(RoleDialogModel message) + { + var hub = _services.GetRequiredService(); + if (hub.HubConn == null) + { + return; + } + var routing = _services.GetRequiredService(); + + message.Role = AgentRole.Function; + + if (message.FunctionName == "route_to_agent") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") ?? new(); + message.Content = $"I'm your AI assistant '{inst.AgentName}' to help with: '{inst.NextActionReason}'"; + hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + var instruction = await hub.Completer.UpdateSession(hub.HubConn); + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user task: {inst.NextActionReason}"); + } + else if (message.FunctionName == "util-routing-fallback_to_router") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") ?? new(); + message.Content = $"Returned to Router due to {inst.Reason}"; + hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + var instruction = await hub.Completer.UpdateSession(hub.HubConn); + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference(instruction); + } + else + { + // Update session for changed states + var instruction = await hub.Completer.UpdateSession(hub.HubConn); + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference(instruction); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs new file mode 100644 index 00000000..fa9ee268 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; +using BotSharp.Core.Realtime.Hooks; +using BotSharp.Core.Realtime.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Realtime; + +public class RealtimePlugin : IBotSharpPlugin +{ + public string Id => "68c1c737-5c21-49de-b141-cd5c8d9bf978"; + public string Name => "Realtime Hub"; + public string? IconUrl => "https://thumbs.dreamstime.com/b/microphone-icon-sound-waves-voice-command-recording-message-sign-349007898.jpg"; + public string Description => "Build low-latency, multi-modal experiences with the Realtime API."; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("RealtimeModel"); + }); + + services.AddScoped(); + services.AddScoped(); + } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs new file mode 100644 index 00000000..55378b01 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -0,0 +1,284 @@ +namespace BotSharp.Core.Realtime.Services; + +public class RealtimeHub : IRealtimeHub +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private RealtimeHubConnection _conn; + public RealtimeHubConnection HubConn => _conn; + + private IRealTimeCompletion _completer; + public IRealTimeCompletion Completer => _completer; + + public RealtimeHub(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Listen(WebSocket userWebSocket, + Action onUserMessageReceived) + { + var buffer = new byte[1024 * 16]; + WebSocketReceiveResult result; + + + do + { + result = await userWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + onUserMessageReceived(receivedText); + + if (_conn.Event == "user_connected") + { + await ConnectToModel(userWebSocket); + } + else if (_conn.Event == "user_data_received") + { + await _completer.AppenAudioBuffer(_conn.Data); + } + else if (_conn.Event == "user_dtmf_received") + { + await HandleUserDtmfReceived(); + } + else if (_conn.Event == "user_disconnected") + { + await _completer.Disconnect(); + await HandleUserDisconnected(); + } + } while (!result.CloseStatus.HasValue); + + await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task ConnectToModel(WebSocket userWebSocket) + { + var hookProvider = _services.GetRequiredService(); + var convService = _services.GetRequiredService(); + convService.SetConversationId(_conn.ConversationId, []); + var conversation = await convService.GetConversation(_conn.ConversationId); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conversation.AgentId); + _conn.CurrentAgentId = agent.Id; + + // Set model + var model = agent.LlmConfig.Model; + if (!model.Contains("-realtime-")) + { + var llmProviderService = _services.GetRequiredService(); + model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name; + } + + _completer.SetModelName(model); + _conn.Model = model; + + var routing = _services.GetRequiredService(); + routing.Context.Push(agent.Id); + + var storage = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(); + if (dialogs.Count == 0) + { + dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); + storage.Append(_conn.ConversationId, dialogs.First()); + } + + routing.Context.SetDialogs(dialogs); + + var states = _services.GetRequiredService(); + + await _completer.Connect(_conn, + onModelReady: async () => + { + // Not TriggerModelInference, waiting for user utter. + var instruction = await _completer.UpdateSession(_conn); + + // Trigger model inference if there is no audio file in the conversation + if (!states.ContainsState("init_audio_file")) + { + if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) + { + await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + } + else + { + await _completer.TriggerModelInference("Reply based on the conversation context."); + } + } + else + { + // Push dialogs into model context + foreach (var message in dialogs) + { + await _completer.InsertConversationItem(message); + } + + await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement."); + } + }, + onModelAudioDeltaReceived: async (audioDeltaData, itemId) => + { + var data = _conn.OnModelMessageReceived(audioDeltaData); + await SendEventToUser(userWebSocket, data); + + // If this is the first delta of a new response, set the start timestamp + if (!_conn.ResponseStartTimestamp.HasValue) + { + _conn.ResponseStartTimestamp = _conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {_conn.ResponseStartTimestamp}ms"); + } + // Record last assistant item ID for interruption handling + if (!string.IsNullOrEmpty(itemId)) + { + _conn.LastAssistantItemId = itemId; + } + + // Send mark messages to Media Streams so we know if and when AI response playback is finished + await SendMark(userWebSocket, _conn); + }, + onModelAudioResponseDone: async () => + { + var data = _conn.OnModelAudioResponseDone(); + await SendEventToUser(userWebSocket, data); + }, + onAudioTranscriptDone: async transcript => + { + + }, + onModelResponseDone: async messages => + { + foreach (var message in messages) + { + // Invoke function + if (message.MessageType == MessageTypeName.FunctionCall && + !string.IsNullOrEmpty(message.FunctionName)) + { + await routing.InvokeFunction(message.FunctionName, message); + } + else + { + // append output audio transcript to conversation + dialogs.Add(message); + storage.Append(_conn.ConversationId, message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnResponseGenerated(message); + } + } + } + }, + onConversationItemCreated: async response => + { + + }, + onInputAudioTranscriptionCompleted: async message => + { + // append input audio transcript to conversation + dialogs.Add(message); + storage.Append(_conn.ConversationId, message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnMessageReceived(message); + } + }, + onUserInterrupted: async () => + { + // Reset states + _conn.ResetResponseState(); + + var data = _conn.OnModelUserInterrupted(); + await SendEventToUser(userWebSocket, data); + }); + } + + private async Task SendMark(WebSocket userWebSocket, RealtimeHubConnection conn) + { + if (!string.IsNullOrEmpty(conn.StreamId)) + { + var markEvent = new + { + @event = "mark", + streamSid = conn.StreamId, + mark = new { name = "responsePart" } + }; + await SendEventToUser(userWebSocket, markEvent); + conn.MarkQueue.Enqueue("responsePart"); + } + } + + private async Task HandleUserDtmfReceived() + { + var routing = _services.GetRequiredService(); + var hookProvider = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(_conn.CurrentAgentId); + var dialogs = routing.Context.GetDialogs(); + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(_conn.ConversationId); + + var message = new RoleDialogModel(AgentRole.User, _conn.Data) + { + CurrentAgentId = routing.Context.GetCurrentAgentId() + }; + dialogs.Add(message); + + var storage = _services.GetRequiredService(); + storage.Append(_conn.ConversationId, message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnMessageReceived(message); + } + + await _completer.InsertConversationItem(message); + var instruction = await _completer.UpdateSession(_conn); + await _completer.TriggerModelInference($"{instruction}\r\n\r\nReply based on the user input: {message.Content}"); + } + + private async Task HandleUserDisconnected() + { + + } + + private async Task SendEventToUser(WebSocket webSocket, object message) + { + var data = JsonSerializer.Serialize(message); + var buffer = Encoding.UTF8.GetBytes(data); + await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } + + public RealtimeHubConnection SetHubConnection(string conversationId) + { + _conn = new RealtimeHubConnection + { + ConversationId = conversationId + }; + + return _conn; + } + + public IRealTimeCompletion SetCompleter(string provider) + { + _completer = _services.GetServices().First(x => x.Provider == provider); + return _completer; + } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Using.cs b/src/Infrastructure/BotSharp.Core.Realtime/Using.cs new file mode 100644 index 00000000..68c2458a --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Using.cs @@ -0,0 +1,17 @@ +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.DependencyInjection; + +global using System.Net.WebSockets; +global using System.Text; +global using System.Text.Json; + +global using BotSharp.Abstraction.Realtime; +global using BotSharp.Abstraction.Realtime.Models; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Conversations.Enums; +global using BotSharp.Abstraction.Routing.Models; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Conversations.Models; diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index d4ff408b..5becbec6 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -185,8 +185,8 @@ public partial class AgentService Enabled = parsedTask.Enabled, Content = parsedTask.Content, AgentId = agentId, - CreatedDateTime = parsedTask.CreatedDateTime, - UpdatedDateTime = parsedTask.UpdatedDateTime + CreatedTime = parsedTask.CreatedTime, + UpdatedTime = parsedTask.UpdatedTime }; tasks.Add(task); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index ec95a51e..092273fa 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -30,9 +30,12 @@ public partial class AgentService #if !DEBUG [SharpCache(10, perInstanceCache: true)] #endif - public async Task> GetAgentOptions() + public async Task> GetAgentOptions(List? agentIds) { - var agents = _db.GetAgents(AgentFilter.Empty()); + var agents = _db.GetAgents(new AgentFilter + { + AgentIds = !agentIds.IsNullOrEmpty() ? agentIds : null + }); return agents?.Select(x => new IdName(x.Id, x.Name))?.OrderBy(x => x.Name)?.ToList() ?? []; } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index eb0497ae..c9b3c90c 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -18,7 +18,7 @@ public partial class AgentService var userIdentity = _services.GetRequiredService(); var userService = _services.GetRequiredService(); - var isValid = await userService.IsAdminUser(userIdentity.Id); + var (isValid, _) = await userService.IsAdminUser(userIdentity.Id); if (!isValid) { return "Unauthorized user."; diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6bc7596c..00e1f30a 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -6,7 +6,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true @@ -66,6 +65,8 @@ + + @@ -82,6 +83,7 @@ + @@ -146,6 +148,15 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs index 742b39e9..6b10ad81 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs @@ -16,7 +16,6 @@ using BotSharp.Abstraction.Templating; using BotSharp.Core.Templating; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Realtime; -using BotSharp.Core.Realtime; namespace BotSharp.Core; @@ -173,7 +172,5 @@ public static class BotSharpCoreExtensions }); services.AddSingleton(loader); - - services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs deleted file mode 100644 index 7b583f3f..00000000 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs +++ /dev/null @@ -1,22 +0,0 @@ -using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Repositories; - -namespace BotSharp.Core.Conversations.Services; - -public partial class ConversationService -{ - public async Task> GetConversationContentLogs(string conversationId) - { - var db = _services.GetRequiredService(); - var logs = db.GetConversationContentLogs(conversationId); - return await Task.FromResult(logs); - } - - - public async Task> GetConversationStateLogs(string conversationId) - { - var db = _services.GetRequiredService(); - var logs = db.GetConversationStateLogs(conversationId); - return await Task.FromResult(logs); - } -} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs new file mode 100644 index 00000000..23012706 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs @@ -0,0 +1,98 @@ +using NetTopologySuite.Algorithm; +using System.Diagnostics; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10) + { + var db = _services.GetRequiredService(); + var isSuccess = true; + var errorCount = 0; + var batchNum = 0; + var info = string.Empty; + var error = string.Empty; + +#if DEBUG + Console.WriteLine($"\r\n#Start migrating Conversation Latest States...\r\n"); +#else + _logger.LogInformation($"#Start migrating Conversation Latest States..."); +#endif + var sw = Stopwatch.StartNew(); + + var convIds = db.GetConversationsToMigrate(batchSize); + + while (!convIds.IsNullOrEmpty()) + { + batchNum++; + var innerSw = Stopwatch.StartNew(); +#if DEBUG + Console.WriteLine($"\r\n#Start migrating Conversation Latest States (batch number: {batchNum})\r\n"); +#else + _logger.LogInformation($"#Start migrating Conversation Latest States (batch number: {batchNum})"); +#endif + + for (int i = 0; i < convIds.Count; i++) + { + var convId = convIds.ElementAt(i); + try + { + var done = db.MigrateConvsersationLatestStates(convId); + info = $"Conversation {convId} latest states have been migrated ({i + 1}/{convIds.Count})!"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + } + catch (Exception ex) + { + errorCount++; + error = $"Conversation {convId} latest states fail to be migrated! ({i + 1}/{convIds.Count})\r\n{ex.Message}\r\n{ex.InnerException}"; +#if DEBUG + Console.WriteLine($"\r\n{error}\r\n"); +#else + _logger.LogError($"{error}"); +#endif + } + } + + if (errorCount >= errorLimit) + { + error = $"\r\nErrors exceed limit => stop the migration!\r\n"; +#if DEBUG + Console.WriteLine($"{error}"); +#else + _logger.LogError($"{error}"); +#endif + innerSw.Stop(); + isSuccess = false; + break; + } + + innerSw.Stop(); + info = $"#Done migrating Conversation Latest States (batch number: {batchNum}) " + + $"(Total time: {innerSw.Elapsed.Hours} hrs, {innerSw.Elapsed.Minutes} mins, {innerSw.Elapsed.Seconds} seconds)"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + + await Task.Delay(100); + convIds = db.GetConversationsToMigrate(batchSize); + } + + sw.Stop(); + info = $"#Done with migrating Conversation Latest States! " + + $"(Total time: {sw.Elapsed.Days} days, {sw.Elapsed.Hours} hrs, {sw.Elapsed.Minutes} mins, {sw.Elapsed.Seconds} seconds)"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + + return isSuccess; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index f60d32cf..3967aaaa 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Routing.Settings; @@ -36,7 +37,17 @@ public partial class ConversationService // Enqueue receiving agent first in case it stop completion by OnMessageReceived var routing = _services.GetRequiredService(); routing.Context.SetMessageId(_conversationId, message.MessageId); - routing.Context.Push(agent.Id, reason: "request started"); + + // Check the routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false); + + if (routingMode == "lazy") + { + message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId); + routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false); + } // Save payload in order to assign the payload before hook is invoked if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) @@ -77,7 +88,7 @@ public partial class ConversationService { agent = await agentService.LoadAgent(message.CurrentAgentId); } - + if (agent.Type == AgentType.Routing) { response = await routing.InstructLoop(message, dialogs); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 0cec37d8..9fc97a72 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -59,10 +59,10 @@ public partial class ConversationService : IConversationService return conversation; } - public async Task UpdateConversationTags(string conversationId, List tags) + public async Task UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) { var db = _services.GetRequiredService(); - return db.UpdateConversationTags(conversationId, tags); + return db.UpdateConversationTags(conversationId, toAddTags, toDeleteTags); } public async Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request) @@ -222,17 +222,26 @@ public partial class ConversationService : IConversationService _state.Save(); } - public async Task> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false) + public async Task> GetConversationStateSearhKeys(ConversationStateKeysFilter filter) { + if (filter == null) + { + filter = ConversationStateKeysFilter.Empty(); + } + var keys = new List(); - if (!preload && string.IsNullOrWhiteSpace(query)) + if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query)) { return keys; } + var userService = _services.GetRequiredService(); var db = _services.GetRequiredService(); - keys = db.GetConversationStateSearchKeys(convUpperLimit: convLimit); - keys = preload ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); - return keys.OrderBy(x => x).Take(keyLimit).ToList(); + + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); + filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : []; + keys = db.GetConversationStateSearchKeys(filter); + keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList(); } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 4d229866..5d19e2d0 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -220,7 +220,6 @@ public class ConversationStateService : IConversationStateService { if (_conversationId == null || _sidecar?.IsEnabled() == true) { - Reset(); return; } @@ -253,7 +252,6 @@ public class ConversationStateService : IConversationStateService } _db.UpdateConversationStates(_conversationId, states); - Reset(); _logger.LogInformation($"Saved states of conversation {_conversationId}"); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 89e9fcaa..75d1a501 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -52,8 +52,7 @@ public class ConversationStorage : IConversationStorage MetaData = meta, Content = dialog.Content, SecondaryContent = dialog.SecondaryContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } else @@ -84,8 +83,7 @@ public class ConversationStorage : IConversationStorage SecondaryContent = dialog.SecondaryContent, RichContent = richContent, SecondaryRichContent = secondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } @@ -123,8 +121,7 @@ public class ConversationStorage : IConversationStorage MetaData = meta, Content = dialog.Content, SecondaryContent = dialog.SecondaryContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } else @@ -155,8 +152,7 @@ public class ConversationStorage : IConversationStorage SecondaryContent = dialog.SecondaryContent, RichContent = richContent, SecondaryRichContent = secondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } } @@ -200,8 +196,7 @@ public class ConversationStorage : IConversationStorage RichContent = richContent, SecondaryContent = secondaryContent, SecondaryRichContent = secondaryRichContent, - Payload = payload, - Data = dialog.Data + Payload = payload }; results.Add(record); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 9fa5c718..0698c0cb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -60,12 +60,15 @@ public class TokenStatistics : ITokenStatistics stat.SetState("llm_total_cost", total_cost, isNeedVersion: false, source: StateSource.Application); // Save stats + var metric = StatsMetric.AgentLlmCost; + var dim = "agent"; + var agentId = message.CurrentAgentId ?? string.Empty; var globalStats = _services.GetRequiredService(); var body = new BotSharpStatsInput { - Metric = StatsMetric.AgentLlmCost, - Dimension = "agent", - DimRefVal = message.CurrentAgentId, + Metric = metric, + Dimension = dim, + DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ @@ -75,7 +78,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats("global-llm-cost", body); + globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body); } public void PrintStatistics() @@ -110,6 +113,10 @@ public class TokenStatistics : ITokenStatistics public void StopTimer() { + if (_timer == null) + { + return; + } _timer.Stop(); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 4de657c3..2976ab47 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Instructs; using System.IO; namespace BotSharp.Core.Files.Services; @@ -6,10 +8,11 @@ public partial class FileInstructService { public async Task ReadImages(string? provider, string? model, string text, IEnumerable images, string? agentId = null) { + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { - Id = agentId ?? Guid.Empty.ToString(), + Id = innerAgentId, }, new List { new RoleDialogModel(AgentRole.User, text) @@ -17,16 +20,55 @@ public partial class FileInstructService Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() } }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text, string? agentId = null) { + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); var message = await completion.GetImageGeneration(new Agent() { - Id = agentId ?? Guid.Empty.ToString(), + Id = innerAgentId, }, new RoleDialogModel(AgentRole.User, text)); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message; } @@ -37,6 +79,7 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image url or data!"); } + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var bytes = await DownloadFile(image); using var stream = new MemoryStream(); @@ -46,10 +89,29 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { - Id = agentId ?? Guid.Empty.ToString() + Id = innerAgentId }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = string.Empty, + CompletionText = message.Content + }); + } + return message; } @@ -60,6 +122,7 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image url or data!"); } + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var bytes = await DownloadFile(image); using var stream = new MemoryStream(); @@ -69,10 +132,29 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = agentId ?? Guid.Empty.ToString() + Id = innerAgentId }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message; } @@ -84,6 +166,7 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image/mask url or data"); } + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var imageBytes = await DownloadFile(image); var maskBytes = await DownloadFile(mask); @@ -100,11 +183,30 @@ public partial class FileInstructService var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = agentId ?? Guid.Empty.ToString() + Id = innerAgentId }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index e1362455..9f2132a1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Files.Converters; +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Instructs; namespace BotSharp.Core.Files.Services; @@ -23,11 +25,12 @@ public partial class FileInstructService var images = await ConvertPdfToImages(pdfFiles); if (images.IsNullOrEmpty()) return content; + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model, modelId: modelId ?? "gpt-4", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { - Id = agentId ?? Guid.Empty.ToString(), + Id = innerAgentId, }, new List { new RoleDialogModel(AgentRole.User, prompt) @@ -35,6 +38,25 @@ public partial class FileInstructService Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = prompt, + CompletionText = message.Content + }); + } + return message.Content; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs index 7b4596ff..c44cb874 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Instructs.Settings; using BotSharp.Abstraction.Plugins.Models; +using BotSharp.Abstraction.Settings; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Instructs; @@ -11,13 +13,25 @@ public class InsturctionPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("Instruction"); + }); } public bool AttachMenu(List menu) { var section = menu.First(x => x.Label == "Apps"); - menu.Add(new PluginMenuDef("Instruction", link: "page/instruction", icon: "bx bx-book-content", weight: section.Weight + 5)); + menu.Add(new PluginMenuDef("Instruction", icon: "bx bx-book-content", weight: section.Weight + 5) + { + SubMenu = new List + { + new PluginMenuDef("Testing", link: "page/instruction/testing"), + new PluginMenuDef("Log", link: "page/instruction/log") + } + }); + return true; } } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index 1d9dd1d0..36f5b523 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -43,6 +43,9 @@ public partial class InstructService } } + var provider = string.Empty; + var model = string.Empty; + // Render prompt var prompt = string.IsNullOrEmpty(templateName) ? agentService.RenderedInstruction(agent) : @@ -57,11 +60,18 @@ public partial class InstructService }; if (completer is ITextCompletion textCompleter) { + instruction = null; + provider = textCompleter.Provider; + model = textCompleter.Model; + var result = await textCompleter.GetCompletion(prompt, agentId, message.MessageId); response.Text = result; } else if (completer is IChatCompletion chatCompleter) { + provider = chatCompleter.Provider; + model = chatCompleter.Model; + if (instruction == "#TEMPLATE#") { instruction = prompt; @@ -93,6 +103,16 @@ public partial class InstructService } await hook.AfterCompletion(agent, response); + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = agentId, + Provider = provider, + Model = model, + TemplateName = templateName, + UserMessage = prompt, + SystemInstruction = instruction, + CompletionText = response.Text + }); } return response; diff --git a/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs b/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs new file mode 100644 index 00000000..bc99b9c8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Loggers; + +public class LoggerPlugin : IBotSharpPlugin +{ + public string Id => "ea1aade7-7e29-4f13-a78b-2b1835aa4fea"; + public string Name => "Logger"; + public string Description => "Provide log service"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs new file mode 100644 index 00000000..c424d237 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Loggers.Models; + +namespace BotSharp.Core.Loggers.Services; + +public partial class LoggerService +{ + public async Task> GetConversationContentLogs(string conversationId, ConversationLogFilter filter) + { + if (filter == null) + { + filter = ConversationLogFilter.Empty(); + } + + var db = _services.GetRequiredService(); + var logs = db.GetConversationContentLogs(conversationId, filter); + return await Task.FromResult(logs); + } + + + public async Task> GetConversationStateLogs(string conversationId, ConversationLogFilter filter) + { + if (filter == null) + { + filter = ConversationLogFilter.Empty(); + } + + var db = _services.GetRequiredService(); + var logs = db.GetConversationStateLogs(conversationId, filter); + return await Task.FromResult(logs); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs new file mode 100644 index 00000000..42ad45ab --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs @@ -0,0 +1,84 @@ +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users.Models; + +namespace BotSharp.Core.Loggers.Services; + +public partial class LoggerService +{ + public async Task> GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var userService = _services.GetRequiredService(); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); + if (!isAdmin && user?.Id == null) return new(); + + filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null; + + var agents = new List(); + var users = new List(); + + var db = _services.GetRequiredService(); + var logs = db.GetInstructionLogs(filter); + var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); + var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList(); + agents = db.GetAgents(new AgentFilter + { + AgentIds = agentIds, + Pager = new Pagination { Size = filter.Size } + }); + + if (isAdmin) + { + users = db.GetUserByIds(userIds); + } + + var items = logs.Items.Select(x => + { + x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + + if (!isAdmin) + { + x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null; + } + else + { + var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null; + x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null; + } + return x; + }).ToList(); + + return new PagedItems + { + Items = items, + Count = logs.Count + }; + } + + public async Task> GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + { + if (filter == null) + { + filter = InstructLogKeysFilter.Empty(); + } + + var keys = new List(); + if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query)) + { + return keys; + } + + var userService = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); + filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null; + keys = db.GetInstructionLogSearchKeys(filter); + keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs new file mode 100644 index 00000000..bd9a5147 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Core.Loggers.Services; + +public partial class LoggerService : ILoggerService +{ + private readonly IServiceProvider _services; + private readonly IUserIdentity _user; + private readonly ILogger _logger; + + public LoggerService( + IServiceProvider services, + IUserIdentity user, + ILogger logger) + { + _services = services; + _user = user; + _logger = logger; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs deleted file mode 100644 index c44f74d0..00000000 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ /dev/null @@ -1,170 +0,0 @@ -using BotSharp.Abstraction.Realtime; -using System.Net.WebSockets; -using System; -using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Agents.Models; - -namespace BotSharp.Core.Realtime; - -public class RealtimeHub : IRealtimeHub -{ - private readonly IServiceProvider _services; - private readonly ILogger _logger; - public RealtimeHub(IServiceProvider services, ILogger logger) - { - _services = services; - _logger = logger; - } - - public async Task Listen(WebSocket userWebSocket, - Func onUserMessageReceived) - { - var buffer = new byte[1024 * 4]; - WebSocketReceiveResult result; - - var llmProviderService = _services.GetRequiredService(); - var model = llmProviderService.GetProviderModel("openai", "gpt-4", - realTime: true).Name; - - var completer = _services.GetServices().First(x => x.Provider == "openai"); - completer.SetModelName(model); - - do - { - result = await userWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); - string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); - _logger.LogDebug($"Received from user: {receivedText}"); - if (string.IsNullOrEmpty(receivedText)) - { - continue; - } - - var conn = onUserMessageReceived(receivedText); - conn.Model = model; - - if (conn.Event == "user_connected") - { - await ConnectToModel(completer, userWebSocket, conn); - } - else if (conn.Event == "user_data_received") - { - await completer.AppenAudioBuffer(conn.Data); - } - else if (conn.Event == "user_disconnected") - { - await completer.Disconnect(); - } - } while (!result.CloseStatus.HasValue); - - await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); - } - - private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn) - { - var hookProvider = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - - var convService = _services.GetRequiredService(); - convService.SetConversationId(conn.ConversationId, []); - var conversation = await convService.GetConversation(conn.ConversationId); - - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conversation.AgentId); - conn.EntryAgentId = agent.Id; - - var routing = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(); - routing.Context.SetDialogs(dialogs); - - await completer.Connect(conn, - onModelReady: async () => - { - // Control initial session - await completer.UpdateInitialSession(conn); - - - // Add dialog history - foreach (var item in dialogs) - { - await completer.InsertConversationItem(item); - } - - if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) - { - // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); - } - else - { - await completer.TriggerModelInference("Reply based on the conversation context."); - } - }, - onModelAudioDeltaReceived: async audioDeltaData => - { - var data = conn.OnModelMessageReceived(audioDeltaData); - await SendEventToUser(userWebSocket, data); - }, - onModelAudioResponseDone: async () => - { - var data = conn.OnModelAudioResponseDone(); - await SendEventToUser(userWebSocket, data); - }, - onAudioTranscriptDone: async transcript => - { - - }, - onModelResponseDone: async messages => - { - foreach (var message in messages) - { - // Invoke function - if (message.MessageType == "function_call") - { - await routing.InvokeFunction(message.FunctionName, message); - message.Role = AgentRole.Function; - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); - } - else - { - // append transcript to conversation - storage.Append(conn.ConversationId, message); - dialogs.Add(message); - - foreach (var hook in hookProvider.HooksOrderByPriority) - { - hook.SetAgent(agent) - .SetConversation(conversation); - - if (!string.IsNullOrEmpty(message.Content)) - { - await hook.OnMessageReceived(message); - } - } - } - } - }, - onConversationItemCreated: async response => - { - - }, - onInputAudioTranscriptionCompleted: async message => - { - // append transcript to conversation - storage.Append(conn.ConversationId, message); - dialogs.Add(message); - }, - onUserInterrupted: async () => - { - var data = conn.OnModelUserInterrupted(); - await SendEventToUser(userWebSocket, data); - }); - } - - private async Task SendEventToUser(WebSocket webSocket, object message) - { - var data = JsonSerializer.Serialize(message); - var buffer = Encoding.UTF8.GetBytes(data); - await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); - } -} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs index 6d47c4cc..016e0c36 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -129,8 +129,8 @@ public partial class FileRepository Name = task.Name, Description = task.Description, Enabled = task.Enabled, - CreatedDateTime = DateTime.UtcNow, - UpdatedDateTime = DateTime.UtcNow + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow }; var fileContent = BuildAgentTaskFileContent(metaData, task.Content); @@ -163,8 +163,8 @@ public partial class FileRepository Name = parsedTask.Name, Description = parsedTask.Description, Enabled = parsedTask.Enabled, - CreatedDateTime = parsedTask.CreatedDateTime, - UpdatedDateTime = DateTime.UtcNow + CreatedTime = parsedTask.CreatedTime, + UpdatedTime = DateTime.UtcNow }; var content = parsedTask.Content; diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index ec4848d7..647282a4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users.Models; using System.IO; namespace BotSharp.Core.Repository; @@ -33,13 +34,19 @@ public partial class FileRepository var stateFile = Path.Combine(dir, STATE_FILE); if (!File.Exists(stateFile)) { - File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); + File.WriteAllText(stateFile, "[]"); + } + + var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE); + if (!File.Exists(latestStateFile)) + { + File.WriteAllText(latestStateFile, "{}"); } var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); if (!File.Exists(breakpointFile)) { - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); + File.WriteAllText(breakpointFile, "[]"); } } @@ -151,7 +158,7 @@ public partial class FileRepository } } - public bool UpdateConversationTags(string conversationId, List tags) + public bool UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) { if (string.IsNullOrEmpty(conversationId)) return false; @@ -163,7 +170,11 @@ public partial class FileRepository var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); - conv.Tags = tags ?? new(); + + var tags = conv.Tags ?? []; + tags = tags.Concat(toAddTags).Distinct().ToList(); + conv.Tags = tags.Where(x => !toDeleteTags.Contains(x, StringComparer.OrdinalIgnoreCase)).ToList(); + conv.UpdatedTime = DateTime.UtcNow; File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); return true; @@ -300,14 +311,21 @@ public partial class FileRepository if (states.IsNullOrEmpty()) return; var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (string.IsNullOrEmpty(convDir)) return; + + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) { - var stateFile = Path.Combine(convDir, STATE_FILE); - if (File.Exists(stateFile)) - { - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateFile, stateStr); - } + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); + } + + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) + { + var latestStates = BuildLatestStates(states); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateFile, stateStr); } } @@ -427,25 +445,63 @@ public partial class FileRepository } // Check states - if (filter != null && !filter.States.IsNullOrEmpty()) + if (matched && filter != null && !filter.States.IsNullOrEmpty()) { - var stateFile = Path.Combine(d, STATE_FILE); - var convStates = CollectConversationStates(stateFile); - foreach (var pair in filter.States) + var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE); + var convStates = CollectConversationLatestStates(latestStateFile); + + if (convStates.IsNullOrEmpty()) { - if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - - var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); - if (foundState == null) + matched = false; + } + else + { + foreach (var pair in filter.States) { - matched = false; - break; - } + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - if (!string.IsNullOrWhiteSpace(pair.Value)) - { - var curValue = foundState.Values.LastOrDefault()?.Data; - matched = matched && pair.Value.IsEqualTo(curValue); + var components = pair.Key.Split(".").ToList(); + var primaryKey = components[0]; + if (convStates.TryGetValue(primaryKey, out var doc)) + { + var elem = doc.RootElement.GetProperty("data"); + if (components.Count < 2) + { + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + if (elem.ValueKind == JsonValueKind.Null) + { + matched = false; + } + else if (elem.ValueKind == JsonValueKind.Array) + { + matched = elem.EnumerateArray().Where(x => x.ValueKind != JsonValueKind.Null) + .Select(x => x.ToString()) + .Any(x => x == pair.Value); + } + else if (elem.ValueKind == JsonValueKind.String) + { + matched = elem.GetString() == pair.Value; + } + else + { + matched = elem.GetRawText() == pair.Value; + } + } + } + else + { + var paths = components.Where((_, idx) => idx > 0); + var found = FindState(elem, paths, pair.Value); + matched = found != null; + } + } + else + { + matched = false; + } + + if (!matched) break; } } } @@ -500,16 +556,6 @@ public partial class FileRepository batchSize = batchLimit; } - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - foreach (var d in Directory.GetDirectories(dir)) { var convFile = Path.Combine(d, CONVERSATION_FILE); @@ -585,8 +631,9 @@ public partial class FileRepository // Handle truncated states var refTime = dialogs.ElementAt(foundIdx).MetaData.CreatedTime; var stateDir = Path.Combine(convDir, STATE_FILE); + var latestStateDir = Path.Combine(convDir, CONV_LATEST_STATE_FILE); var states = CollectConversationStates(stateDir); - isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); + isSaved = HandleTruncatedStates(stateDir, latestStateDir, states, messageId, refTime); // Handle truncated breakpoints var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); @@ -605,7 +652,7 @@ public partial class FileRepository #if !DEBUG [SharpCache(10)] #endif - public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) + public List GetConversationStateSearchKeys(ConversationStateKeysFilter filter) { var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); if (!Directory.Exists(dir)) return []; @@ -616,26 +663,29 @@ public partial class FileRepository foreach (var d in Directory.GetDirectories(dir)) { var convFile = Path.Combine(d, CONVERSATION_FILE); - var stateFile = Path.Combine(d, STATE_FILE); - if (!File.Exists(convFile) || !File.Exists(stateFile)) + var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE); + if (!File.Exists(convFile) || !File.Exists(latestStateFile)) { continue; } var convJson = File.ReadAllText(convFile); - var stateJson = File.ReadAllText(stateFile); + var stateJson = File.ReadAllText(latestStateFile); var conv = JsonSerializer.Deserialize(convJson, _options); - var states = JsonSerializer.Deserialize>(stateJson, _options); - if (conv == null || conv.DialogCount < messageLowerLimit) + var states = JsonSerializer.Deserialize>(stateJson, _options); + if (conv == null + || states.IsNullOrEmpty() + || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(conv.AgentId)) + || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId))) { continue; } - var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; + var stateKeys = states?.Select(x => x.Key)?.ToList() ?? []; keys.AddRange(stateKeys); count++; - if (count >= convUpperLimit) + if (count >= filter.ConvLimit) { break; } @@ -645,6 +695,56 @@ public partial class FileRepository } + + public List GetConversationsToMigrate(int batchSize = 100) + { + var baseDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(baseDir)) return []; + + var convIds = new List(); + var dirs = Directory.GetDirectories(baseDir); + + foreach (var dir in dirs) + { + var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) continue; + + var convId = dir.Split(Path.DirectorySeparatorChar).Last(); + if (string.IsNullOrEmpty(convId)) continue; + + convIds.Add(convId); + if (convIds.Count >= batchSize) + { + break; + } + } + + return convIds; + } + + + public bool MigrateConvsersationLatestStates(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) + { + return false; + } + + var stateFile = Path.Combine(convDir, STATE_FILE); + var states = CollectConversationStates(stateFile); + var latestStates = BuildLatestStates(states); + + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateFile, stateStr); + + return true; + } + + #region Private methods private string? FindConversationDirectory(string conversationId) { @@ -713,7 +813,7 @@ public partial class FileRepository return isSaved; } - private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) + private bool HandleTruncatedStates(string stateDir, string latestStateDir, List states, string refMsgId, DateTime refTime) { var truncatedStates = new List(); foreach (var state in states) @@ -734,6 +834,10 @@ public partial class FileRepository } var isSaved = SaveTruncatedStates(stateDir, truncatedStates); + if (isSaved) + { + SaveTruncatedLatestStates(latestStateDir, truncatedStates); + } return isSaved; } @@ -804,6 +908,17 @@ public partial class FileRepository return true; } + private bool SaveTruncatedLatestStates(string latestStateDir, List states) + { + if (string.IsNullOrEmpty(latestStateDir) || states == null) return false; + if (!File.Exists(latestStateDir)) File.Create(latestStateDir); + + var latestStates = BuildLatestStates(states); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateDir, stateStr); + return true; + } + private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) { if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; @@ -814,22 +929,113 @@ public partial class FileRepository return true; } - private string? EncodeText(string? text) + private Dictionary CollectConversationLatestStates(string latestStateDir) { - if (string.IsNullOrEmpty(text)) return text; + if (string.IsNullOrEmpty(latestStateDir) || !File.Exists(latestStateDir)) return []; - var bytes = Encoding.UTF8.GetBytes(text); - var encoded = Convert.ToBase64String(bytes); - return encoded; + var str = File.ReadAllText(latestStateDir); + var states = JsonSerializer.Deserialize>(str, _options); + return states ?? []; } - private string? DecodeText(string? text) + private Dictionary BuildLatestStates(List states) { - if (string.IsNullOrEmpty(text)) return text; + var endNodes = new Dictionary(); + if (states.IsNullOrEmpty()) + { + return endNodes; + } - var decoded = Convert.FromBase64String(text); - var origin = Encoding.UTF8.GetString(decoded); - return origin; + foreach (var pair in states) + { + var value = pair.Values?.LastOrDefault(); + if (value == null || !value.Active) continue; + + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _options); + var json = JsonDocument.Parse(jsonStr); + endNodes[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = value.Data }, _options); + var json = JsonDocument.Parse(str); + endNodes[pair.Key] = json; + } + } + + return endNodes; + } + + private JsonElement? FindState(JsonElement? root, IEnumerable paths, string? targetValue) + { + var elem = root; + + if (elem == null || paths.IsNullOrEmpty()) + { + return null; + } + + for (int i = 0; i < paths.Count(); i++) + { + if (elem == null) return null; + + var field = paths.ElementAt(i); + if (elem.Value.ValueKind == JsonValueKind.Array) + { + if (!elem.Value.EnumerateArray().IsNullOrEmpty()) + { + foreach (var item in elem.Value.EnumerateArray()) + { + var subPaths = paths.Where((_, idx) => idx >= i); + elem = FindState(item, subPaths, targetValue); + if (elem != null) + { + return elem; + } + } + } + else + { + return null; + } + } + else if (elem.Value.ValueKind == JsonValueKind.Object && elem.Value.TryGetProperty(field, out var prop)) + { + elem = prop; + } + else + { + return null; + } + } + + if (elem != null && !string.IsNullOrWhiteSpace(targetValue)) + { + if (elem.Value.ValueKind == JsonValueKind.Null) + { + return null; + } + else if (elem.Value.ValueKind == JsonValueKind.Array) + { + var isInArray = elem.Value.EnumerateArray().Where(x => x.ValueKind != JsonValueKind.Null) + .Select(x => x.ToString()) + .Any(x => x == targetValue); + return isInArray ? elem : null; + } + else if ((elem.Value.ValueKind == JsonValueKind.String && elem.Value.GetString() == targetValue) + || (elem.Value.ValueKind != JsonValueKind.String && elem.Value.GetRawText() == targetValue)) + { + return elem; + } + else + { + return null; + } + } + + return elem; } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index f82cb6fa..3913a404 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Loggers.Models; +using Microsoft.IdentityModel.Logging; using System.IO; namespace BotSharp.Core.Repository @@ -54,26 +55,34 @@ namespace BotSharp.Core.Repository File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } - public List GetConversationContentLogs(string conversationId) + public DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; + if (string.IsNullOrEmpty(conversationId)) return new(); var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return logs; + if (string.IsNullOrEmpty(convDir)) return new(); var logDir = Path.Combine(convDir, "content_log"); - if (!Directory.Exists(logDir)) return logs; + if (!Directory.Exists(logDir)) return new(); + var logs = new List(); foreach (var file in Directory.GetFiles(logDir)) { var text = File.ReadAllText(file); var log = JsonSerializer.Deserialize(text); - if (log == null) continue; + if (log == null || log.CreatedTime >= filter.StartTime) continue; logs.Add(log); } - return logs.OrderBy(x => x.CreatedTime).ToList(); + + logs = logs.OrderByDescending(x => x.CreatedTime).Take(filter.Size).ToList(); + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion @@ -99,26 +108,222 @@ namespace BotSharp.Core.Repository File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } - public List GetConversationStateLogs(string conversationId) + public DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; + if (string.IsNullOrEmpty(conversationId)) return new(); var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return logs; + if (string.IsNullOrEmpty(convDir)) return new(); var logDir = Path.Combine(convDir, "state_log"); - if (!Directory.Exists(logDir)) return logs; + if (!Directory.Exists(logDir)) return new(); + var logs = new List(); foreach (var file in Directory.GetFiles(logDir)) { var text = File.ReadAllText(file); var log = JsonSerializer.Deserialize(text); - if (log == null) continue; + if (log == null || log.CreatedTime >= filter.StartTime) continue; logs.Add(log); } - return logs.OrderBy(x => x.CreatedTime).ToList(); + + logs = logs.OrderByDescending(x => x.CreatedTime).Take(filter.Size).ToList(); + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; + } + #endregion + + #region Instruction Log + public bool SaveInstructionLogs(IEnumerable logs) + { + if (logs.IsNullOrEmpty()) return false; + + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + Directory.CreateDirectory(baseDir); + } + + foreach (var log in logs) + { + var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.json"); + log.InnerStates = BuildLogStates(log.States); + var text = JsonSerializer.Serialize(log, _options); + File.WriteAllText(file, text); + } + return true; + } + + public PagedItems GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + return new(); + } + + var logs = new List(); + var files = Directory.GetFiles(baseDir); + foreach (var file in files) + { + var json = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(json, _options); + if (log == null) continue; + + var matched = true; + if (!filter.AgentIds.IsNullOrEmpty()) + { + matched = matched && filter.AgentIds.Contains(log.AgentId); + } + if (!filter.Providers.IsNullOrEmpty()) + { + matched = matched && filter.Providers.Contains(log.Provider); + } + if (!filter.Models.IsNullOrEmpty()) + { + matched = matched && filter.Models.Contains(log.Model); + } + if (!filter.TemplateNames.IsNullOrEmpty()) + { + matched = matched && filter.TemplateNames.Contains(log.TemplateName); + } + if (!filter.UserIds.IsNullOrEmpty()) + { + matched = matched && filter.UserIds.Contains(log.UserId); + } + + // Check states + if (matched && filter != null && !filter.States.IsNullOrEmpty()) + { + var logStates = log.InnerStates; + if (logStates.IsNullOrEmpty()) + { + matched = false; + } + else + { + foreach (var pair in filter.States) + { + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; + + var components = pair.Key.Split(".").ToList(); + var primaryKey = components[0]; + if (logStates.TryGetValue(primaryKey, out var doc)) + { + var elem = doc.RootElement.GetProperty("data"); + if (components.Count < 2) + { + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + if (elem.ValueKind == JsonValueKind.Null) + { + matched = false; + } + else if (elem.ValueKind == JsonValueKind.Array) + { + matched = elem.EnumerateArray().Where(x => x.ValueKind != JsonValueKind.Null) + .Select(x => x.ToString()) + .Any(x => x == pair.Value); + } + else if (elem.ValueKind == JsonValueKind.String) + { + matched = elem.GetString() == pair.Value; + } + else + { + matched = elem.GetRawText() == pair.Value; + } + } + } + else + { + var paths = components.Where((_, idx) => idx > 0); + var found = FindState(elem, paths, pair.Value); + matched = found != null; + } + } + else + { + matched = false; + } + + if (!matched) break; + } + } + } + + if (!matched) continue; + + log.Id = Path.GetFileNameWithoutExtension(file); + logs.Add(log); + } + + var records = logs.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size); + records = records.Select(x => + { + var states = x.InnerStates.ToDictionary(p => p.Key, p => + { + var data = p.Value.RootElement.GetProperty("data"); + return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; + }); + x.States = states ?? []; + return x; + }).ToList(); + + return new PagedItems + { + Items = records, + Count = logs.Count() + }; + } + + public List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + { + var keys = new List(); + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + return keys; + } + + var count = 0; + var files = Directory.GetFiles(baseDir); + foreach (var file in files) + { + var json = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(json, _options); + if (log == null) continue; + + if (log == null + || log.InnerStates.IsNullOrEmpty() + || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(log.UserId)) + || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId))) + { + continue; + } + + var stateKeys = log.InnerStates.Select(x => x.Key)?.ToList() ?? []; + keys.AddRange(stateKeys); + count++; + + if (count >= filter.LogLimit) + { + break; + } + } + + return keys.Distinct().ToList(); } #endregion @@ -141,6 +346,28 @@ namespace BotSharp.Core.Repository return logIndexes.IsNullOrEmpty() ? 0 : logIndexes.Max() + 1; } + + private Dictionary BuildLogStates(Dictionary states) + { + var dic = new Dictionary(); + foreach (var pair in states) + { + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _options); + var json = JsonDocument.Parse(jsonStr); + dic[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = pair.Value }, _options); + var json = JsonDocument.Parse(str); + dic[pair.Key] = json; + } + } + + return dic; + } #endregion } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 78e90e4b..f26e8bc0 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -6,7 +6,6 @@ using BotSharp.Abstraction.Users.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; - namespace BotSharp.Core.Repository; public partial class FileRepository : IBotSharpRepository @@ -34,6 +33,7 @@ public partial class FileRepository : IBotSharpRepository private const string DIALOG_FILE = "dialogs.json"; private const string STATE_FILE = "state.json"; private const string BREAKPOINT_FILE = "breakpoint.json"; + private const string CONV_LATEST_STATE_FILE = "latest-state.json"; private const string TRANSLATION_MEMORY_FILE = "memory.json"; private const string USERS_FOLDER = "users"; @@ -54,6 +54,7 @@ public partial class FileRepository : IBotSharpRepository private const string STATS_FILE = "stats.json"; private const string CRON_FILE = "cron.json"; + private const string INSTRUCTION_LOG_FOLDER = "instruction-logs"; public FileRepository( IServiceProvider services, diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs index 592a2c29..58e43819 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs @@ -5,8 +5,9 @@ namespace BotSharp.Core.Routing.Functions; public class FallbackToRouterFn : IFunctionCallback { - public string Name => "fallback_to_router"; + public string Name => "util-routing-fallback_to_router"; private readonly IServiceProvider _services; + public FallbackToRouterFn(IServiceProvider services) { _services = services; @@ -14,30 +15,10 @@ public class FallbackToRouterFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - var agentService = _services.GetRequiredService(); - var agents = await agentService.GetAgents(new AgentFilter - { - AgentNames = [args.AgentName] - }); - var targetAgent = agents.Items.FirstOrDefault(); - if (targetAgent == null) - { - message.Content = $"Can't find routing agent {args.AgentName}"; - return false; - } - - var conv = _services.GetRequiredService(); - var dialogs = conv.GetDialogHistory(); - + var args = JsonSerializer.Deserialize(message.FunctionArgs); var routing = _services.GetRequiredService(); - routing.Context.Replace(targetAgent.Id); - message.CurrentAgentId = targetAgent.Id; - - var response = await routing.InstructLoop(message, dialogs); - - message.Content = response.Content; - message.StopCompletion = true; + routing.Context.PopTo(routing.Context.EntryAgentId, "pop to entry agent"); + message.Content = args.Question; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs new file mode 100644 index 00000000..e6fedc05 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Core.Routing.Hooks; + +public class RoutingUtilityHook : IAgentUtilityHook +{ + private static string PREFIX = "util-routing-"; + private static string REDIRECT_TO_AGENT = $"{PREFIX}redirect_to_agent"; + private static string FALLBACK_TO_ROUTER = $"{PREFIX}fallback_to_router"; + + public void AddUtilities(List utilities) + { + var utility = new AgentUtility + { + Name = "routing.tools", + Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")], + Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")] + }; + + utilities.Add(utility); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs index 931b53e8..8980246e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs @@ -73,7 +73,7 @@ public class NaiveReasoner : IRoutingReasoner }; var response = await completion.GetChatCompletions(router, dialogs); - inst = response.Content.JsonContent(); + inst = (response.FunctionArgs ?? response.Content).JsonContent(); break; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index cf604c55..47e47631 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Routing.Settings; namespace BotSharp.Core.Routing; @@ -79,7 +80,7 @@ public class RoutingContext : IRoutingContext /// /// Id or Name /// - public void Push(string agentId, string? reason = null) + public void Push(string agentId, string? reason = null, bool updateLazyRouting = true) { // Convert id to name if (!Guid.TryParse(agentId, out _)) @@ -99,13 +100,15 @@ public class RoutingContext : IRoutingContext HookEmitter.Emit(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason) ).Wait(); + + UpdateLazyRoutingAgent(updateLazyRouting); } } /// /// Pop current agent /// - public void Pop(string? reason = null) + public void Pop(string? reason = null, bool updateLazyRouting = true) { if (_stack.Count == 0) { @@ -149,15 +152,17 @@ public class RoutingContext : IRoutingContext _stack.Push(agentId); } } + + UpdateLazyRoutingAgent(updateLazyRouting); } - public void PopTo(string agentId, string reason) + public void PopTo(string agentId, string reason, bool updateLazyRouting = true) { var currentAgentId = GetCurrentAgentId(); while (!string.IsNullOrEmpty(currentAgentId) && currentAgentId != agentId) { - Pop(reason); + Pop(reason, updateLazyRouting: updateLazyRouting); currentAgentId = GetCurrentAgentId(); } } @@ -181,7 +186,7 @@ public class RoutingContext : IRoutingContext return _stack.ToArray().Contains(agentId); } - public void Replace(string agentId, string? reason = null) + public void Replace(string agentId, string? reason = null, bool updateLazyRouting = true) { var fromAgent = agentId; var toAgent = agentId; @@ -200,6 +205,8 @@ public class RoutingContext : IRoutingContext await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason) ).Wait(); } + + UpdateLazyRoutingAgent(updateLazyRouting); } public void Empty(string? reason = null) @@ -275,4 +282,24 @@ public class RoutingContext : IRoutingContext { _dialogs = []; } + + private void UpdateLazyRoutingAgent(bool updateLazyRouting) + { + if (!updateLazyRouting) + { + return; + } + + // Set next handling agent for lazy routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + if (routingMode == "lazy") + { + var agentId = GetCurrentAgentId(); + if (agentId != BuiltInAgentId.Fallback) + { + states.SetState(StateConst.LAZY_ROUTING_AGENT_ID, agentId); + } + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs index c5317ba4..2e409fcf 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs @@ -37,5 +37,7 @@ public class RoutingPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index a4d4b37a..bec44bc4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -53,6 +53,7 @@ public partial class RoutingService // Handle output routing exception. if (agent.Type == AgentType.Routing) { + // Forgot about what situation needs to handle in this way response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; } diff --git a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs index 5ad0637e..ce5d72b8 100644 --- a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs +++ b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs @@ -31,7 +31,7 @@ public class AgentTaskService : IAgentTaskService return new PagedItems { - Items = items.OrderByDescending(x => x.UpdatedDateTime) + Items = items.OrderByDescending(x => x.UpdatedTime) .Skip(filter.Pager.Offset).Take(filter.Pager.Size), Count = items.Count() }; diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs index 3df585f2..d5e61c12 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Translation; using System; @@ -26,6 +27,11 @@ namespace BotSharp.Logger.Hooks { return; } + + if (_states.GetState("channel") == ConversationChannel.Phone) + { + return; + } // Handle multi-language for output var agentService = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 60192e5f..e4ebe311 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -418,23 +418,24 @@ public class UserService : IUserService return users; } - public async Task IsAdminUser(string userId) + [SharpCache(10, perInstanceCache: true)] + public async Task<(bool, User?)> IsAdminUser(string userId) { var db = _services.GetRequiredService(); var user = db.GetUserById(userId); - return user != null && UserConstant.AdminRoles.Contains(user.Role); + var isAdmin = user != null && UserConstant.AdminRoles.Contains(user.Role); + return (isAdmin, user); } public async Task GetUserAuthorizations(IEnumerable? agentIds = null) { var db = _services.GetRequiredService(); - var user = db.GetUserById(_user.Id); + var (isAdmin, user) = await IsAdminUser(_user.Id); var auth = new UserAuthorization(); if (user == null) return auth; - auth.IsAdmin = UserConstant.AdminRoles.Contains(user.Role); - + auth.IsAdmin = isAdmin; var role = db.GetRoles(new RoleFilter { Names = [user.Role] }).FirstOrDefault(); var permissions = user.Permissions?.Any() == true ? user.Permissions : role?.Permissions ?? []; auth.Permissions = permissions; diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 8655afb1..0afac090 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -39,6 +39,8 @@ global using BotSharp.Abstraction.SideCar.Attributes; global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Enums; global using BotSharp.Abstraction.Statistics.Services; +global using BotSharp.Abstraction.Loggers.Services; +global using BotSharp.Abstraction.Infrastructures.Events; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; @@ -46,4 +48,4 @@ global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; global using BotSharp.Core.Statistics.Services; -global using BotSharp.Abstraction.Infrastructures.Events; \ No newline at end of file +global using BotSharp.Core.Loggers.Services; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json index c54a67e0..3200865e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json @@ -1,7 +1,7 @@ { "id": "01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d", "name": "Fallback Agent", - "description": "Don't have sufficient confidence to trigger any of existing agent.", + "description": "Handle initiated conversation without specific task given yet or don't have sufficient confidence to handle user task.", "type": "task", "createdDateTime": "2024-05-07T10:00:00Z", "updatedDateTime": "2024-05-07T10:00:00Z", diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json new file mode 100644 index 00000000..08f938ec --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -0,0 +1,31 @@ +{ + "name": "route_to_agent", + "description": "Route request to appropriate AI agent.", + "visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "next_action_agent": { + "type": "string", + "description": "Agent for next action based on user latest response" + }, + "next_action_reason": { + "type": "string", + "description": "The reason why route to this agent." + }, + "user_goal_agent": { + "type": "string", + "description": "Agent who can acheive user initial task." + }, + "conversation_end": { + "type": "boolean", + "description": "User is ending the conversation." + }, + "args": { + "type": "object", + "description": "Required parameters of next action agent" + } + }, + "required": [ "next_action_agent", "user_goal_agent", "next_action_reason", "args" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index bc614f81..7ee8f56e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -5,7 +5,9 @@ Follow these steps to handle user request: 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. +{% if routing_mode != 'lazy' %} 5. Response must be in JSON format. +{% endif %} {% if routing_requirements and routing_requirements != empty %} [REQUIREMENTS] @@ -14,6 +16,7 @@ Follow these steps to handle user request: {%- endfor %} {% endif %} +{% if routing_mode != 'lazy' %} [FUNCTIONS] {% for handler in routing_handlers -%} # {{ handler.description}} @@ -26,6 +29,7 @@ Parameters: {%- endif %} {{ "\r\n" }} {%- endfor %} +{% endif %} [AGENTS] {% for agent in routing_agents -%} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json new file mode 100644 index 00000000..3902d8ae --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -0,0 +1,18 @@ +{ + "name": "util-routing-fallback_to_router", + "description": "Return to the Router to find the appropriate agent who can handle the user's request.", + "parameters": { + "type": "object", + "properties": { + "fallback_reason": { + "type": "string", + "description": "The reason why you need to reach out to other agent." + }, + "user_question": { + "type": "string", + "description": "User question or statement." + } + }, + "required": [ "fallback_reason" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid new file mode 100644 index 00000000..a1bd9ceb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid @@ -0,0 +1 @@ +Carefully consider whether the current user request is related to your responsibilities. Only when it is not relevant should you consider Return to the Router. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs index 6ccd70ac..0404ceb4 100644 --- a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs +++ b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs @@ -15,6 +15,7 @@ public static class BotSharpLoggerExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); return services; } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 2cc2c9fb..8ff9be86 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -25,17 +25,20 @@ public class GlobalStatsConversationHook : IContentGeneratingHook // record agent call var globalStats = _services.GetRequiredService(); + var metric = StatsMetric.AgentCall; + var dim = "agent"; + var agentId = message.CurrentAgentId ?? string.Empty; var body = new BotSharpStatsInput { - Metric = StatsMetric.AgentCall, - Dimension = "agent", - DimRefVal = message.CurrentAgentId ?? string.Empty, + Metric = metric, + Dimension = dim, + DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats("global-agent-call", body); + globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body); } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs new file mode 100644 index 00000000..30e1d066 --- /dev/null +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -0,0 +1,54 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Instructs.Settings; +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users; + +namespace BotSharp.Logger.Hooks; + +public class InstructionLogHook : InstructHookBase +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly IUserIdentity _user; + + public override string SelfId => string.Empty; + + public InstructionLogHook( + IServiceProvider services, + ILogger logger, + IUserIdentity user) + { + _services = services; + _logger = logger; + _user = user; + } + + public override async Task OnResponseGenerated(InstructResponseModel response) + { + var settings = _services.GetRequiredService(); + if (!settings.EnableLog || response == null) return; + + var db = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + var user = db.GetUserById(_user.Id); + var templateName = response.TemplateName ?? state.GetState("instruct_template_name") ?? null; + + db.SaveInstructionLogs(new List + { + new InstructionLogModel + { + AgentId = response.AgentId, + Provider = response.Provider, + Model = response.Model, + TemplateName = !string.IsNullOrWhiteSpace(templateName) ? templateName : null, + UserMessage = response.UserMessage, + SystemInstruction = response.SystemInstruction, + CompletionText = response.CompletionText, + States = state.GetStates(), + UserId = user?.Id + } + }); + return; + } +} diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index 9740bb20..7affc3a4 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -35,7 +35,7 @@ public class VerboseLogHook : IContentGeneratingHook public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) { - if (!_convSettings.ShowVerboseLog) return; + if (!_convSettings.ShowVerboseLog || string.IsNullOrEmpty(tokenStats.Prompt)) return; var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Infrastructure/BotSharp.Logger/Using.cs b/src/Infrastructure/BotSharp.Logger/Using.cs index 5115e776..9c975007 100644 --- a/src/Infrastructure/BotSharp.Logger/Using.cs +++ b/src/Infrastructure/BotSharp.Logger/Using.cs @@ -12,4 +12,5 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Instructs; global using BotSharp.Logger.Hooks; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 58157b2d..9162cd16 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -8,7 +8,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 75dc1113..f032f504 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,10 +1,8 @@ -using Azure; using BotSharp.Abstraction.Files.Constants; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Users.Enums; using BotSharp.Core.Infrastructures; namespace BotSharp.OpenAPI.Controllers; @@ -45,27 +43,33 @@ public class ConversationController : ControllerBase return ConversationViewModel.FromSession(conv); } - [HttpPost("/conversations")] - public async Task> GetConversations([FromBody] ConversationFilter filter) + [HttpGet("/conversations")] + public async Task> GetConversations([FromQuery] ConversationFilter filter) { var convService = _services.GetRequiredService(); var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); if (user == null) { return new PagedItems(); } - filter.UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : filter.UserId; + filter.UserId = !isAdmin ? user.Id : filter.UserId; var conversations = await convService.GetConversations(filter); var agentService = _services.GetRequiredService(); var list = conversations.Items.Select(x => ConversationViewModel.FromSession(x)).ToList(); + var agentIds = list.Select(x => x.AgentId).ToList(); + var agents = await agentService.GetAgentOptions(agentIds); + + var userIds = list.Select(x => x.User.Id).ToList(); + var users = await userService.GetUsers(new UserFilter { UserIds = userIds, Size = filter.Pager.Size }); + foreach (var item in list) { - user = await userService.GetUser(item.User.Id); + user = users.Items.FirstOrDefault(x => x.Id == item.User.Id); item.User = UserViewModel.FromUser(user); - var agent = await agentService.GetAgent(item.AgentId); + var agent = agents.FirstOrDefault(x => x.Id == item.AgentId); item.AgentName = agent?.Name ?? "Unkown"; } @@ -77,11 +81,11 @@ public class ConversationController : ControllerBase } [HttpGet("/conversation/{conversationId}/dialogs")] - public async Task> GetDialogs([FromRoute] string conversationId) + public async Task> GetDialogs([FromRoute] string conversationId, [FromQuery] int count = 100) { var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, [], isReadOnly: true); - var history = conv.GetDialogHistory(fromBreakpoint: false); + var history = conv.GetDialogHistory(lastCount: count, fromBreakpoint: false); var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); @@ -138,7 +142,7 @@ public class ConversationController : ControllerBase { var service = _services.GetRequiredService(); var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); if (user == null) { return null; @@ -147,7 +151,7 @@ public class ConversationController : ControllerBase var filter = new ConversationFilter { Id = conversationId, - UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : null + UserId = !isAdmin ? user.Id : null }; var conversations = await service.GetConversations(filter); if (conversations.Items.IsNullOrEmpty()) @@ -206,11 +210,11 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); var filter = new ConversationFilter { Id = conversationId, - UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : null + UserId = !isAdmin ? user?.Id : null }; var conversations = await conv.GetConversations(filter); @@ -229,11 +233,11 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var conversationService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); var filter = new ConversationFilter { Id = conversationId, - UserId = user.Role != UserRole.Admin ? user.Id : null + UserId = !isAdmin ? user?.Id : null }; var conversations = await conversationService.GetConversations(filter); @@ -251,7 +255,7 @@ public class ConversationController : ControllerBase public async Task UpdateConversationTags([FromRoute] string conversationId, [FromBody] UpdateConversationRequest request) { var conv = _services.GetRequiredService(); - return await conv.UpdateConversationTags(conversationId, request.Tags); + return await conv.UpdateConversationTags(conversationId, request.ToAddTags, request.ToDeleteTags); } [HttpPut("/conversation/{conversationId}/update-message")] @@ -283,11 +287,11 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var conversationService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); var filter = new ConversationFilter { Id = conversationId, - UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : null + UserId = !isAdmin ? user?.Id : null }; var conversations = await conversationService.GetConversations(filter); @@ -555,14 +559,24 @@ public class ConversationController : ControllerBase #region Search state keys [HttpGet("/conversation/state/keys")] - public async Task> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] int convLimit = 100, [FromQuery] bool preload = false) + public async Task> GetConversationStateKeys([FromQuery] ConversationStateKeysFilter request) { var convService = _services.GetRequiredService(); - var keys = await convService.GetConversationStateSearhKeys(query, keyLimit: keyLimit, convLimit: convLimit, preload: preload); + var keys = await convService.GetConversationStateSearhKeys(request); return keys; } #endregion + #region Migrate Latest States + [HttpPost("/conversation/latest-state/migrate")] + public async Task MigrateConversationLatestStates([FromBody] MigrateLatestStateRequest request) + { + var convService = _services.GetRequiredService(); + var res = await convService.MigrateLatestStates(request.BatchSize, request.ErrorLimit); + return res; + } + #endregion + #region Private methods private void SetStates(IConversationService conv, NewMessageModel input) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs index c95d819d..93cfefde 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs @@ -19,16 +19,16 @@ public class DashboardController : ControllerBase } #region User Components [HttpGet("/dashboard/components")] - public async Task GetComponents() + public async Task GetComponents() { var userService = _services.GetRequiredService(); var dashboardProfile = await userService.GetDashboard(); - if (dashboardProfile == null) return new UserDashboardModel(); + if (dashboardProfile == null) return new(); - var result = new UserDashboardModel + var result = new UserDashboardViewModel { ConversationList = dashboardProfile.ConversationList.Select( - x => new UserDashboardConversationModel + x => new UserDashboardConversationViewModel { Name = x.Name, ConversationId = x.ConversationId, @@ -40,7 +40,7 @@ public class DashboardController : ControllerBase } [HttpPost("/dashboard/component/conversation")] - public async Task UpdateDashboardConversationInstruction(UserDashboardConversationModel dashConv) + public async Task UpdateDashboardConversationInstruction(UserDashboardConversationViewModel dashConv) { if (string.IsNullOrEmpty(dashConv.Name) && string.IsNullOrEmpty(dashConv.Instruction)) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index dbce9598..6a1c3705 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -52,8 +52,29 @@ public class InstructModeController : ControllerBase .SetState("model", input.Model, source: StateSource.External) .SetState("model_id", input.ModelId, source: StateSource.External); + var agentId = input.AgentId ?? Guid.Empty.ToString(); var textCompletion = CompletionProvider.GetTextCompletion(_services); - return await textCompletion.GetCompletion(input.Text, input.AgentId ?? Guid.Empty.ToString(), Guid.NewGuid().ToString()); + var response = await textCompletion.GetCompletion(input.Text, agentId, Guid.NewGuid().ToString()); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = agentId, + Provider = textCompletion.Provider, + Model = textCompletion.Model, + TemplateName = input.Template, + UserMessage = input.Text, + CompletionText = response + }); + } + return response; } #region Chat @@ -66,15 +87,36 @@ public class InstructModeController : ControllerBase .SetState("model", input.Model, source: StateSource.External) .SetState("model_id", input.ModelId, source: StateSource.External); + var agentId = input.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services); var message = await completion.GetChatCompletions(new Agent() { - Id = input.AgentId ?? Guid.Empty.ToString(), + Id = agentId, Instruction = input.Instruction }, new List { new RoleDialogModel(AgentRole.User, input.Text) }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = agentId, + Provider = completion.Provider, + Model = completion.Model, + TemplateName = input.Template, + UserMessage = input.Text, + SystemInstruction = message.RenderedInstruction, + CompletionText = message.Content + }); + } return message.Content; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 50b7c528..f5c67070 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -12,7 +12,9 @@ public class KnowledgeBaseController : ControllerBase private readonly IKnowledgeService _knowledgeService; private readonly IServiceProvider _services; - public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvider services) + public KnowledgeBaseController( + IKnowledgeService knowledgeService, + IServiceProvider services) { _knowledgeService = knowledgeService; _services = services; @@ -117,6 +119,46 @@ public class KnowledgeBaseController : ControllerBase #endregion + #region Snapshot + [HttpGet("/knowledge/vector/{collection}/snapshots")] + public async Task> GetVectorCollectionSnapshots([FromRoute] string collection) + { + var snapshots = await _knowledgeService.GetVectorCollectionSnapshots(collection); + return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x)); + } + + [HttpPost("/knowledge/vector/{collection}/snapshot")] + public async Task CreateVectorCollectionSnapshot([FromRoute] string collection) + { + var snapshot = await _knowledgeService.CreateVectorCollectionSnapshot(collection); + return VectorCollectionSnapshotViewModel.From(snapshot); + } + + [HttpGet("/knowledge/vector/{collection}/snapshot")] + public async Task GetVectorCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName) + { + var snapshot = await _knowledgeService.DownloadVectorCollectionSnapshot(collection, snapshotFileName); + return BuildFileResult(snapshotFileName, snapshot); + } + + [HttpPost("/knowledge/vector/{collection}/snapshot/recover")] + public async Task RecoverVectorCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile) + { + var fileName = snapshotFile.FileName; + var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile); + var done = await _knowledgeService.RecoverVectorCollectionFromSnapshot(collection, fileName, binary); + return done; + } + + [HttpDelete("/knowledge/vector/{collection}/snapshot")] + public async Task DeleteVectorCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteVectorCollectionSnapshotRequest request) + { + var done = await _knowledgeService.DeleteVectorCollectionSnapshot(collection, request.SnapshotName); + return done; + } + #endregion + + #region Document [HttpPost("/knowledge/document/{collection}/upload")] public async Task UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) @@ -187,7 +229,6 @@ public class KnowledgeBaseController : ControllerBase #endregion - #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) @@ -214,4 +255,13 @@ public class KnowledgeBaseController : ControllerBase return saved ? "Success" : "Fail"; } #endregion + + #region Private methods + private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) + { + var stream = fileData.ToStream(); + stream.Position = 0; + return File(stream, "application/octet-stream", Path.GetFileName(fileName)); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index de07a482..7032bcd3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Loggers.Services; +using BotSharp.OpenAPI.ViewModels.Instructs; using Microsoft.AspNetCore.Hosting; namespace BotSharp.OpenAPI.Controllers; @@ -9,7 +11,8 @@ public class LoggerController : ControllerBase { private readonly IServiceProvider _services; - public LoggerController(IServiceProvider services) + public LoggerController( + IServiceProvider services) { _services = services; } @@ -32,17 +35,46 @@ public class LoggerController : ControllerBase } } + #region Conversation log [HttpGet("/logger/conversation/{conversationId}/content-log")] - public async Task> GetConversationContentLogs([FromRoute] string conversationId) + public async Task> GetConversationContentLogs( + [FromRoute] string conversationId, + [FromQuery] ConversationLogFilter request) { - var conversationService = _services.GetRequiredService(); - return await conversationService.GetConversationContentLogs(conversationId); + var logging = _services.GetRequiredService(); + return await logging.GetConversationContentLogs(conversationId, request); } [HttpGet("/logger/conversation/{conversationId}/state-log")] - public async Task> GetConversationStateLogs([FromRoute] string conversationId) + public async Task> GetConversationStateLogs( + [FromRoute] string conversationId, + [FromQuery] ConversationLogFilter request) { - var conversationService = _services.GetRequiredService(); - return await conversationService.GetConversationStateLogs(conversationId); + var logging = _services.GetRequiredService(); + return await logging.GetConversationStateLogs(conversationId, request); } + #endregion + + #region Instruction log + [HttpGet("/logger/instruction/log")] + public async Task> GetInstructionLogs([FromQuery] InstructLogFilter request) + { + var logging = _services.GetRequiredService(); + var logs = await logging.GetInstructionLogs(request); + + return new PagedItems + { + Items = logs.Items.Select(x => InstructionLogViewModel.From(x)), + Count = logs.Count + }; + } + + [HttpGet("/logger/instruction/log/keys")] + public async Task> GetInstructionLogKeys([FromQuery] InstructLogKeysFilter request) + { + var logging = _services.GetRequiredService(); + var keys = await logging.GetInstructionLogSearchKeys(request); + return keys; + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs index 55a5f1a8..4f6d3f2b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs @@ -89,6 +89,7 @@ public class PluginController(IServiceProvider services, IUserIdentity user, Plu private async Task IsValidUser() { var userService = services.GetRequiredService(); - return await userService.IsAdminUser(_user.Id); + var (isAdmin, _) = await userService.IsAdminUser(_user.Id); + return isAdmin; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs index d4ca64fc..ee3ac863 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Roles; -using BotSharp.Abstraction.Users.Enums; namespace BotSharp.OpenAPI.Controllers; @@ -83,6 +82,7 @@ public class RoleController : ControllerBase private async Task IsValidUser() { var userService = _services.GetRequiredService(); - return await userService.IsAdminUser(_user.Id); + var (isAdmin, _) = await userService.IsAdminUser(_user.Id); + return isAdmin; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a853e1eb..7cb1aa2b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -254,7 +254,8 @@ public class UserController : ControllerBase private async Task IsValidUser() { var userService = _services.GetRequiredService(); - return await userService.IsAdminUser(_user.Id); + var (isAdmin, _) = await userService.IsAdminUser(_user.Id); + return isAdmin; } private FileContentResult BuildFileResult(string file) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskCreateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskCreateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTemplatePatchModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTemplatePatchModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTemplatePatchModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTemplatePatchModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/RoutingRuleUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/RoutingRuleUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/RoutingRuleUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/RoutingRuleUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentTaskViewModel.cs similarity index 77% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentTaskViewModel.cs index cf051544..d46d134e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentTaskViewModel.cs @@ -13,11 +13,11 @@ public class AgentTaskViewModel public string Status { get; set; } = null!; - [JsonPropertyName("created_datetime")] - public DateTime CreatedDateTime { get; set; } + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } - [JsonPropertyName("updated_datetime")] - public DateTime UpdatedDateTime { get; set; } + [JsonPropertyName("updated_time")] + public DateTime UpdatedTime { get; set; } [JsonPropertyName("agent_id")] public string AgentId { get; set; } = null!; @@ -37,8 +37,8 @@ public class AgentTaskViewModel AgentId = task.AgentId, AgentName = task.Agent?.Name, Status = task.Status, - CreatedDateTime = task.CreatedDateTime, - UpdatedDateTime = task.UpdatedDateTime + CreatedTime = task.CreatedTime, + UpdatedTime = task.UpdatedTime }; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationCreationModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationCreationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationCreationModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationSummaryModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationSummaryModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationSummaryModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationSummaryModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/InputMessageFiles.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/InputMessageFiles.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/MigrateLatestStateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/MigrateLatestStateRequest.cs new file mode 100644 index 00000000..99085b0a --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/MigrateLatestStateRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class MigrateLatestStateRequest +{ + public int BatchSize { get; set; } = 1000; + public int ErrorLimit { get; set; } = 10; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/NewMessageModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/NewMessageModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs new file mode 100644 index 00000000..c177221c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class UpdateConversationRequest +{ + public List ToAddTags { get; set; } = []; + public List ToDeleteTags { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleAliasModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleAliasModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleAliasModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleAliasModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateMessageModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateMessageModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationRequest.cs deleted file mode 100644 index c9b89747..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationRequest.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.OpenAPI.ViewModels.Conversations; - -public class UpdateConversationRequest -{ - public List Tags { get; set; } = []; -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/EmbeddingInputModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/Request/EmbeddingInputModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/EmbeddingInputModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/Request/EmbeddingInputModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/View/MessageFileViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/View/MessageFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs similarity index 92% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs index 3b0303fd..1ad3aead 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs @@ -15,4 +15,5 @@ public class IncomingInstructRequest : IncomingMessageModel { public string? AgentId { get; set; } public string? Instruction { get; set; } + public string? Template { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/ImageGenerationViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/ImageGenerationViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructBaseViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructBaseViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructionLogViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructionLogViewModel.cs new file mode 100644 index 00000000..aca73f76 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructionLogViewModel.cs @@ -0,0 +1,72 @@ +using BotSharp.Abstraction.Loggers.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructionLogViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } = default!; + + [JsonPropertyName("agent_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentId { get; set; } + + [JsonPropertyName("agent_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentName { get; set; } + + [JsonPropertyName("provider")] + public string Provider { get; set; } = default!; + + [JsonPropertyName("model")] + public string Model { get; set; } = default!; + + [JsonPropertyName("template_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TemplateName { get; set; } + + [JsonPropertyName("user_message")] + public string UserMessage { get; set; } = string.Empty; + + [JsonPropertyName("system_instruction")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SystemInstruction { get; set; } + + [JsonPropertyName("completion_text")] + public string CompletionText { get; set; } = string.Empty; + + [JsonPropertyName("user_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserId { get; set; } + + [JsonPropertyName("user_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserName { get; set; } + + [JsonPropertyName("states")] + public Dictionary States { get; set; } = []; + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } = DateTime.UtcNow; + + public static InstructionLogViewModel From(InstructionLogModel log) + { + return new InstructionLogViewModel + { + Id = log.Id, + AgentId = log.AgentId, + AgentName = log.AgentName, + Provider = log.Provider, + Model = log.Model, + TemplateName = log.TemplateName, + UserMessage = log.UserMessage, + SystemInstruction = log.SystemInstruction, + CompletionText = log.CompletionText, + UserId = log.UserId, + UserName = log.UserName, + States = log.States, + CreatedTime = log.CreatedTime + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/MultiModalViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/MultiModalViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/PdfCompletionViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/PdfCompletionViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/SpeechToTextViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/SpeechToTextViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs new file mode 100644 index 00000000..30398b2c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteVectorCollectionSnapshotRequest +{ + [JsonPropertyName("snapshot_name")] + public string SnapshotName { get; set; } = default!; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/GraphKnowledgeViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/GraphKnowledgeViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeFileViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorCollectionConfigViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorCollectionConfigViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs new file mode 100644 index 00000000..95b8a325 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs @@ -0,0 +1,35 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorCollectionSnapshotViewModel +{ + [JsonPropertyName("name")] + public string Name { get; set; } = default!; + + [JsonPropertyName("size")] + public long Size { get; set; } + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } + + [JsonPropertyName("check_sum")] + public string? CheckSum { get; set; } + + public static VectorCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model) + { + if (model == null) + { + return null; + } + + return new VectorCollectionSnapshotViewModel + { + Name = model.Name, + Size = model.Size, + CreatedTime = model.CreatedTime, + CheckSum = model.CheckSum + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleAgentActionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleAgentActionViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleAgentActionViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleAgentActionViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/View/RoleUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/View/RoleUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserAvatarModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserAvatarModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserCreationModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserCreationModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserResetPasswordModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserResetPasswordModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAgentActionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserAgentActionViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAgentActionViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserAgentActionViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserDashboardConversationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs similarity index 57% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserDashboardConversationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs index 3a5f3491..7165fa87 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserDashboardConversationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs @@ -1,19 +1,14 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.OpenAPI.ViewModels.Users; -public class UserDashboardModel -{ +public class UserDashboardViewModel +{ [JsonPropertyName("conversation_list")] - public IList ConversationList { get; set; } = []; + public IList ConversationList { get; set; } = []; } -public class UserDashboardConversationModel +public class UserDashboardConversationViewModel { [JsonPropertyName("name")] public string? Name { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserViewModel.cs diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj index 4360a7a9..7309b457 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index b71306cf..178bc67b 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -9,10 +9,12 @@ namespace BotSharp.Plugin.AnthropicAI.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "anthropic"; + public string Model => _model; protected readonly AnthropicSettings _settings; protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private List renderedInstructions = []; protected string _model; @@ -57,6 +59,7 @@ public class ChatCompletionProvider : IChatCompletion ToolCallId = toolResult.Id, FunctionName = toolResult.Name, FunctionArgs = JsonSerializer.Serialize(toolResult.Input), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } else @@ -66,6 +69,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -98,12 +102,15 @@ public class ChatCompletionProvider : IChatCompletion private (string, MessageParameters) PrepareOptions(Agent agent, List conversations, LlmModelSetting settings) { var instruction = ""; + renderedInstructions = []; var agentService = _services.GetRequiredService(); if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { - instruction += agentService.RenderedInstruction(agent); + var text = agentService.RenderedInstruction(agent); + instruction += text; + renderedInstructions.Add(text); } /*var routing = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index e7459219..abc9749a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -9,7 +9,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 1947ebc4..00d1da58 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -15,6 +15,8 @@ public class NativeWhisperProvider : IAudioCompletion private readonly ILogger _logger; public string Provider => "native-whisper"; + public string Model => _model; + private string _model; public NativeWhisperProvider( BotSharpDatabaseSettings dbSettings, @@ -56,11 +58,13 @@ public class NativeWhisperProvider : IAudioCompletion { if (Enum.TryParse(model, true, out GgmlType ggmlType)) { + _model = model; LoadWhisperModel(ggmlType); } else { _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + _model = "Tiny"; LoadWhisperModel(GgmlType.Tiny); } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 284e9b3c..494326fa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs index 8cc75ad8..2948703e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -5,6 +5,8 @@ public partial class AudioCompletionProvider : IAudioCompletion private readonly IServiceProvider _services; public string Provider => "openai"; + public string Model => _model; + private string _model; public AudioCompletionProvider(IServiceProvider service) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 3fef7c0c..e49d91ce 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -9,10 +9,12 @@ public class ChatCompletionProvider : IChatCompletion protected readonly AzureOpenAiSettings _settings; protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private List renderedInstructions = []; protected string _model; public virtual string Provider => "azure-openai"; + public string Model => _model; public ChatCompletionProvider( AzureOpenAiSettings settings, @@ -58,7 +60,8 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -73,6 +76,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } } @@ -83,6 +87,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } catch (Exception ex) @@ -92,6 +97,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -136,7 +142,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -163,7 +170,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -199,7 +207,10 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; Console.Write(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); continue; } @@ -207,7 +218,10 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); } return true; @@ -221,6 +235,7 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; var messages = new List(); @@ -251,6 +266,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); + renderedInstructions.Add(instruction); messages.Add(new SystemChatMessage(instruction)); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 6878ca3c..b89e338c 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -13,6 +13,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected int _dimension; public virtual string Provider => "azure-openai"; + public string Model => _model; public TextEmbeddingProvider( AzureOpenAiSettings settings, @@ -49,6 +50,7 @@ public class TextEmbeddingProvider : ITextEmbedding _model = model; } + public void SetDimension(int dimension) { _dimension = dimension > 0 ? dimension : DEFAULT_DIMENSION; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 967f3e08..1b435d61 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -14,6 +14,7 @@ public partial class ImageCompletionProvider : IImageCompletion protected string _model; public virtual string Provider => "azure-openai"; + public string Model => _model; public ImageCompletionProvider( AzureOpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index 4953e114..2b6f3dfb 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -22,6 +22,7 @@ public class TextCompletionProvider : ITextCompletion }; public virtual string Provider => "azure-openai"; + public string Model => _model; public TextCompletionProvider( AzureOpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 9f2c1008..99b734e6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -7,6 +7,7 @@ public class ChatHubConversationHook : ConversationHookBase { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly BotSharpOptions _options; private readonly ChatHubSettings _settings; @@ -23,12 +24,14 @@ public class ChatHubConversationHook : ConversationHookBase public ChatHubConversationHook( IServiceProvider services, IHubContext chatHub, + ILogger logger, BotSharpOptions options, ChatHubSettings settings, IUserIdentity user) { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _options = options; _settings = settings; @@ -177,74 +180,122 @@ public class ChatHubConversationHook : ConversationHookBase private async Task InitClientConversation(string conversationId, ConversationViewModel conversation) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + _logger.LogWarning($"Failed to init client conversation in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task ReceiveClientMessage(string conversationId, ChatResponseModel model) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task ReceiveAssistantMessage(string conversationId, string? json) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } - + } private async Task GenerateSenderAction(string conversationId, ConversationSenderActionModel action) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + _logger.LogWarning($"Failed to generate sender action in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task DeleteMessage(string conversationId, ChatResponseModel model) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(DELETE_MESSAGE, model); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(DELETE_MESSAGE, model); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(DELETE_MESSAGE, model); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(DELETE_MESSAGE, model); + _logger.LogWarning($"Failed to delete message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task GenerateNotification(string conversationId, string? json) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_NOTIFICATION, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_NOTIFICATION, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_NOTIFICATION, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_NOTIFICATION, json); + _logger.LogWarning($"Failed to generate notification in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } #endregion diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs index a00dae90..cad0189d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs @@ -8,8 +8,8 @@ public class ChatHubCrontabHook : ICrontabHook { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; - private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; private readonly ChatHubSettings _settings; @@ -19,15 +19,15 @@ public class ChatHubCrontabHook : ICrontabHook public ChatHubCrontabHook(IServiceProvider services, IHubContext chatHub, + ILogger logger, IUserIdentity user, - IConversationStorage storage, BotSharpOptions options, ChatHubSettings settings) { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; - _storage = storage; _options = options; _settings = settings; } @@ -48,13 +48,15 @@ public class ChatHubCrontabHook : ICrontabHook } }, _options.JsonSerializerOptions); - if (_settings.EventDispatchBy == EventDispatchType.Group) - { - await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); - } - else + await SendEvent(item, json); + } + + private async Task SendEvent(CrontabItem item, string json) + { + try { await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); } + catch { } } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index a2f527f4..22ec7d4d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -12,6 +12,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR private readonly ChatHubSettings _settings; private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IConversationStateService _state; private readonly IUserIdentity _user; private readonly IAgentService _agentService; @@ -30,6 +31,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR ChatHubSettings settings, IServiceProvider serivces, IHubContext chatHub, + ILogger logger, IConversationStateService state, IUserIdentity user, IAgentService agentService, @@ -40,6 +42,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR _settings = settings; _services = serivces; _chatHub = chatHub; + _logger = logger; _state = state; _user = user; _agentService = agentService; @@ -82,6 +85,33 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR await SendContentLog(conversationId, input); } + public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) + { + var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + + // Agent queue log + var log = $"{instruction}"; + if (functions.Length > 0) + { + log += $"\r\n\r\n[FUNCTIONS]:\r\n\r\n{string.Join("\r\n\r\n", functions.Select(x => JsonSerializer.Serialize(x, BotSharpOptions.defaultJsonOptions)))}"; + } + _logger.LogInformation(log); + + var message = new RoleDialogModel(AgentRole.Assistant, log) + { + MessageId = _routingCtx.MessageId + }; + var input = new ContentLogInputModel(conversationId, message) + { + Name = agent.Name, + AgentId = agent.Id, + Source = ContentLogSource.Prompt, + Log = log + }; + await SendContentLog(conversationId, input); + } + public async Task OnRenderingTemplate(Agent agent, string name, string content) { if (!_convSettings.ShowVerboseLog) return; @@ -439,54 +469,85 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR #region Private methods private async Task SendContentLog(string conversationId, ContentLogInputModel input) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + _logger.LogWarning($"Failed to send content log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendStateLog(string conversationId, string agentId, Dictionary states, RoleDialogModel message) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + _logger.LogWarning($"Failed to send state log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendAgentQueueLog(string conversationId, string log) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + _logger.LogWarning($"Failed to send agent queue log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendStateChange(string conversationId, StateChangeModel stateChange) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + _logger.LogWarning($"Failed to send state change in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } - private string BuildContentLog(ContentLogInputModel input) { var output = new ContentLogOutputModel diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index c7aedd83..70545305 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -6,6 +6,7 @@ public class WelcomeHook : ConversationHookBase { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; @@ -17,6 +18,7 @@ public class WelcomeHook : ConversationHookBase public WelcomeHook(IServiceProvider services, IHubContext chatHub, + ILogger logger, IUserIdentity user, IConversationStorage storage, BotSharpOptions options, @@ -24,6 +26,7 @@ public class WelcomeHook : ConversationHookBase { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _storage = storage; _options = options; @@ -78,17 +81,30 @@ public class WelcomeHook : ConversationHookBase _storage.Append(conversation.Id, dialog); - if (_settings.EventDispatchBy == EventDispatchType.Group) - { - await _chatHub.Clients.Group(conversation.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); - } - else - { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); - } + await SendEvent(conversation.Id, json); } } await base.OnUserAgentConnectedInitially(conversation); } + + private async Task SendEvent(string conversationId, string json) + { + try + { + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Failed to send event in {nameof(WelcomeHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs index 240b5993..63ded25a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs @@ -33,12 +33,7 @@ public class SignalRHub : Hub if (!string.IsNullOrEmpty(conversationId)) { _logger.LogInformation($"Connection {Context.ConnectionId} is with conversation {conversationId}"); - - var settings = _services.GetRequiredService(); - if (settings.EventDispatchBy == EventDispatchType.Group) - { - await Groups.AddToGroupAsync(Context.ConnectionId, conversationId); - } + await AddGroup(conversationId); var conv = await convService.GetConversation(conversationId); if (conv != null) @@ -56,4 +51,21 @@ public class SignalRHub : Hub await base.OnConnectedAsync(); } + + private async Task AddGroup(string conversationId) + { + try + { + var settings = _services.GetRequiredService(); + if (settings.EventDispatchBy == EventDispatchType.Group) + { + await Groups.AddToGroupAsync(Context.ConnectionId, conversationId); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Failed to add chat group in {nameof(SignalRHub)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 56eb14ef..2f7e326a 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index ef4d0c8a..4c21dd03 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -9,9 +9,11 @@ public class ChatCompletionProvider : IChatCompletion { protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private List renderedInstructions = []; protected string _model; public virtual string Provider => "deepseek-ai"; + public string Model => _model; public ChatCompletionProvider( IServiceProvider services, @@ -51,7 +53,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -66,6 +69,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -107,7 +111,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -134,7 +139,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -170,7 +176,10 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; _logger.LogInformation(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); continue; } @@ -178,7 +187,10 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); } return true; @@ -197,6 +209,7 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; var messages = new List(); @@ -226,6 +239,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var text = agentService.RenderedInstruction(agent); + renderedInstructions.Add(text); messages.Add(new SystemChatMessage(text)); } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs index a1a88978..27198238 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs @@ -10,6 +10,7 @@ public class TextCompletionProvider : ITextCompletion protected string _model; public string Provider => "deepseek-ai"; + public string Model => _model; public TextCompletionProvider( IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f1a6c28e..c9ceb1e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -33,7 +33,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index 08cfd79a..51355e9c 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -27,9 +27,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index b6e0b24e..08d1f499 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 9567ae57..174efcf2 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -2,7 +2,6 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Loggers; -using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; using Mscc.GenerativeAI; @@ -12,10 +11,12 @@ public class GeminiChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public string Provider => "google-ai"; + public string Model => _model; public GeminiChatCompletionProvider( IServiceProvider services, @@ -53,7 +54,8 @@ public class GeminiChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = part.FunctionCall.Name, FunctionName = part.FunctionCall.Name, - FunctionArgs = part.FunctionCall.Args?.ToString() + FunctionArgs = part.FunctionCall.Args?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } else @@ -62,6 +64,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -98,6 +101,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var googleSettings = _services.GetRequiredService(); + renderedInstructions = []; // Add settings aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; @@ -117,6 +121,7 @@ public class GeminiChatCompletionProvider : IChatCompletion Role = AgentRole.User }); + renderedInstructions.Add(instruction); systemPrompts.Add(instruction); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index c62aebb8..4fe6ad17 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -13,10 +13,12 @@ public class PalmChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public string Provider => "google-palm"; + public string Model => _model; public PalmChatCompletionProvider( IServiceProvider services, @@ -61,7 +63,8 @@ public class PalmChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, FunctionName = llmResponse.FunctionName, - FunctionArgs = JsonSerializer.Serialize(llmResponse.Args) + FunctionArgs = JsonSerializer.Serialize(llmResponse.Args), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } else @@ -75,7 +78,8 @@ public class PalmChatCompletionProvider : IChatCompletion msg = new RoleDialogModel(llmResponse.Role, llmResponse.Content ?? message.Content) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index e14071f7..ebbca98a 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -14,6 +14,7 @@ public class GeminiTextCompletionProvider : ITextCompletion private string _model; public string Provider => "google-ai"; + public string Model => _model; public GeminiTextCompletionProvider( IServiceProvider services, @@ -74,7 +75,6 @@ public class GeminiTextCompletionProvider : ITextCompletion _model = model; } - private void PrepareOptions(GenerativeModel aiModel) { var settings = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs index ebaf6c5f..7df18bb7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -14,6 +14,7 @@ public class PalmTextCompletionProvider : ITextCompletion private string _model; public string Provider => "google-palm"; + public string Model => _model; public PalmTextCompletionProvider( IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 0465dc4e..99378ddc 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -60,9 +60,9 @@ public class GraphDb : IGraphDb using (var client = http.CreateClient()) { - var uri = new Uri(url); try { + var uri = new Uri(url); var data = JsonSerializer.Serialize(request, _jsonOptions); var message = new HttpRequestMessage { diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj index ae6a05be..0a9efc58 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 88b37dbb..460677ba 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -10,10 +10,12 @@ namespace BotSharp.Plugin.HuggingFace.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "huggingface"; + public string Model => _model; private readonly IServiceProvider _services; private readonly HuggingFaceSettings _settings; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj index 4596ce1b..95d1c014 100644 --- a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 77a5d076..aad7d494 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -23,8 +23,6 @@ - - @@ -46,17 +44,17 @@ PreserveNewest - + PreserveNewest - + PreserveNewest - - + + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs index de90430d..e4518344 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs @@ -2,5 +2,5 @@ namespace BotSharp.Plugin.KnowledgeBase.Enum; public class UtilityName { - public const string KnowledgeRetrieval = "knowledge.knowledge-retrieval"; + public const string KnowledgeRetrieval = "kg.knowledge-base"; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f3f8cbf1..940c3834 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { - public string Name => "util-knowledge-knowledge_retrieval"; + public string Name => "util-kg-knowledge_retrieval"; public string Indication => "searching my brain"; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index 700b8457..84254841 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook { - private static string PREFIX = "util-knowledge-"; + private static string PREFIX = "util-kg-"; private static string KNOWLEDGE_RETRIEVAL_FN = $"{PREFIX}knowledge_retrieval"; public void AddUtilities(List utilities) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index da4add53..90aa0114 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -40,6 +40,20 @@ public class KnowledgeHook : IKnowledgeHook var result = await _knowledgeService.SearchGraphKnowledge(text, options); results.Add(result.Result); } + else if (knowledgeBase.Type == "document") + { + var options = new VectorSearchOptions + { + Fields = null, + Limit = 5, + Confidence = 0.25f, + WithVector = true + }; + var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) + .Select(x => x.Data["text"].ToString()) + .Where(x => x != null)!); + } else { var options = new VectorSearchOptions diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs new file mode 100644 index 00000000..4b839332 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs @@ -0,0 +1,64 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetVectorCollectionSnapshots(string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return Enumerable.Empty(); + } + + var db = GetVectorDb(); + var snapshots = await db.GetCollectionSnapshots(collectionName); + return snapshots; + } + + public async Task CreateVectorCollectionSnapshot(string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return null; + } + + var db = GetVectorDb(); + var snapshot = await db.CreateCollectionShapshot(collectionName); + return snapshot; + } + + public async Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) + { + return BinaryData.Empty; + } + + var db = GetVectorDb(); + var snapshot = await db.DownloadCollectionSnapshot(collectionName, snapshotFileName); + return snapshot; + } + + public async Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + var done = await db.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData); + return done; + } + + public async Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) + { + return false; + } + + var db = GetVectorDb(); + var done = await db.DeleteCollectionShapshot(collectionName, snapshotName); + return done; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json similarity index 86% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json index 088f4ea6..cb79298a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json @@ -1,5 +1,5 @@ { - "name": "util-knowledge-knowledge_retrieval", + "name": "util-kg-knowledge_retrieval", "description": "Retrieve related domain knowledge to handle user request", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid similarity index 63% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid index f0bc60c8..e7d61b7f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid @@ -1,3 +1,3 @@ -Call function util-knowledge-knowledge_retrieval to retrieve related domain knowledge to handle user request. +Call function util-kg-knowledge_retrieval to retrieve related domain knowledge to handle user request. You must retrieve existing KnowledgeBase to get prerequisite informations before you writing SQL query; You must retrieve existing API specification from KnowledgeBase before calling a Web API; diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index d0bea79a..af7a6237 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 32f7220b..45f41e95 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -8,6 +8,7 @@ public class ChatCompletionProvider : IChatCompletion private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly LlamaSharpSettings _settings; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, @@ -20,6 +21,7 @@ public class ChatCompletionProvider : IChatCompletion } public string Provider => "llama-sharp"; + public string Model => _model; public async Task GetChatCompletions(Agent agent, List conversations) { @@ -64,7 +66,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = instruction }; // After chat completion hook @@ -146,7 +149,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = agent.Instruction }; // Text response received diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs index 2a98becc..e911a8ff 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs @@ -10,6 +10,7 @@ public class TextCompletionProvider : ITextCompletion private readonly ITokenStatistics _tokenStatistics; private string _model; public string Provider => "llama-sharp"; + public string Model => _model; public TextCompletionProvider(IServiceProvider services, ILogger logger, diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs index 1ec461ae..5c58f470 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs @@ -1,6 +1,4 @@ using System.IO; -using System.Xml.Linq; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.LLamaSharp.Providers; @@ -14,6 +12,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected int _dimension = DEFAULT_DIMENSION; public string Provider => "llama-sharp"; + public string Model => string.Empty; public TextEmbeddingProvider(IServiceProvider services, LlamaSharpSettings settings) { diff --git a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj index 004d7233..7ed224d3 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj +++ b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs index 61f66119..aa6b1175 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs @@ -20,6 +20,8 @@ namespace BotSharp.Plugin.VertexAI.Providers IServiceProvider services) : IChatCompletion { public string Provider => "vertexai"; + public string Model => _model; + private readonly VertexAIConfiguration _config = config; private readonly ChatSettings? _settings = settings; private readonly IServiceProvider _services = services; diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs index 46186452..70f0cbdc 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs @@ -19,6 +19,8 @@ namespace BotSharp.Plugin.VertexAI.Providers IServiceProvider services) : ITextCompletion { public string Provider => "vertexai"; + public string Model => _model; + private readonly VertexAIConfiguration _config = config; private readonly ChatSettings? _settings = settings; private readonly IServiceProvider _services = services; diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 3d390f56..231b4b3f 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -15,8 +15,8 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs index 7d9c6141..07b5b76e 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using BotSharp.Plugin.MetaAI.Settings; using FastText.NetWrapper; using Microsoft.Extensions.DependencyInjection; @@ -17,6 +18,7 @@ public class fastTextEmbeddingProvider : ITextEmbedding private int _dimension; public string Provider => "meta-ai"; + public string Model => string.Empty; public fastTextEmbeddingProvider(IServiceProvider services) { diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj index 2e43183c..4e60cc00 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index f99293f4..b5b19671 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -5,11 +5,13 @@ namespace BotSharp.Plugin.MetaGLM.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "metaglm"; + public string Model => _model; private readonly MetaGLMSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly MetaGLMClient metaGLMClient; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, @@ -49,7 +51,8 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, response?.choices.FirstOrDefault()?.message.content) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.Last().MessageId, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -61,7 +64,8 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id, MessageId = conversations.Last().MessageId, FunctionName = toolcall.function.name, - FunctionArgs = toolcall.function.arguments + FunctionArgs = toolcall.function.arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -87,10 +91,12 @@ public class ChatCompletionProvider : IChatCompletion var agentService = _services.GetRequiredService(); List messages = new List(); List toolcalls = new List(); + renderedInstructions = []; if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); + renderedInstructions.Add(instruction); messages.Add(new MessageItem("system", instruction)); } diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj index 225d7a90..284a0b4b 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj @@ -14,8 +14,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj index b2667f35..29dcc453 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 8619ecad..6ee77772 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -45,6 +45,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio /// public string Provider => "microsoft.extensions.ai"; + public string Model => _model ?? ""; /// public void SetModelName(string model) => _model = model; @@ -54,6 +55,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio { // Before chat completion hook var hooks = _services.GetServices().ToArray(); + List renderedInstructions = []; await Task.WhenAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations))); // Configure options @@ -82,6 +84,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio if (_services.GetRequiredService().RenderedInstruction(agent) is string instruction && instruction.Length > 0) { + renderedInstructions.Add(instruction); messages.Add(new(ChatRole.System, instruction)); } @@ -141,12 +144,13 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio var completion = await _client.GetResponseAsync(messages); - RoleDialogModel result = new(AgentRole.Assistant, string.Concat(completion.Message.Contents.OfType())) + RoleDialogModel result = new(AgentRole.Assistant, completion.Text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + //RenderedInstruction = renderedInstructions, }; - if (completion.Message.Contents.OfType().FirstOrDefault() is { } fcc) + if (completion.Messages.SelectMany(m => m.Contents).OfType().FirstOrDefault() is { } fcc) { result.Role = AgentRole.Function; result.MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs index 94d26678..ef3b35c7 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs @@ -24,6 +24,7 @@ public sealed class MicrosoftExtensionsAITextCompletionProvider : ITextCompletio /// public string Provider => "microsoft-extensions-ai"; + public string Model => _model; /// /// Creates an instance of the class. @@ -50,7 +51,7 @@ public sealed class MicrosoftExtensionsAITextCompletionProvider : ITextCompletio _tokenStatistics.StartTimer(); var completion = await _chatClient.GetResponseAsync(text); - var result = string.Concat(completion.Message.Contents.OfType()); + var result = completion.Text; _tokenStatistics.StopTimer(); // After chat completion hook diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs index 958c4694..1758468a 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs @@ -23,6 +23,7 @@ public sealed class MicrosoftExtensionsAITextEmbeddingProvider : ITextEmbedding /// public string Provider => "microsoft-extensions-ai"; + public string Model => _model; /// public async Task GetVectorAsync(string text) => diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj index 5ac884e0..26843035 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs index 7f09318a..8cade792 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs @@ -23,8 +23,8 @@ public class AgentTaskDocument : MongoBase Enabled = model.Enabled, AgentId = model.AgentId, Status = model.Status, - CreatedDateTime = model.CreatedTime, - UpdatedDateTime = model.UpdatedTime + CreatedTime = model.CreatedTime, + UpdatedTime = model.UpdatedTime }; } @@ -38,8 +38,8 @@ public class AgentTaskDocument : MongoBase Enabled = model.Enabled, AgentId = model.AgentId, Status = model.Status, - CreatedTime = model.CreatedDateTime, - UpdatedTime = model.UpdatedDateTime + CreatedTime = model.CreatedTime, + UpdatedTime = model.UpdatedTime }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index ace9472d..8fb85239 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -14,4 +14,5 @@ public class ConversationDocument : MongoBase public List Tags { get; set; } = []; public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } + public Dictionary LatestStates { get; set; } = new(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogDocument.cs new file mode 100644 index 00000000..91bcdc68 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogDocument.cs @@ -0,0 +1,50 @@ +using BotSharp.Abstraction.Loggers.Models; + +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class InstructionLogDocument : MongoBase +{ + public string? AgentId { get; set; } + public string Provider { get; set; } = default!; + public string Model { get; set; } = default!; + public string? TemplateName { get; set; } + public string UserMessage { get; set; } = default!; + public string? SystemInstruction { get; set; } + public string CompletionText { get; set; } = default!; + public string? UserId { get; set; } + public Dictionary States { get; set; } = new(); + public DateTime CreatedTime { get; set; } + + public static InstructionLogDocument ToMongoModel(InstructionLogModel log) + { + return new InstructionLogDocument + { + AgentId = log.AgentId, + Provider = log.Provider, + Model = log.Model, + TemplateName = log.TemplateName, + UserMessage = log.UserMessage, + SystemInstruction = log.SystemInstruction, + CompletionText = log.CompletionText, + UserId = log.UserId, + CreatedTime = log.CreatedTime + }; + } + + public static InstructionLogModel ToDomainModel(InstructionLogDocument log) + { + return new InstructionLogModel + { + Id = log.Id, + AgentId = log.AgentId, + Provider = log.Provider, + Model = log.Model, + TemplateName = log.TemplateName, + UserMessage = log.UserMessage, + SystemInstruction = log.SystemInstruction, + CompletionText = log.CompletionText, + UserId = log.UserId, + CreatedTime = log.CreatedTime + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 87022130..030a93ab 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -11,7 +11,6 @@ public class DialogMongoElement public string? RichContent { get; set; } public string? SecondaryRichContent { get; set; } public string? Payload { get; set; } - public object? Data { get; set; } public static DialogMongoElement ToMongoElement(DialogElement dialog) { @@ -22,8 +21,7 @@ public class DialogMongoElement SecondaryContent = dialog.SecondaryContent, RichContent = dialog.RichContent, SecondaryRichContent = dialog.SecondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }; } @@ -36,8 +34,7 @@ public class DialogMongoElement SecondaryContent = dialog.SecondaryContent, RichContent = dialog.RichContent, SecondaryRichContent = dialog.SecondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 4aa82b2e..73853230 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -115,7 +115,7 @@ public class MongoDbContext { var collection = GetCollectionOrCreate("ConversationContentLogs"); var indexes = collection.Indexes.List().ToList(); - var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); if (createTimeIndex == null) { var indexDef = Builders.IndexKeys.Ascending(x => x.CreatedTime); @@ -128,7 +128,7 @@ public class MongoDbContext { var collection = GetCollectionOrCreate("ConversationStateLogs"); var indexes = collection.Indexes.List().ToList(); - var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); if (createTimeIndex == null) { var indexDef = Builders.IndexKeys.Ascending(x => x.CreatedTime); @@ -136,6 +136,19 @@ public class MongoDbContext } return collection; } + + private IMongoCollection CreateInstructionLogIndex() + { + var collection = GetCollectionOrCreate("InstructionLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } #endregion #endregion @@ -193,4 +206,6 @@ public class MongoDbContext public IMongoCollection GlobalStatistics => GetCollectionOrCreate("GlobalStatistics"); + public IMongoCollection InstructionLogs + => CreateInstructionLogIndex(); } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 06512fb7..ebae075b 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,7 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Driver; +using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -23,7 +25,8 @@ public partial class MongoRepository Status = conversation.Status, Tags = conversation.Tags ?? new(), CreatedTime = utcNow, - UpdatedTime = utcNow + UpdatedTime = utcNow, + LatestStates = [] }; var dialogDoc = new ConversationDialogDocument @@ -72,8 +75,9 @@ public partial class MongoRepository var cronDeleted = _dc.CrontabItems.DeleteMany(conbTabItems); var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0 - || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0; + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 + || promptLogDeleted.DeletedCount > 0 || contentLogDeleted.DeletedCount > 0 + || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0; } [SideCar] @@ -130,13 +134,20 @@ public partial class MongoRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } - public bool UpdateConversationTags(string conversationId, List tags) + public bool UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) { if (string.IsNullOrEmpty(conversationId)) return false; var filter = Builders.Filter.Eq(x => x.Id, conversationId); + var conv = _dc.Conversations.Find(filter).FirstOrDefault(); + if (conv == null) return false; + + var tags = conv.Tags ?? []; + tags = tags.Concat(toAddTags).Distinct().ToList(); + tags = tags.Where(x => !toDeleteTags.Contains(x, StringComparer.OrdinalIgnoreCase)).ToList(); + var update = Builders.Update - .Set(x => x.Tags, tags ?? new()) + .Set(x => x.Tags, tags) .Set(x => x.UpdatedTime, DateTime.UtcNow); var res = _dc.Conversations.UpdateOne(filter, update); @@ -266,6 +277,14 @@ public partial class MongoRepository .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); + + // Update latest states + var endNodes = BuildLatestStates(saveStates); + var filter = Builders.Filter.Eq(x => x.Id, conversationId); + var update = Builders.Update.Set(x => x.LatestStates, endNodes) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Conversations.UpdateOne(filter, update); } public void UpdateConversationStatus(string conversationId, string status) @@ -371,21 +390,47 @@ public partial class MongoRepository } // Filter states - var stateFilters = new List>(); - if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty()) + if (filter != null && !filter.States.IsNullOrEmpty()) { foreach (var pair in filter.States) { - var elementFilters = new List> { Builders.Filter.Eq(x => x.Key, pair.Key) }; - if (!string.IsNullOrEmpty(pair.Value)) - { - elementFilters.Add(Builders.Filter.Eq("Values.Data", pair.Value)); - } - stateFilters.Add(Builders.Filter.ElemMatch(x => x.States, Builders.Filter.And(elementFilters))); - } + if (string.IsNullOrWhiteSpace(pair.Key)) continue; - var targetConvIds = _dc.ConversationStates.Find(Builders.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList(); - convFilters.Add(convBuilder.In(x => x.Id, targetConvIds)); + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "LatestStates"); + var formattedKey = string.Join(".", keys); + + if (string.IsNullOrWhiteSpace(pair.Value)) + { + convFilters.Add(convBuilder.Exists(formattedKey)); + } + else if (bool.TryParse(pair.Value, out var boolValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, boolValue)); + } + else if (int.TryParse(pair.Value, out var intValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, intValue)); + } + else if (decimal.TryParse(pair.Value, out var decimalValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, decimalValue)); + } + else if (float.TryParse(pair.Value, out var floatValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, floatValue)); + } + else if (double.TryParse(pair.Value, out var doubleValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, doubleValue)); + } + else + { + convFilters.Add(convBuilder.Eq(formattedKey, pair.Value)); + } + } } // Sort and paginate @@ -466,16 +511,6 @@ public partial class MongoRepository batchSize = batchLimit; } - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - while (true) { var skip = (page - 1) * batchSize; @@ -537,6 +572,7 @@ public partial class MongoRepository var stateFilter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault(); + var endNodes = new Dictionary(); if (foundStates != null) { // Truncate states @@ -560,6 +596,7 @@ public partial class MongoRepository truncatedStates.Add(state); } foundStates.States = truncatedStates; + endNodes = BuildLatestStates(truncatedStates); } // Truncate breakpoints @@ -583,6 +620,7 @@ public partial class MongoRepository // Update conversation var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.LatestStates, endNodes) .Set(x => x.DialogCount, truncatedDialogs.Count); _dc.Conversations.UpdateOne(convFilter, updateConv); @@ -613,26 +651,65 @@ public partial class MongoRepository #if !DEBUG [SharpCache(10)] #endif - public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) + public List GetConversationStateSearchKeys(ConversationStateKeysFilter filter) { - var stateBuilder = Builders.Filter; - var sortDef = Builders.Sort.Descending(x => x.UpdatedTime); - var stateFilters = new List>() + var builder = Builders.Filter; + var sortDef = Builders.Sort.Descending(x => x.UpdatedTime); + var filters = new List>() { - stateBuilder.Exists(x => x.States), - stateBuilder.Ne(x => x.States, []) + builder.Exists(x => x.LatestStates), + builder.Ne(x => x.LatestStates, []) }; - var states = _dc.ConversationStates.Find(stateBuilder.And(stateFilters)) - .Sort(sortDef) - .Limit(convUpperLimit) - .ToList(); - var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + if (!filter.AgentIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + } + + if (!filter.UserIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.UserId, filter.UserIds)); + } + + var convDocs = _dc.Conversations.Find(builder.And(filters)) + .Sort(sortDef) + .Limit(filter.ConvLimit) + .ToList(); + var keys = convDocs.SelectMany(x => x.LatestStates.Select(x => x.Key)).Distinct().ToList(); return keys; } + public List GetConversationsToMigrate(int batchSize = 100) + { + var convFilter = Builders.Filter.Exists(x => x.LatestStates, false); + var sortDef = Builders.Sort.Ascending(x => x.CreatedTime); + var convIds = _dc.Conversations.Find(convFilter).Sort(sortDef) + .Limit(batchSize).ToEnumerable() + .Select(x => x.Id).ToList(); + return convIds ?? []; + } + + public bool MigrateConvsersationLatestStates(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var stateFilter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault(); + if (foundStates?.States == null) return false; + + var states = foundStates.States.ToList(); + var latestStates = BuildLatestStates(states); + + var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); + var convUpdate = Builders.Update.Set(x => x.LatestStates, latestStates); + _dc.Conversations.UpdateOne(convFilter, convUpdate); + + return true; + } + + #region Private methods private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); @@ -650,4 +727,35 @@ public partial class MongoRepository return pascalCase.ToString(); } + + private Dictionary BuildLatestStates(List states) + { + var endNodes = new Dictionary(); + if (states.IsNullOrEmpty()) + { + return endNodes; + } + + foreach (var pair in states) + { + var value = pair.Values?.LastOrDefault(); + if (value == null || !value.Active) continue; + + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + endNodes[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = value.Data }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(str); + endNodes[pair.Key] = json; + } + } + + return endNodes; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index e4c64527..8cd6e158 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,7 @@ using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Driver; +using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -32,7 +35,8 @@ public partial class MongoRepository { if (log == null) return; - var found = _dc.Conversations.AsQueryable().FirstOrDefault(x => x.Id == log.ConversationId); + var filter = Builders.Filter.Eq(x => x.Id, log.ConversationId); + var found = _dc.Conversations.Find(filter).FirstOrDefault(); if (found == null) return; var logDoc = new ConversationContentLogDocument @@ -50,25 +54,36 @@ public partial class MongoRepository _dc.ContentLogs.InsertOne(logDoc); } - public List GetConversationContentLogs(string conversationId) + public DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { - var logs = _dc.ContentLogs - .AsQueryable() - .Where(x => x.ConversationId == conversationId) - .Select(x => new ContentLogOutputModel - { - ConversationId = x.ConversationId, - MessageId = x.MessageId, - Name = x.Name, - AgentId = x.AgentId, - Role = x.Role, - Source = x.Source, - Content = x.Content, - CreatedTime = x.CreatedTime - }) - .OrderBy(x => x.CreatedTime) - .ToList(); - return logs; + var builder = Builders.Filter; + var logFilters = new List> + { + builder.Eq(x => x.ConversationId, conversationId), + builder.Lt(x => x.CreatedTime, filter.StartTime) + }; + var logSortDef = Builders.Sort.Descending(x => x.CreatedTime); + + var docs = _dc.ContentLogs.Find(builder.And(logFilters)).Sort(logSortDef).Limit(filter.Size).ToList(); + var logs = docs.Select(x => new ContentLogOutputModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + Name = x.Name, + AgentId = x.AgentId, + Role = x.Role, + Source = x.Source, + Content = x.Content, + CreatedTime = x.CreatedTime + }).ToList(); + + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion @@ -77,7 +92,8 @@ public partial class MongoRepository { if (log == null) return; - var found = _dc.Conversations.AsQueryable().FirstOrDefault(x => x.Id == log.ConversationId); + var filter = Builders.Filter.Eq(x => x.Id, log.ConversationId); + var found = _dc.Conversations.Find(filter).FirstOrDefault(); if (found == null) return; var logDoc = new ConversationStateLogDocument @@ -92,22 +108,190 @@ public partial class MongoRepository _dc.StateLogs.InsertOne(logDoc); } - public List GetConversationStateLogs(string conversationId) + public DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { - var logs = _dc.StateLogs - .AsQueryable() - .Where(x => x.ConversationId == conversationId) - .Select(x => new ConversationStateLogModel - { - ConversationId = x.ConversationId, - AgentId = x.AgentId, - MessageId = x.MessageId, - States = x.States, - CreatedTime = x.CreatedTime - }) - .OrderBy(x => x.CreatedTime) - .ToList(); - return logs; + var builder = Builders.Filter; + var logFilters = new List> + { + builder.Eq(x => x.ConversationId, conversationId), + builder.Lt(x => x.CreatedTime, filter.StartTime) + }; + var logSortDef = Builders.Sort.Descending(x => x.CreatedTime); + + var docs = _dc.StateLogs.Find(builder.And(logFilters)).Sort(logSortDef).Limit(filter.Size).ToList(); + var logs = docs.Select(x => new ConversationStateLogModel + { + ConversationId = x.ConversationId, + AgentId = x.AgentId, + MessageId = x.MessageId, + States = x.States, + CreatedTime = x.CreatedTime + }).ToList(); + + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; + } + #endregion + + #region Instruction Log + public bool SaveInstructionLogs(IEnumerable logs) + { + if (logs.IsNullOrEmpty()) return false; + + var docs = new List(); + foreach (var log in logs) + { + var doc = InstructionLogDocument.ToMongoModel(log); + foreach (var pair in log.States) + { + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + doc.States[pair.Key] = json; + } + catch + { + var jsonStr = JsonSerializer.Serialize(new { Data = pair.Value }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + doc.States[pair.Key] = json; + } + } + docs.Add(doc); + } + + _dc.InstructionLogs.InsertMany(docs); + return true; + } + + public PagedItems GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var logBuilder = Builders.Filter; + var logFilters = new List>() { logBuilder.Empty }; + + // Filter logs + if (!filter.AgentIds.IsNullOrEmpty()) + { + logFilters.Add(logBuilder.In(x => x.AgentId, filter.AgentIds)); + } + if (!filter.Providers.IsNullOrEmpty()) + { + logFilters.Add(logBuilder.In(x => x.Provider, filter.Providers)); + } + if (!filter.Models.IsNullOrEmpty()) + { + logFilters.Add(logBuilder.In(x => x.Model, filter.Models)); + } + if (!filter.TemplateNames.IsNullOrEmpty()) + { + logFilters.Add(logBuilder.In(x => x.TemplateName, filter.TemplateNames)); + } + + // Filter states + if (filter != null && !filter.States.IsNullOrEmpty()) + { + foreach (var pair in filter.States) + { + if (string.IsNullOrWhiteSpace(pair.Key)) continue; + + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "States"); + var formattedKey = string.Join(".", keys); + + if (string.IsNullOrWhiteSpace(pair.Value)) + { + logFilters.Add(logBuilder.Exists(formattedKey)); + } + else if (bool.TryParse(pair.Value, out var boolValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, boolValue)); + } + else if (int.TryParse(pair.Value, out var intValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, intValue)); + } + else if (decimal.TryParse(pair.Value, out var decimalValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, decimalValue)); + } + else if (float.TryParse(pair.Value, out var floatValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, floatValue)); + } + else if (double.TryParse(pair.Value, out var doubleValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, doubleValue)); + } + else + { + logFilters.Add(logBuilder.Eq(formattedKey, pair.Value)); + } + } + } + + var filterDef = logBuilder.And(logFilters); + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); + var count = _dc.InstructionLogs.CountDocuments(filterDef); + + var logs = docs.Select(x => + { + var log = InstructionLogDocument.ToDomainModel(x); + log.States = x.States.ToDictionary(p => p.Key, p => + { + var jsonStr = p.Value.ToJson(); + var jsonDoc = JsonDocument.Parse(jsonStr); + var data = jsonDoc.RootElement.GetProperty("data"); + return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; + }); + return log; + }).ToList(); + + return new PagedItems + { + Items = logs, + Count = (int)count + }; + } + + public List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + { + var builder = Builders.Filter; + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var filters = new List>() + { + builder.Exists(x => x.States), + builder.Ne(x => x.States, []) + }; + + if (!filter.AgentIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + } + + if (!filter.UserIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.UserId, filter.UserIds)); + } + + var convDocs = _dc.InstructionLogs.Find(builder.And(filters)) + .Sort(sortDef) + .Limit(filter.LogLimit) + .ToList(); + var keys = convDocs.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + return keys; } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 258c1883..c91a291a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Options; using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -7,16 +8,19 @@ public partial class MongoRepository : IBotSharpRepository private readonly MongoDbContext _dc; private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly BotSharpOptions _botSharpOptions; private UpdateOptions _options; public MongoRepository( MongoDbContext dc, IServiceProvider services, - ILogger logger) + ILogger logger, + BotSharpOptions botSharpOptions) { _dc = dc; _services = services; _logger = logger; + _botSharpOptions = botSharpOptions; _options = new UpdateOptions { IsUpsert = true, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 9a9c57fb..fb8af0d3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -11,13 +11,14 @@ - - - + + + + \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index 1aca181b..2f7f79c6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -47,11 +47,14 @@ public class RealtimeSessionBody public FunctionDef[] Tools { get; set; } = []; [JsonPropertyName("turn_detection")] - public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); + public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new(); } public class RealtimeSessionTurnDetection { + [JsonPropertyName("interrupt_response")] + public bool InterruptResponse { get; set; } = true; + /// /// Milliseconds /// @@ -72,4 +75,11 @@ public class InputAudioTranscription { [JsonPropertyName("model")] public string Model { get; set; } = null!; + + [JsonPropertyName("language")] + public string Language { get; set; } = "en"; + + [JsonPropertyName("prompt")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Prompt { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs index b2125e99..cabf3d30 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -1,8 +1,27 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionCreationRequest : RealtimeSessionBody +public class RealtimeSessionCreationRequest { + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Model { get; set; } = null!; + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = ["audio", "text"]; + + [JsonPropertyName("instructions")] + public string Instructions { get; set; } = null!; + + [JsonPropertyName("tool_choice")] + public string ToolChoice { get; set; } = "auto"; + + [JsonPropertyName("tools")] + public FunctionDef[] Tools { get; set; } = []; + + [JsonPropertyName("turn_detection")] + public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); } /// diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs index 3311fd43..338affd9 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -7,6 +7,8 @@ public partial class AudioCompletionProvider : IAudioCompletion private readonly IServiceProvider _services; public string Provider => "openai"; + public string Model => _model; + private string _model; public AudioCompletionProvider(IServiceProvider service) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 15eb04c5..031fab9c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -10,8 +10,10 @@ public class ChatCompletionProvider : IChatCompletion protected readonly ILogger _logger; protected string _model; + private List renderedInstructions = []; public virtual string Provider => "openai"; + public string Model => _model; public ChatCompletionProvider( OpenAiSettings settings, @@ -53,7 +55,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -68,6 +71,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -112,7 +116,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -139,7 +144,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -175,7 +181,10 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; _logger.LogInformation(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); continue; } @@ -183,7 +192,10 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); } return true; @@ -198,6 +210,7 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; var messages = new List(); @@ -227,6 +240,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var text = agentService.RenderedInstruction(agent); + renderedInstructions.Add(text); messages.Add(new SystemChatMessage(text)); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 5b92d896..4131b814 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -13,6 +13,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected int _dimension = DEFAULT_DIMENSION; public virtual string Provider => "openai"; + public string Model => _model; public TextEmbeddingProvider( OpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 0e45e789..7cfda0f5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -14,6 +14,7 @@ public partial class ImageCompletionProvider : IImageCompletion protected string _model; public virtual string Provider => "openai"; + public string Model => _model; public ImageCompletionProvider( OpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 9b981cb1..2510c852 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,6 +1,11 @@ +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; using System.Net.WebSockets; @@ -37,9 +42,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Connect(RealtimeHubConnection conn, Action onModelReady, - Action onModelAudioDeltaReceived, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, @@ -56,13 +61,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (_webSocket.State == WebSocketState.Open) { - onModelReady(); - // Receive a message _ = ReceiveMessage(conn, + onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, - onAudioTranscriptDone, + onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, onInputAudioTranscriptionCompleted, @@ -72,7 +76,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Disconnect() { - await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + if (_webSocket.State == WebSocketState.Open) + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } } public async Task AppenAudioBuffer(string message) @@ -99,23 +106,41 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } - private async Task ReceiveMessage(RealtimeHubConnection conn, - Action onModelAudioDeltaReceived, + public async Task CancelModelResponse() + { + await SendEventToModel(new + { + type = "response.cancel" + }); + } + + public async Task RemoveConversationItem(string itemId) + { + await SendEventToModel(new + { + type = "conversation.item.delete", + item_id = itemId + }); + } + + private async Task ReceiveMessage(RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, + Action onUserAudioTranscriptionCompleted, Action onUserInterrupted) { - var buffer = new byte[1024 * 1024 * 1]; + var buffer = new byte[1024 * 32]; WebSocketReceiveResult result; - string lastAssistantItem = ""; + do { result = await _webSocket.ReceiveAsync( new ArraySegment(buffer), CancellationToken.None); - + // Convert received data to text/audio (Twilio sends Base64-encoded audio) string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); if (string.IsNullOrEmpty(receivedText)) @@ -133,6 +158,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "session.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); + onModelReady(); } else if (response.Type == "session.updated") { @@ -146,16 +172,16 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); - onAudioTranscriptDone(data.Transcript); + await Task.Delay(1000); + onModelAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { var audio = JsonSerializer.Deserialize(receivedText); - lastAssistantItem = audio?.ItemId ?? ""; - - if (audio != null && audio.Delta != null) + if (audio?.Delta != null) { - onModelAudioDeltaReceived(audio.Delta); + _logger.LogDebug($"{response.Type}: {receivedText}"); + onModelAudioDeltaReceived(audio.Delta, audio.ItemId); } } else if (response.Type == "response.audio.done") @@ -166,7 +192,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - await Task.Delay(1000); var messages = await OnResponsedDone(conn, receivedText); onModelResponseDone(messages); } @@ -178,23 +203,34 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "conversation.item.input_audio_transcription.completed") { _logger.LogInformation($"{response.Type}: {receivedText}"); - var message = await OnInputAudioTranscriptionCompleted(conn, receivedText); - onInputAudioTranscriptionCompleted(message); + var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); + if (!string.IsNullOrEmpty(message.Content)) + { + onUserAudioTranscriptionCompleted(message); + } } else if (response.Type == "input_audio_buffer.speech_started") { - // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; - // handle use interuption - var truncateEvent = new + // Handle user interuption + if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) { - type = "conversation.item.truncate", - item_id = lastAssistantItem, - content_index = 0, - audio_end_ms = 100 - }; + var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; - await SendEventToModel(truncateEvent); - onUserInterrupted(); + if (!string.IsNullOrEmpty(conn.LastAssistantItemId)) + { + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = conn.LastAssistantItemId, + content_index = 0, + audio_end_ms = elapsedTime + }; + + await SendEventToModel(truncateEvent); + } + + onUserInterrupted(); + } } } while (!result.CloseStatus.HasValue); @@ -204,12 +240,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { + if (_webSocket.State != WebSocketState.Open) + { + return; + } + if (message is not string data) { - data = JsonSerializer.Serialize(message); + data = JsonSerializer.Serialize(message, BotSharpOptions.defaultJsonOptions); } var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } @@ -225,6 +267,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var args = new RealtimeSessionCreationRequest { + Model = _model, Instructions = instruction, ToolChoice = "auto", Tools = options.Tools.Select(x => @@ -240,26 +283,41 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, args.Model); + var settings = settingsService.GetSetting(Provider, args.Model ?? _model); var api = _services.GetRequiredService(); var session = await api.GetSessionAsync(args, settings.ApiKey); return session; } - public async Task UpdateInitialSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conv.AgentId); + var agent = await agentService.LoadAgent(conn.CurrentAgentId); var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, []); var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + var functions = options.Tools.Select(x => + { + var fn = new FunctionDef + { + Name = x.FunctionName, + Description = x.FunctionDescription + }; + fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); + return fn; + }).ToArray(); + + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + + var realitmeModelSettings = _services.GetRequiredService(); var sessionUpdate = new { @@ -271,26 +329,34 @@ public class RealTimeCompletionProvider : IRealTimeCompletion InputAudioTranscription = new InputAudioTranscription { Model = "whisper-1", + Language = "en", + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }, Voice = "alloy", Instructions = instruction, ToolChoice = "auto", - Tools = options.Tools.Select(x => - { - var fn = new FunctionDef - { - Name = x.FunctionName, - Description = x.FunctionDescription - }; - fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); - return fn; - }).ToArray(), + Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) + Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f), + MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens, + TurnDetection = new RealtimeSessionTurnDetection + { + InterruptResponse = interruptResponse, + Threshold = realitmeModelSettings.TurnDetection.Threshold, + PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding, + SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration + } } }; + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionUpdated(agent, instruction, functions); + }); + await SendEventToModel(sessionUpdate); + + return instruction; } public async Task InsertConversationItem(RoleDialogModel message) @@ -494,7 +560,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion })); prompt += $"{verbose}\r\n"; - prompt += "\r\n[CONVERSATION]"; verbose = string.Join("\r\n", messages .Where(x => x as SystemChatMessage == null) .Select(x => @@ -525,7 +590,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return string.Empty; })); - prompt += $"\r\n{verbose}\r\n"; + + if (!string.IsNullOrEmpty(verbose)) + { + prompt += $"\r\n[CONVERSATION]\r\n{verbose}\r\n"; + } } if (!options.Tools.IsNullOrEmpty()) @@ -550,16 +619,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var outputs = new List(); var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + return []; + } + foreach (var output in data.Outputs) { if (output.Type == "function_call") { outputs.Add(new RoleDialogModel(output.Role, output.Arguments) { - CurrentAgentId = conn.EntryAgentId, + CurrentAgentId = conn.CurrentAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, - ToolCallId = output.CallId + ToolCallId = output.CallId, + MessageId = output.Id, + MessageType = MessageTypeName.FunctionCall }); } else if (output.Type == "message") @@ -568,20 +644,38 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(output.Role, content.Transcript) { - CurrentAgentId = conn.EntryAgentId + CurrentAgentId = conn.CurrentAgentId, + MessageId = output.Id, + MessageType = MessageTypeName.Plain }); } } + var contentHooks = _services.GetServices().ToList(); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } + return outputs; } - public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) { var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) { - CurrentAgentId = conn.EntryAgentId + CurrentAgentId = conn.CurrentAgentId }; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index c2180076..91b39848 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -3,6 +3,7 @@ using System.Net.Http; using System.Net.Mime; using System.Text.Json; using System.Text; +using BotSharp.Abstraction.Options; namespace BotSharp.Plugin.OpenAI.Providers.Text; @@ -13,15 +14,8 @@ public class TextCompletionProvider : ITextCompletion private readonly OpenAiSettings _settings; protected string _model; - protected readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - PropertyNameCaseInsensitive = true, - WriteIndented = true, - AllowTrailingCommas = true, - }; - public virtual string Provider => "openai"; + public string Model => _model; public TextCompletionProvider( OpenAiSettings settings, @@ -110,7 +104,7 @@ public class TextCompletionProvider : ITextCompletion MaxTokens = maxTokens, Temperature = temperature }; - var data = JsonSerializer.Serialize(request, _jsonOptions); + var data = JsonSerializer.Serialize(request, BotSharpOptions.defaultJsonOptions); var httpRequest = new HttpRequestMessage { Method = HttpMethod.Post, @@ -121,7 +115,7 @@ public class TextCompletionProvider : ITextCompletion var httpResponse = await httpClient.SendAsync(httpRequest); httpResponse.EnsureSuccessStatusCode(); var responseStr = await httpResponse.Content.ReadAsStringAsync(); - var response = JsonSerializer.Deserialize(responseStr, _jsonOptions); + var response = JsonSerializer.Deserialize(responseStr, BotSharpOptions.defaultJsonOptions); return response; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 75f0246a..27a51abf 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -11,15 +11,15 @@ - - - - - - - - - + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj index c5e3d4ff..391593b3 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj @@ -25,7 +25,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj index fb19756b..67ee3934 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj +++ b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs new file mode 100644 index 00000000..f346aa1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Qdrant.Models; + +public class RecoverFromSnapshotResponse +{ + [JsonPropertyName("time")] + public decimal Time { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("result")] + public bool Result { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index ef9faa93..3c5d61e5 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,8 +1,14 @@ +using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Plugin.Qdrant.Models; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; +using System.Net.Http; +using System.Net.Mime; +using System.Text.Json; namespace BotSharp.Plugin.Qdrant; @@ -10,15 +16,18 @@ public class QdrantDb : IVectorDb { private QdrantClient _client; private readonly QdrantSetting _setting; + private readonly BotSharpOptions _options; private readonly IServiceProvider _services; private readonly ILogger _logger; public QdrantDb( QdrantSetting setting, + BotSharpOptions options, ILogger logger, IServiceProvider services) { _setting = setting; + _options = options; _logger = logger; _services = services; } @@ -39,6 +48,7 @@ public class QdrantDb : IVectorDb return _client; } + #region Collection public async Task DoesCollectionExist(string collectionName) { var client = GetClient(); @@ -86,7 +96,9 @@ public class QdrantDb : IVectorDb var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } + #endregion + #region Collection data public async Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { var exist = await DoesCollectionExist(collectionName); @@ -332,4 +344,150 @@ public class QdrantDb : IVectorDb var result = await client.DeleteAsync(collectionName, new Filter()); return result.Status == UpdateStatus.Completed; } + #endregion + + #region Snapshots + public async Task> GetCollectionSnapshots(string collectionName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return Enumerable.Empty(); + } + + var client = GetClient(); + var data = await client.ListSnapshotsAsync(collectionName); + var snapshots = data.Select(x => new VectorCollectionSnapshot + { + Name = x.Name, + Size = x.Size, + CreatedTime = x.CreationTime.ToDateTime(), + CheckSum = x.Checksum + }); + return snapshots; + } + + public async Task CreateCollectionShapshot(string collectionName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return null; + } + + var client = GetClient(); + var desc = await client.CreateSnapshotAsync(collectionName); + if (desc == null) + { + return null; + } + + return new VectorCollectionSnapshot + { + Name = desc.Name, + Size = desc.Size, + CreatedTime = desc.CreationTime.ToDateTime(), + CheckSum = desc.Checksum + }; + } + + public async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return BinaryData.Empty; + } + + var domain = $"https://{_setting.Url}:6333"; + var url = $"{domain}/collections/{collectionName}/snapshots/{snapshotFileName}"; + + var http = _services.GetRequiredService(); + using (var client = http.CreateClient()) + { + try + { + var uri = new Uri(url); + var message = new HttpRequestMessage + { + Method = HttpMethod.Get, + RequestUri = uri + }; + + client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey); + var rawResponse = await client.SendAsync(message); + rawResponse.EnsureSuccessStatusCode(); + + using var contentStream = await rawResponse.Content.ReadAsStreamAsync(); + return BinaryData.FromStream(contentStream); + } + catch (Exception ex) + { + _logger.LogError($"Error when downloading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}"); + return BinaryData.Empty; + } + } + } + + public async Task RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) + { + var domain = $"https://{_setting.Url}:6333"; + var url = $"{domain}/collections/{collectionName}/snapshots/upload"; + + var http = _services.GetRequiredService(); + using (var client = http.CreateClient()) + { + try + { + var uri = new Uri(url); + var data = new MultipartFormDataContent + { + { new StringContent(snapshotFileName), "name" }, + { new StringContent(MediaTypeNames.Application.Octet), "type" }, + { new StreamContent(snapshotData.ToStream()), "snapshot", snapshotFileName } + }; + + var message = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = uri, + Content = data + }; + + client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey); + var rawResponse = await client.SendAsync(message); + rawResponse.EnsureSuccessStatusCode(); + + var responseStr = await rawResponse.Content.ReadAsStringAsync(); + var response = JsonSerializer.Deserialize(responseStr, _options.JsonSerializerOptions); + return response?.Result == true; + } + catch (Exception ex) + { + _logger.LogError($"Error when uploading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + } + + public async Task DeleteCollectionShapshot(string collectionName, string snapshotName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return false; + } + + try + { + var client = GetClient(); + await client.DeleteSnapshotAsync(collectionName, snapshotName); + return true; + } + catch + { + return false; + } + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj index 488e0db5..a9ff968b 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj @@ -29,8 +29,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj index 2a4e3e08..e11e847c 100644 --- a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj +++ b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj index b8c4a8e9..ebc46a11 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj @@ -11,9 +11,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index 156f238c..3f742777 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -26,6 +26,7 @@ namespace BotSharp.Plugin.SemanticKernel /// public string Provider => "semantic-kernel"; + public string Model => _model; /// /// Create a new instance of @@ -74,7 +75,8 @@ namespace BotSharp.Plugin.SemanticKernel var response = chatMessageContent != null ? chatMessageContent.Content :string.Empty; var msg = new RoleDialogModel(AgentRole.Assistant, response) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = instruction }; // After chat completion hook diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs index 91dec640..8b03c011 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs @@ -23,6 +23,7 @@ namespace BotSharp.Plugin.SemanticKernel /// public string Provider => "semantic-kernel"; + public string Model => _model; /// /// Create a new instance of diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs index 5486d972..64915090 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel.Embeddings; using System.Collections.Generic; @@ -33,6 +34,7 @@ namespace BotSharp.Plugin.SemanticKernel protected int _dimension; public string Provider => "semantic-kernel"; + public string Model => string.Empty; /// public async Task GetVectorAsync(string text) diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj index 04969796..51f26f87 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 5edf4708..d5e64e7d 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -7,10 +7,12 @@ namespace BotSharp.Plugin.SparkDesk.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "sparkdesk"; + public string Model => _model; private readonly SparkDeskSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, @@ -42,7 +44,8 @@ public class ChatCompletionProvider : IChatCompletion var responseMessage = new RoleDialogModel(AgentRole.Assistant, response.Text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.Last().MessageId, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; if (response.FunctionCall != null) @@ -52,7 +55,8 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id, MessageId = conversations.Last().MessageId, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments + FunctionArgs = response.FunctionCall.Arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -92,7 +96,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, response.Text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -116,7 +121,8 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments + FunctionArgs = response.FunctionCall.Arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -150,14 +156,16 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments + FunctionArgs = response.FunctionCall.Arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }); continue; } await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, response.Text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }); } @@ -175,10 +183,12 @@ public class ChatCompletionProvider : IChatCompletion var functions = new List(); var agentService = _services.GetRequiredService(); var messages = new List(); + renderedInstructions = []; if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); + renderedInstructions.Add(instruction); messages.Add(ChatMessage.FromSystem(instruction)); } if (!string.IsNullOrEmpty(agent.Knowledges)) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index dbd3228d..ec2d91c1 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -100,9 +100,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj index b52e930a..4d78adc0 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj +++ b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 91c81e90..8fe2ecce 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -6,27 +6,26 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages + enable - - - - - - + PreserveNewest - + + PreserveNewest + + PreserveNewest - - - - + + + + @@ -34,4 +33,8 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs new file mode 100644 index 00000000..65b2a5ad --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -0,0 +1,44 @@ +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioRecordController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TwilioRecordController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/record/status")] + public async Task PhoneRecordingStatus(ConversationalVoiceRequest request) + { + if (request.RecordingStatus == "completed") + { + _logger.LogInformation($"Recording completed for {request.CallSid}, the record URL is {request.RecordingUrl}"); + + // Set the recording URL to the conversation state + var convService = _services.GetRequiredService(); + convService.SetConversationId(request.ConversationId, new List + { + new("phone_recording_url", request.RecordingUrl) + }); + convService.SaveStates(); + + // recording completed + await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request)); + } + + return Ok(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e6de9fc5..6960f489 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -1,13 +1,12 @@ using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Twilio.TwiML.Voice; using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -36,25 +35,27 @@ public class TwilioStreamController : TwilioController throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } - VoiceResponse response = null; + VoiceResponse response = default!; + + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + response = new VoiceResponse(); + response.Play(new Uri(request.InitAudioFile)); + return TwiML(response); + } + var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = [], ActionOnEmptyResult = true }; - if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file")) + if (request.InitAudioFile != null) { - instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]); - } - - if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) - { - request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; - } - else - { - request.ConversationId = request.CallSid; + instruction.SpeechPaths.Add(request.InitAudioFile); } await HookEmitter.Emit(_services, async hook => @@ -65,7 +66,7 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - await InitConversation(request); + request.ConversationId = await InitConversation(request); var twilio = _services.GetRequiredService(); @@ -82,7 +83,7 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - private async Task InitConversation(ConversationalVoiceRequest request) + private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); @@ -90,10 +91,10 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - Id = request.CallSid, - AgentId = _settings.AgentId, + AgentId = request.AgentId ?? _settings.AgentId, Channel = ConversationChannel.Phone, - Title = $"Phone call from {request.From}", + ChannelId = request.CallSid, + Title = $"Incoming phone call from {request.From}", Tags = [], }; @@ -103,10 +104,20 @@ public class TwilioStreamController : TwilioController var states = new List { new("channel", ConversationChannel.Phone), - new("calling_phone", request.From) + new("calling_phone", request.From), + new("twilio_call_sid", request.CallSid), + // Enable lazy routing mode to optimize realtime experience + new(StateConst.ROUTING_MODE, "lazy"), }; + if (request.InitAudioFile != null) + { + states.Add(new("init_audio_file", request.InitAudioFile)); + } + convService.SetConversationId(conversation.Id, states); convService.SaveStates(); + + return conversation.Id; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 19144673..079e13c5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,13 +1,11 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Repositories; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using System.ComponentModel.DataAnnotations; using Twilio.Http; namespace BotSharp.Plugin.Twilio.Controllers; @@ -54,6 +52,7 @@ public class TwilioVoiceController : TwilioController VoiceResponse response = null; var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = ["twilio/welcome.mp3"], ActionOnEmptyResult = true }; @@ -66,7 +65,7 @@ public class TwilioVoiceController : TwilioController }); request.ConversationId = $"TwilioVoice_{request.CallSid}"; - instruction.CallbackPath = $"twilio/voice/{request.ConversationId}/receive/0?{GenerateStatesParameter(request.States)}"; + instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"; var twilio = _services.GetRequiredService(); if (string.IsNullOrWhiteSpace(request.Intent)) @@ -89,7 +88,7 @@ public class TwilioVoiceController : TwilioController }; await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post); } await HookEmitter.Emit(_services, async hook => @@ -108,8 +107,8 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] - [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] + [ValidateRequest] + [HttpPost("twilio/voice/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { var twilio = _services.GetRequiredService(); @@ -142,7 +141,7 @@ public class TwilioVoiceController : TwilioController await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); await HookEmitter.Emit(_services, async hook => { @@ -172,8 +171,9 @@ public class TwilioVoiceController : TwilioController { var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = new List(), - CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{request.SeqNum}?{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", + CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", ActionOnEmptyResult = true }; @@ -202,8 +202,8 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] - [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] + [ValidateRequest] + [HttpPost("twilio/voice/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { var nextSeqNum = request.SeqNum + 1; @@ -217,8 +217,22 @@ public class TwilioVoiceController : TwilioController var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum); VoiceResponse response; + + if (request.AIResponseWaitTime > 5) + { + // Wait AI Response Timeout + await HookEmitter.Emit(_services, async hook => + { + request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!"; + await hook.OnAgentHangUp(request); + }, new HookEmitOption + { + OnlyOnce = true + }); - if (reply == null) + response = twilio.HangUp($"twilio/error.mp3"); + } + else if (reply == null) { var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); if (indication != null) @@ -261,8 +275,9 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = speechPaths, - CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; @@ -300,8 +315,9 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = instructions, - CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; @@ -346,8 +362,9 @@ public class TwilioVoiceController : TwilioController { var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], - CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{nextSeqNum}?{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}", ActionOnEmptyResult = true, Hints = reply.Hints }; @@ -367,28 +384,39 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] - [HttpPost("twilio/voice/init-call")] - public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) + [ValidateRequest] + [HttpPost("twilio/voice/init-outbound-call")] + public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request) { + VoiceResponse response = default!; + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + response = new VoiceResponse(); + response.Play(new Uri(request.InitAudioFile)); + return TwiML(response); + } + var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, ActionOnEmptyResult = true, - CallbackPath = $"twilio/voice/{conversationId}/receive/1", - SpeechPaths = new List - { - $"twilio/voice/speeches/{conversationId}/intial.mp3" - } + CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}", }; - string tag = $"twilio:{Request.Form["AnsweredBy"]}"; - var db = _services.GetRequiredService(); - db.AppendConversationTags(conversationId, new List { tag }); + + if (request.InitAudioFile != null) + { + instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}"; + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); + } + var twilio = _services.GetRequiredService(); - var response = twilio.ReturnNoninterruptedInstructions(instruction); + response = twilio.ReturnNoninterruptedInstructions(instruction); return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { @@ -401,6 +429,41 @@ public class TwilioVoiceController : TwilioController return result; } + [ValidateRequest] + [HttpPost("twilio/voice/hang-up")] + public async Task Hangup(ConversationalVoiceRequest request) + { + var twilio = _services.GetRequiredService(); + var response = twilio.HangUp("twilio/bye.mp3"); + return TwiML(response); + } + + [ValidateRequest] + [HttpPost("twilio/voice/status")] + public async Task PhoneCallStatus(ConversationalVoiceRequest request) + { + if (request.CallStatus == "completed") + { + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + // voicemail + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailLeft(request); + }); + } + else + { + // phone call completed + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); + } + } + + return Ok(); + } + private Dictionary ParseStates(List states) { var result = new Dictionary(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs new file mode 100644 index 00000000..c9ed4710 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -0,0 +1,11 @@ +using BotSharp.Plugin.Twilio.Models; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Interfaces; + +public interface ITwilioCallStatusHook +{ + Task OnVoicemailLeft(ConversationalVoiceRequest request); + Task OnUserDisconnected(ConversationalVoiceRequest request); + Task OnRecordingCompleted(ConversationalVoiceRequest request); +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 2587fed9..547b09d9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.Twilio.Models public bool HumanIntervationNeeded { get; set; } public string Content { get; set; } public string MessageId { get; set; } - public string SpeechFileName { get; set; } + public string? SpeechFileName { get; set; } public string Hints { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 1fe78116..25e3ba0d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -4,15 +4,41 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceRequest : VoiceRequest { - [FromRoute] - public string ConversationId { get; set; } + [FromQuery(Name = "agent-id")] + public string AgentId { get; set; } = string.Empty; + + [FromQuery(Name = "conversation-id")] + public string ConversationId { get; set; } = string.Empty; [FromRoute] public int SeqNum { get; set; } public int Attempts { get; set; } = 1; + public int AIResponseWaitTime { get; set; } = 0; + public string? AIResponseErrorMessage { get; set; } = string.Empty; - public string Intent { get; set; } + public string Intent { get; set; } = string.Empty; + + [FromQuery(Name = "init-audio-file")] + public string? InitAudioFile { get; set; } public List States { get; set; } = []; + + [FromForm] + public string? CallbackSource { get; set; } + + /// + /// machine_start + /// + [FromForm] + public string? AnsweredBy { get; set; } + + [FromForm] + public int MachineDetectionDuration { get; set; } + + [FromForm] + public int Duration { get; set; } + + [FromForm] + public int CallDuration { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs index ea8072d6..7870f961 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceResponse { + public string ConversationId { get; set; } = null!; public List SpeechPaths { get; set; } = []; public string CallbackPath { get; set; } public bool ActionOnEmptyResult { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs new file mode 100644 index 00000000..c74eca98 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs @@ -0,0 +1,17 @@ +using BotSharp.Plugin.Twilio.Models.Stream; +using System.Text.Json.Serialization; + +public class StreamEventDtmfResponse : StreamEventResponse +{ + [JsonPropertyName("dtmf")] + public StreamEventDtmfBody Body { get; set; } +} + +public class StreamEventDtmfBody +{ + [JsonPropertyName("track")] + public string Track { get; set; } + + [JsonPropertyName("digit")] + public string Digit { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs index 63252a57..b7244e7d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs @@ -2,6 +2,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Enums { public class UtilityName { - public const string OutboundPhoneCall = "twilio.twilio-outbound-phone-call"; + public const string OutboundPhoneCall = "phone.twilio-phone-call"; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs deleted file mode 100644 index 12b827af..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ /dev/null @@ -1,106 +0,0 @@ -using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Abstraction.Options; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.Twilio.Interfaces; -using BotSharp.Plugin.Twilio.Models; -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; -using Twilio.Rest.Api.V2010.Account; -using Twilio.Types; - -namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions -{ - public class HandleOutboundPhoneCallFn : IFunctionCallback - { - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly BotSharpOptions _options; - private readonly TwilioSetting _twilioSetting; - - public string Name => "util-twilio-twilio_outbound_phone_call"; - public string Indication => "Dialing the number"; - - public HandleOutboundPhoneCallFn( - IServiceProvider services, - ILogger logger, - BotSharpOptions options, - TwilioSetting twilioSetting) - { - _services = services; - _logger = logger; - _options = options; - _twilioSetting = twilioSetting; - } - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); - if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) - { - var error = $"Invalid phone number format: {args.PhoneNumber}"; - _logger.LogError(error); - message.Content = error; - return false; - } - - if (string.IsNullOrWhiteSpace(args.InitialMessage)) - { - _logger.LogError("Initial message is empty."); - message.Content = "There is an error when generating phone message."; - return false; - } - - var convService = _services.GetRequiredService(); - var convStorage = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var sessionManager = _services.GetRequiredService(); - var states = _services.GetRequiredService(); - - // Fork conversation - var entryAgentId = routing.EntryAgentId; - var newConv = await convService.NewConversation(new Abstraction.Conversations.Models.Conversation - { - AgentId = entryAgentId, - Channel = ConversationChannel.Phone - }); - var conversationId = newConv.Id; - convStorage.Append(conversationId, new List - { - new RoleDialogModel(AgentRole.User, "Hi") - { - CurrentAgentId = entryAgentId - }, - new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) - { - CurrentAgentId = entryAgentId - } - }); - states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); - - // Generate audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - - // Call phone number - /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage - { - Content = args.InitialMessage, - SpeechFileName = fileName - });*/ - - var call = await CallResource.CreateAsync( - // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"), - to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber)); - - message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; - message.StopCompletion = true; - return true; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs new file mode 100644 index 00000000..9fa7d03c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -0,0 +1,59 @@ +using BotSharp.Abstraction.Routing; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class HangupPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-hangup_phone_call"; + public string Indication => "Hangup"; + + public HangupPhoneCallFn( + IServiceProvider services, + ILogger logger, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var states = _services.GetRequiredService(); + var callSid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(callSid)) + { + message.Content = "The call has not been initiated."; + _logger.LogError(message.Content); + return false; + } + + if (args.AnythingElseToHelp) + { + message.Content = "Tell me how I can help."; + } + else + { + var call = CallResource.Update( + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"), + pathSid: callSid + ); + + message.Content = "The call is ending."; + message.StopCompletion = true; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs new file mode 100644 index 00000000..1e10a135 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -0,0 +1,166 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; +using Twilio.Types; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class OutboundPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-outbound_phone_call"; + public string Indication => "Dialing the phone number"; + + public OutboundPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) + { + var error = $"Invalid phone number format: {args.PhoneNumber}"; + _logger.LogError(error); + message.Content = error; + return false; + } + + if (string.IsNullOrWhiteSpace(args.InitialMessage)) + { + _logger.LogError("Initial message is empty."); + message.Content = "There is an error when generating phone message."; + return false; + } + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + + // Fork conversation + var newConversationId = Guid.NewGuid().ToString(); + states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + + var processUrl = $"{_twilioSetting.CallbackHost}/twilio"; + var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}"; + var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}"; + + // Generate initial assistant audio + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.InitialMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + initAudioFile = "intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data); + + statusUrl += $"&init-audio-file={initAudioFile}"; + } + + // Set up process URL streaming or synchronous + if (_twilioSetting.StreamingEnabled) + { + processUrl += "/stream"; + } + else + { + var sessionManager = _services.GetRequiredService(); + await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage + { + Content = args.InitialMessage, + SpeechFileName = initAudioFile + }); + + processUrl += "/voice/init-outbound-call"; + } + + processUrl += $"?conversation-id={newConversationId}"; + if (!string.IsNullOrEmpty(initAudioFile)) + { + processUrl += $"&init-audio-file={initAudioFile}"; + } + + // Make outbound call + var call = await CallResource.CreateAsync( + url: new Uri(processUrl), + to: new PhoneNumber(args.PhoneNumber), + from: new PhoneNumber(_twilioSetting.PhoneNumber), + statusCallback: new Uri(statusUrl), + // https://www.twilio.com/docs/voice/answering-machine-detection + machineDetection: _twilioSetting.MachineDetection, + record: _twilioSetting.RecordingEnabled, + recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}"); + + var convService = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var originConversationId = convService.ConversationId; + var entryAgentId = routing.EntryAgentId; + + await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); + + message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]"; + message.StopCompletion = true; + return true; + } + + private async Task ForkConversation(LlmContextIn args, + string entryAgentId, + string originConversationId, + string newConversationId, + CallResource resource) + { + // new scope service for isolated conversation + using var scope = _services.CreateScope(); + var services = scope.ServiceProvider; + var convService = services.GetRequiredService(); + var convStorage = services.GetRequiredService(); + + var newConv = await convService.NewConversation(new Conversation + { + Id = newConversationId, + AgentId = entryAgentId, + Channel = ConversationChannel.Phone, + ChannelId = resource.Sid, + Title = args.InitialMessage + }); + + convStorage.Append(newConversationId, new List + { + new RoleDialogModel(AgentRole.User, "Hi") + { + CurrentAgentId = entryAgentId + }, + new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) + { + CurrentAgentId = entryAgentId + } + }); + + convService.SetConversationId(newConversationId, + [ + new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId), + new MessageState("phone_number", resource.To) + ]); + convService.SaveStates(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs new file mode 100644 index 00000000..c7c6fc88 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs @@ -0,0 +1,49 @@ +using BotSharp.Abstraction.Options; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class TextMessageFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-text_message"; + public string Indication => "Sending text message"; + + public TextMessageFn( + IServiceProvider services, + ILogger logger, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + // Send the message + var twilioMessage = MessageResource.Create( + to: args.PhoneNumber, + from: _twilioSetting.MessagingShortCode, + body: args.InitialMessage + ); + + if (twilioMessage.Status == MessageResource.StatusEnum.Queued) + { + message.Content = $"Queued message to {args.PhoneNumber}: {args.InitialMessage} [MESSAGING SID: {twilioMessage.Sid}]"; + message.StopCompletion = true; + } + else + { + message.Content = twilioMessage.ErrorMessage; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index 32639c9a..fa0b3179 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -6,15 +6,26 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; - private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}twilio_outbound_phone_call"; + private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; + public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; public void AddUtilities(List utilities) { var utility = new AgentUtility { Name = UtilityName.OutboundPhoneCall, - Functions = [new($"{OUTBOUND_PHONE_CALL_FN}")], - Templates = [new($"{OUTBOUND_PHONE_CALL_FN}.fn")] + Functions = + [ + new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{HANGUP_PHONE_CALL_FN}"), + new($"{TEXT_MESSAGE_FN}") + ], + Templates = + [ + new($"{OUTBOUND_PHONE_CALL_FN}.fn"), + new($"{HANGUP_PHONE_CALL_FN}.fn") + ] }; utilities.Add(utility); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs new file mode 100644 index 00000000..efd8114a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class HangupPhoneCallArgs +{ + [JsonPropertyName("anything_else_to_help")] + public bool AnythingElseToHelp { get; set; } = true; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index cde85aa9..3f7ea8b6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -5,8 +5,8 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class LlmContextIn { [JsonPropertyName("phone_number")] - public string PhoneNumber { get; set; } + public string PhoneNumber { get; set; } = null!; [JsonPropertyName("initial_message")] - public string InitialMessage { get; set; } + public string? InitialMessage { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 492b5dcf..35118ab3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; @@ -10,15 +9,17 @@ using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services.Stream; /// -/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js +/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js /// public class TwilioStreamMiddleware { private readonly RequestDelegate _next; + private readonly ILogger _logger; - public TwilioStreamMiddleware(RequestDelegate next) + public TwilioStreamMiddleware(RequestDelegate next, ILogger logger) { _next = next; + _logger = logger; } public async Task Invoke(HttpContext httpContext) @@ -32,7 +33,14 @@ public class TwilioStreamMiddleware var services = httpContext.RequestServices; var conversationId = request.Path.Value.Split("/").Last(); using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); - await HandleWebSocket(services, conversationId, webSocket); + try + { + await HandleWebSocket(services, conversationId, webSocket); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error in WebSocket communication: {ex.Message} for conversation {conversationId}"); + } return; } } @@ -43,12 +51,9 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); + var conn = hub.SetHubConnection(conversationId); + var completer = hub.SetCompleter("openai"); - var conn = new RealtimeHubConnection - { - ConversationId = conversationId - }; - // load conversation and state var convService = services.GetRequiredService(); convService.SetConversationId(conversationId, []); @@ -63,17 +68,44 @@ public class TwilioStreamMiddleware { var response = JsonSerializer.Deserialize(receivedText); conn.StreamId = response.StreamSid; - conn.Event = response.Event switch - { - "start" => "user_connected", - "media" => "user_data_received", - "stop" => "user_disconnected", - _ => response.Event - }; - if (string.IsNullOrEmpty(conn.Event)) + switch (response.Event) { - return conn; + case "start": + conn.Event = "user_connected"; + var startResponse = JsonSerializer.Deserialize(receivedText); + conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); + conn.ResetStreamState(); + break; + case "media": + conn.Event = "user_data_received"; + var mediaResponse = JsonSerializer.Deserialize(receivedText); + conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp); + conn.Data = mediaResponse.Body.Payload; + break; + case "stop": + conn.Event = "user_disconnected"; + break; + case "mark": + conn.Event = "mark"; + if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _); + break; + case "dtmf": + var dtmfResponse = JsonSerializer.Deserialize(receivedText); + if (dtmfResponse.Body.Digit == "#") + { + conn.Event = "user_dtmf_received"; + conn.Data = conn.KeypadInputBuffer; + conn.KeypadInputBuffer = string.Empty; + } + else + { + conn.KeypadInputBuffer += dtmfResponse.Body.Digit; + } + break; + default: + conn.Event = response.Event; + break; } conn.OnModelMessageReceived = message => @@ -96,19 +128,6 @@ public class TwilioStreamMiddleware @event = "clear", streamSid = response.StreamSid }; - - if (response.Event == "start") - { - var startResponse = JsonSerializer.Deserialize(receivedText); - conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); - } - else if (response.Event == "media") - { - var mediaResponse = JsonSerializer.Deserialize(receivedText); - conn.Data = mediaResponse.Body.Payload; - } - - return conn; }); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 2c942aa5..fb992c4a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -9,175 +9,178 @@ using System.Security.Claims; using System.Threading; using Task = System.Threading.Tasks.Task; -namespace BotSharp.Plugin.Twilio.Services +namespace BotSharp.Plugin.Twilio.Services; + +public class TwilioMessageQueueService : BackgroundService { - public class TwilioMessageQueueService : BackgroundService + private readonly TwilioMessageQueue _queue; + private readonly IServiceProvider _serviceProvider; + private readonly SemaphoreSlim _throttler; + private readonly ILogger _logger; + + public TwilioMessageQueueService( + TwilioMessageQueue queue, + IServiceProvider serviceProvider, + ILogger logger) { - private readonly TwilioMessageQueue _queue; - private readonly IServiceProvider _serviceProvider; - private readonly SemaphoreSlim _throttler; + _queue = queue; + _serviceProvider = serviceProvider; + _throttler = new SemaphoreSlim(10, 10); + _logger = logger; + } - public TwilioMessageQueueService( - TwilioMessageQueue queue, - IServiceProvider serviceProvider) + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken)) { - _queue = queue; - _serviceProvider = serviceProvider; - _throttler = new SemaphoreSlim(4, 4); - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken)) + await _throttler.WaitAsync(stoppingToken); + _ = Task.Run(async () => { - await _throttler.WaitAsync(stoppingToken); - _ = Task.Run(async () => + try { - try - { - Console.WriteLine($"Start processing {message}."); - await ProcessUserMessageAsync(message); - } - catch (Exception ex) - { - Console.WriteLine($"Processing {message} failed due to {ex.Message}."); - } - finally - { - _throttler.Release(); - } - }); - } - } - - public override async Task StopAsync(CancellationToken cancellationToken) - { - _queue.Stop(); - await base.StopAsync(cancellationToken); - } - - private async Task ProcessUserMessageAsync(CallerMessage message) - { - using var scope = _serviceProvider.CreateScope(); - var sp = scope.ServiceProvider; - - // Clean static HttpContext - var httpContext = sp.GetRequiredService(); - httpContext.HttpContext = new DefaultHttpContext(); - httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity()); - foreach (var header in message.RequestHeaders ?? []) - { - httpContext.HttpContext.Request.Headers[header.Key] = header.Value; - } - httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST"; - - AssistantMessage reply = null; - var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); - var conv = sp.GetRequiredService(); - var routing = sp.GetRequiredService(); - var config = sp.GetRequiredService(); - var sessionManager = sp.GetRequiredService(); - var progressService = sp.GetRequiredService(); - InitProgressService(message, sessionManager, progressService); - InitConversation(message, inputMsg, conv, routing); - - // Need to consider Inbound and Outbound call - var conversation = await conv.GetConversation(message.ConversationId); - var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId; - - var result = await conv.SendMessage(agentId, - inputMsg, - replyMessage: BuildPostbackMessageModel(conv, message), - async msg => - { - reply = new AssistantMessage() - { - ConversationEnd = msg.Instruction?.ConversationEnd ?? false, - HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), - Content = msg.Content, - MessageId = msg.MessageId - }; + _logger.LogInformation($"Start processing {message}."); + await ProcessUserMessageAsync(message); } - ); - reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp); - reply.Hints = GetHints(reply); - reply.Content = null; - await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); - } - - private PostbackMessageModel BuildPostbackMessageModel(IConversationService conv, CallerMessage message) - { - var messages = conv.GetDialogHistory(1); - if (!messages.Any()) return null; - var lastMessage = messages[0]; - if (string.IsNullOrEmpty(lastMessage.PostbackFunctionName)) return null; - return new PostbackMessageModel - { - FunctionName = lastMessage.PostbackFunctionName, - ParentId = lastMessage.MessageId, - Payload = message.Digits - }; - } - - private static void InitConversation(CallerMessage message, RoleDialogModel inputMsg, IConversationService conv, IRoutingService routing) - { - routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); - var states = new List - { - new("channel", ConversationChannel.Phone), - new("channel_id", message.From), - new("calling_phone", message.From) - }; - states.AddRange(message.States.Select(kvp => new MessageState(kvp.Key, kvp.Value))); - conv.SetConversationId(message.ConversationId, states); - } - - private static async Task GetReplySpeechFileName(string conversationId, AssistantMessage reply, IServiceProvider sp) - { - var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); - var fileStorage = sp.GetRequiredService(); - var data = await completion.GenerateAudioFromTextAsync(reply.Content); - var fileName = $"reply_{reply.MessageId}.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - return fileName; - } - - private static string GetHints(AssistantMessage reply) - { - var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries); - int capcity = 100; - var hints = new List(capcity); - for (int i = phrases.Length - 1; i >= 0; i--) - { - var words = phrases[i].Split(' ', StringSplitOptions.RemoveEmptyEntries); - for (int j = words.Length - 1; j >= 0; j--) + catch (Exception ex) { - hints.Add(words[j]); - if (hints.Count >= capcity) - { - break; - } + _logger.LogError($"Processing {message} failed due to {ex.Message}."); } + finally + { + _throttler.Release(); + } + }); + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Stop(); + await base.StopAsync(cancellationToken); + } + + private async Task ProcessUserMessageAsync(CallerMessage message) + { + using var scope = _serviceProvider.CreateScope(); + var sp = scope.ServiceProvider; + + // Clean static HttpContext + var httpContext = sp.GetRequiredService(); + httpContext.HttpContext = new DefaultHttpContext(); + httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity()); + foreach (var header in message.RequestHeaders ?? []) + { + httpContext.HttpContext.Request.Headers[header.Key] = header.Value; + } + httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST"; + + AssistantMessage reply = null; + + var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); + var conv = sp.GetRequiredService(); + var routing = sp.GetRequiredService(); + var config = sp.GetRequiredService(); + var sessionManager = sp.GetRequiredService(); + var progressService = sp.GetRequiredService(); + InitProgressService(message, sessionManager, progressService); + InitConversation(message, inputMsg, conv, routing); + + // Need to consider Inbound and Outbound call + var conversation = await conv.GetConversation(message.ConversationId); + var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId; + + var result = await conv.SendMessage(agentId, + inputMsg, + replyMessage: BuildPostbackMessageModel(conv, message), + async msg => + { + reply = new AssistantMessage() + { + ConversationEnd = msg.Instruction?.ConversationEnd ?? false, + HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), + Content = msg.Content, + MessageId = msg.MessageId + }; + } + ); + reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp); + reply.Hints = GetHints(reply); + reply.Content = null; + await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); + } + + private PostbackMessageModel BuildPostbackMessageModel(IConversationService conv, CallerMessage message) + { + var messages = conv.GetDialogHistory(1); + if (!messages.Any()) return null; + var lastMessage = messages[0]; + if (string.IsNullOrEmpty(lastMessage.PostbackFunctionName)) return null; + return new PostbackMessageModel + { + FunctionName = lastMessage.PostbackFunctionName, + ParentId = lastMessage.MessageId, + Payload = message.Digits + }; + } + + private static void InitConversation(CallerMessage message, RoleDialogModel inputMsg, IConversationService conv, IRoutingService routing) + { + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); + var states = new List + { + new("channel", ConversationChannel.Phone), + new("channel_id", message.From), + new("calling_phone", message.From) + }; + states.AddRange(message.States.Select(kvp => new MessageState(kvp.Key, kvp.Value))); + conv.SetConversationId(message.ConversationId, states); + } + + private static async Task GetReplySpeechFileName(string conversationId, AssistantMessage reply, IServiceProvider sp) + { + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); + var fileName = $"reply_{reply.MessageId}.mp3"; + fileStorage.SaveSpeechFile(conversationId, fileName, data); + return fileName; + } + + private static string GetHints(AssistantMessage reply) + { + var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries); + int capcity = 100; + var hints = new List(capcity); + for (int i = phrases.Length - 1; i >= 0; i--) + { + var words = phrases[i].Split(' ', StringSplitOptions.RemoveEmptyEntries); + for (int j = words.Length - 1; j >= 0; j--) + { + hints.Add(words[j]); if (hints.Count >= capcity) { break; } } - // add frequency short words - hints.AddRange(["yes", "no", "correct", "right"]); - return string.Join(", ", hints.Select(x => x.ToLower()).Distinct().Reverse()); - } - - private static void InitProgressService(CallerMessage message, ITwilioSessionManager sessionManager, IConversationProgressService progressService) - { - progressService.OnFunctionExecuting = async msg => + if (hints.Count >= capcity) { - if (!string.IsNullOrEmpty(msg.Indication)) - { - await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); - } - }; - progressService.OnFunctionExecuted = async msg => { }; + break; + } } + // add frequency short words + hints.AddRange(["yes", "no", "correct", "right"]); + return string.Join(", ", hints.Select(x => x.ToLower()).Distinct().Reverse()); + } + + private static void InitProgressService(CallerMessage message, ITwilioSessionManager sessionManager, IConversationProgressService progressService) + { + progressService.OnFunctionExecuting = async msg => + { + if (!string.IsNullOrEmpty(msg.Indication)) + { + await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); + } + }; + progressService.OnFunctionExecuted = async msg => { }; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 1ba7120c..082a0721 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; -using Twilio.TwiML.Messaging; using Token = Twilio.Jwt.AccessToken.Token; namespace BotSharp.Plugin.Twilio.Services; @@ -101,17 +100,25 @@ public class TwilioService return response; } - public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse) { var response = new VoiceResponse(); - response.Pause(2); - if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + + if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any()) { - foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + foreach (var speechPath in voiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } } } + var gather = new Gather() { Input = new List() @@ -119,14 +126,15 @@ public class TwilioService Gather.InputEnum.Speech, Gather.InputEnum.Dtmf }, - Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"), + Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"), Enhanced = true, SpeechModel = Gather.SpeechModelEnum.PhoneCall, SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3", - Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3, - ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult + Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3, + ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult, }; response.Append(gather); + return response; } @@ -185,13 +193,26 @@ public class TwilioService public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + if (speechPath.StartsWith("twilio/")) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}")); + } } } + var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}"); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 4c65481f..2de42ec8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -2,14 +2,37 @@ namespace BotSharp.Plugin.Twilio.Settings; public class TwilioSetting { - public string PhoneNumber { get; set; } + /// + /// Outbound phone number + /// + public string? PhoneNumber { get; set; } + + /// + /// Enable streaming for outbound phone call + /// + public bool StreamingEnabled { get; set; } = false; public string AccountSID { get; set; } public string AuthToken { get; set; } public string AppSID { get; set; } public string ApiKeySID { get; set; } public string ApiSecret { get; set; } public string CallbackHost { get; set; } - public string AgentId { get; set; } - public string CsrAgentNumber { get; set; } + + public string? MessagingShortCode { get; set; } + + /// + /// Default Agent Id to handle inbound phone call + /// + public string? AgentId { get; set; } + + /// + /// Human agent phone number if AI can't handle the call + /// + public string? CsrAgentNumber { get; set; } + public int MaxGatherAttempts { get; set; } = 4; + + public string? MachineDetection { get; set; } + + public bool RecordingEnabled { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json new file mode 100644 index 00000000..be5f3ca6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -0,0 +1,19 @@ +{ + "name": "util-twilio-hangup_phone_call", + "description": "Call this function if the user wants to end the phone call or conversation", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "reason": { + "type": "string", + "description": "The reason why user wants to end the phone call." + }, + "anything_else_to_help": { + "type": "boolean", + "description": "Check if user has anything else to help." + } + }, + "required": [ "reason", "anything_else_to_help" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json similarity index 76% rename from src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json rename to src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json index 768c6fde..a93ec3cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json @@ -1,15 +1,16 @@ { - "name": "util-twilio-twilio_outbound_phone_call", + "name": "util-twilio-outbound_phone_call", "description": "If the user wants to initiate a phone call, you need to capture the phone number and compose the message the users wants to send. Then call this function to make an outbound call via Twilio.", + "visibility_expression": "{% if states.channel != 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { "phone_number": { - "to_read": "string", + "type": "string", "description": "The phone number which will be dialed. It needs to be a valid phone number starting with +1." }, "initial_message": { - "to_read": "string", + "type": "string", "description": "The initial message which will be sent." } }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json new file mode 100644 index 00000000..a4340410 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json @@ -0,0 +1,18 @@ +{ + "name": "util-twilio-text_message", + "description": "If the user wants to send SMS message to a phone.", + "parameters": { + "type": "object", + "properties": { + "phone_number": { + "type": "string", + "description": "The phone number which will receive message. It needs to be a valid phone number starting with +1." + }, + "initial_message": { + "type": "string", + "description": "The initial message which will be sent." + } + }, + "required": [ "phone_number", "initial_message" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid new file mode 100644 index 00000000..5e89cbd2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -0,0 +1,3 @@ +{% if channel == 'phone' %} +** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call. +{% endif %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid new file mode 100644 index 00000000..d8fa9131 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -0,0 +1 @@ +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid deleted file mode 100644 index 3bfb7dc2..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please take a look at the conversation and decide whether user wants to make an outbound call. -** Please call util-twilio-twilio_outbound_phone_call if user wants to make an outbound call. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj index 4a513a97..b7491db8 100644 --- a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj +++ b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj @@ -27,8 +27,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index c5436fe6..2f391853 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 4c65aa32..283ad314 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,4 +1,5 @@ using Azure; +using BotSharp.Abstraction.Browsing.Settings; using System.IO; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -33,9 +34,10 @@ public class PlaywrightInstance : IDisposable public async Task GetContext(string ctxId) { + var _webDriver = _services.GetRequiredService(); if (!_contexts.ContainsKey(ctxId)) { - await InitContext(ctxId, new BrowserActionArgs()); + await InitContext(ctxId, new BrowserActionArgs() { Headless = _webDriver.Headless }); } return _contexts[ctxId]; } @@ -75,7 +77,8 @@ public class PlaywrightInstance : IDisposable Args = [ "--disable-infobars", - "--test-type" + "--test-type", + "--no-sandbox" // "--start-maximized" ] }); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 7327bb40..922e8f58 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -15,7 +15,14 @@ public partial class PlaywrightWebDriver if (page != null) { - if (page.Url != "about:blank") + if (!args.OpenBlankPage) + { + if (!_instance.Pages[message.ContextId].Contains(page)) + { + _instance.Pages[message.ContextId].Add(page); + } + } + if (args.OpenBlankPage && page.Url != "about:blank") { await page.EvaluateAsync(@"() => { window.open('', '_blank'); @@ -48,7 +55,6 @@ public partial class PlaywrightWebDriver // Active current tab await page.BringToFrontAsync(); - var response = await page.GotoAsync(args.Url, new PageGotoOptions { Timeout = args.Timeout > 0 ? args.Timeout : 30000 diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index 294c95db..b74af9c4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver @@ -5,19 +7,21 @@ public partial class PlaywrightWebDriver public async Task ScreenshotAsync(MessageInfo message, string path) { var result = new BrowserActionResult(); - - await _instance.Wait(message.ContextId, waitNetworkIdle: false); - var page = _instance.GetPage(message.ContextId); - - await Task.Delay(300); - var bytes = await page.ScreenshotAsync(new PageScreenshotOptions + var _webDriver = _services.GetRequiredService(); + if (_webDriver.IsEnableScreenshot) { - Path = path, - FullPage = true - }); + await _instance.Wait(message.ContextId, waitNetworkIdle: false); + var page = _instance.GetPage(message.ContextId); + await Task.Delay(300); + var bytes = await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = path, + FullPage = true + }); + result.Body = "data:image/png;base64," + Convert.ToBase64String(bytes); + } result.IsSuccess = true; - result.Body = "data:image/png;base64," + Convert.ToBase64String(bytes); return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index e1041ae6..fe69f9d1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Functions; public class GoToPageFn : IFunctionCallback @@ -25,7 +27,7 @@ public class GoToPageFn : IFunctionCallback var url = webDriverService.ReplaceToken(args.Url); url = url.Replace("https://https://", "https://"); - + var _webDriver = _services.GetRequiredService(); var result = await _browser.GoToPage(new MessageInfo { AgentId = message.CurrentAgentId, @@ -33,7 +35,8 @@ public class GoToPageFn : IFunctionCallback MessageId = message.MessageId }, new PageActionArgs { - Url = url + Url = url, + Timeout = _webDriver.DefaultTimeout }); message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.Message}"; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 6a5a1ac1..0beb80fc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Functions; public class OpenBrowserFn : IFunctionCallback @@ -6,12 +8,15 @@ public class OpenBrowserFn : IFunctionCallback private readonly IServiceProvider _services; private readonly IWebBrowser _browser; + private readonly WebBrowsingSettings _webBrowsingSettings; public OpenBrowserFn(IServiceProvider services, - IWebBrowser browser) + IWebBrowser browser, + WebBrowsingSettings webBrowsingSettings) { _services = services; _browser = browser; + _webBrowsingSettings = webBrowsingSettings; } public async Task Execute(RoleDialogModel message) @@ -21,7 +26,7 @@ public class OpenBrowserFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var url = webDriverService.ReplaceToken(args.Url); - + var _webDriver = _services.GetRequiredService(); url = url.Replace("https://https://", "https://"); var msgInfo = new MessageInfo { @@ -31,11 +36,12 @@ public class OpenBrowserFn : IFunctionCallback }; var result = await _browser.LaunchBrowser(msgInfo, new BrowserActionArgs { - Headless = false + Headless = _webBrowsingSettings.Headless }); result = await _browser.GoToPage(msgInfo, new PageActionArgs { - Url = url + Url = url, + Timeout = _webDriver.DefaultTimeout }); if (result.IsSuccess) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index 675e1ec8..7a5d8f89 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.UtilFunctions; public class UtilWebGoToPageFn : IFunctionCallback @@ -21,9 +23,10 @@ public class UtilWebGoToPageFn : IFunctionCallback { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var _webDriver = _services.GetRequiredService(); + args.Timeout = _webDriver.DefaultTimeout; args.WaitForNetworkIdle = false; - args.WaitTime = 5; - args.OpenNewTab = true; + args.WaitTime = _webDriver.DefaultWaitTime; var conv = _services.GetRequiredService(); @@ -34,7 +37,10 @@ public class UtilWebGoToPageFn : IFunctionCallback MessageId = message.MessageId, ContextId = message.CurrentAgentId, }; - await browser.CloseCurrentPage(msg); + if (!args.KeepBrowserOpen) + { + await browser.CloseCurrentPage(msg); + } var result = await browser.GoToPage(msg, args); if (!result.IsSuccess) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json index 65f088bd..4f6b21fe 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json @@ -7,6 +7,14 @@ "url": { "type": "string", "description": "Web URL" + }, + "open_new_tab": { + "type": "boolean", + "description": "Open new tab" + }, + "open_blank_page": { + "type": "boolean", + "description": "Open blank page" } }, "required": [ "url" ] diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index b2fcd816..78c4537c 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -9,21 +9,21 @@ - - - - - - - - - - + + + + + + + + + + - - + + @@ -36,6 +36,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index a0f02893..9d8afd80 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -222,6 +222,10 @@ "Enabled": false }, + "Instruction": { + "EnableLog": true + }, + "ChatHub": { "EventDispatchBy": "group" }, @@ -391,6 +395,7 @@ "BotSharp.Core", "BotSharp.Core.SideCar", "BotSharp.Core.Crontab", + "BotSharp.Core.Realtime", "BotSharp.Logger", "BotSharp.Plugin.MongoStorage", "BotSharp.Plugin.Dashboard", diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json index 94746011..3a28b3e0 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json @@ -1,7 +1,7 @@ { "id": "8970b1e5-d260-4e2c-90b1-f1415a257c18", "name": "Pizza Bot", - "description": "AI assistant that can help customer place pizza order.", + "description": "AI assistant that can help customer place pizza order, make payment or inquiry order status.", "type": "routing", "inheritAgentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", "createdDateTime": "2023-08-18T10:39:32.2349685Z", @@ -10,10 +10,11 @@ "disabled": false, "isPublic": true, "profiles": [ "pizza" ], + "labels": [ "experiment" ], "routingRules": [ { "type": "reasoner", - "field": "NaiveReasoner" + "field": "Naive Reasoner" } ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json index 4f606251..09f7489b 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json @@ -6,5 +6,6 @@ "id": "b284db86-e9c2-4c25-a59e-4649797dd130", "disabled": false, "isPublic": true, - "profiles": [ "pizza" ] + "profiles": [ "pizza" ], + "labels": [ "experiment" ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json index 62f069bb..40988cb2 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json @@ -1,10 +1,11 @@ { "name": "Ordering", - "description": "Provide types of pizza available, unit price and total cost. Place the order and returned the order number.", + "description": "Provide types of pizza available, unit price, total cost and place the order.", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd", "disabled": false, "isPublic": true, - "profiles": [ "pizza" ] + "profiles": [ "pizza" ], + "labels": [ "experiment" ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json index 1dab7257..ef47f34d 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json @@ -1,12 +1,13 @@ { "name": "Payment", - "description": "Make payment when user wants to pay for the order", + "description": "Make payment when user confirmed the price and going to pay for the order", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "fe8c60aa-b114-4ef3-93cb-a8efeac80f75", "disabled": false, "isPublic": true, "profiles": [ "pizza" ], + "labels": [ "experiment" ], "routingRules": [ { "field": "order_number", diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index c47fcead..c088a873 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -4,22 +4,20 @@ $(TargetFramework) enable enable - false - - - - - - - + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/UnitTest/UnitTest.csproj b/tests/UnitTest/UnitTest.csproj index 67e7d504..f7686736 100644 --- a/tests/UnitTest/UnitTest.csproj +++ b/tests/UnitTest/UnitTest.csproj @@ -4,18 +4,17 @@ $(TargetFramework) enable enable - false true - - - - - - + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive