From fec696493bfab8f6e29992bcbe49a2cce2971f8a Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 13 Feb 2025 13:59:57 -0600 Subject: [PATCH 01/84] enable all ValidateRequest --- .../Controllers/TwilioVoiceController.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 19144673..7b041ca6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -108,7 +108,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { @@ -202,7 +202,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { @@ -367,7 +367,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/init-call")] public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) { @@ -388,7 +388,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { From fab5a17b75f3af7687fa144640b678a3be4d79b4 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 22 Feb 2025 08:55:47 -0800 Subject: [PATCH 02/84] more projects on ManagePackageVersionsCentrally --- Directory.Packages.props | 14 ++++++++++++++ .../BotSharp.Core.Crontab.csproj | 3 ++- .../BotSharp.Plugin.AzureOpenAI.csproj | 5 +++-- .../BotSharp.Plugin.DeepSeekAI.csproj | 3 ++- .../BotSharp.Plugin.EmailHandler.csproj | 3 ++- .../BotSharp.Plugin.ExcelHandler.csproj | 7 ++++--- .../BotSharp.Plugin.GoogleAI.csproj | 5 +++-- .../BotSharp.Plugin.HuggingFace.csproj | 5 +++-- .../BotSharp.Plugin.JavaScriptInterpreter.csproj | 3 ++- .../BotSharp.Plugin.KnowledgeBase.csproj | 5 +++-- 10 files changed, 38 insertions(+), 15 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a031cec3..55683808 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -33,6 +33,20 @@ + + + + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj index ae4c9fef..b4bcaf84 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj +++ b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj @@ -5,6 +5,7 @@ $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) + true $(SolutionDir)packages enable enable @@ -29,7 +30,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 284e9b3c..61758e2e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 56eb14ef..598e1295 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f1a6c28e..a7f4edcf 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -33,7 +34,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index 08cfd79a..3b6db5d3 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -4,6 +4,7 @@ net8.0 enable enable + true @@ -27,9 +28,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index b6e0b24e..9e153d76 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj index ae6a05be..2b8854af 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj index 4596ce1b..6abacd91 100644 --- a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj @@ -4,10 +4,11 @@ net8.0 enable enable + true - + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 77a5d076..b7cf3059 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -55,8 +56,8 @@ - - + + From 15c5fc3a5eb5a63ae4acbe31c7e1af794fd972fa Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 22 Feb 2025 09:08:59 -0800 Subject: [PATCH 03/84] more projects on ManagePackageVersionsCentrally --- Directory.Packages.props | 4 ++++ .../BotSharp.Plugin.LLamaSharp.csproj | 3 ++- .../BotSharp.Plugin.VertexAI.csproj | 3 ++- .../BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj | 7 ++++--- .../BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 55683808..1cee14e9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -47,6 +47,10 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index d0bea79a..d5581998 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj index 004d7233..92509518 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj +++ b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 3d390f56..2c247292 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -15,8 +16,8 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj index 2e43183c..d1bcb1b4 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + From 3006b468dbb9ef11f4bd95d36c24c69492bfa67c Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 22 Feb 2025 09:20:51 -0800 Subject: [PATCH 04/84] bump up version of System.IdentityModel.Tokens.Jwt --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 1cee14e9..f6b1545e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,7 +13,7 @@ - + From 8e668b86279f54aef810d3cd56f7affb47e9ef3e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 24 Feb 2025 12:14:25 -0600 Subject: [PATCH 05/84] Add document type of knowledge retrieval --- .../BotSharp.Plugin.KnowledgeBase.csproj | 6 ++---- .../Enum/UtilityName.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Hooks/KnowledgeBaseUtilityHook.cs | 2 +- .../Hooks/KnowledgeHook.cs | 14 ++++++++++++++ ...ieval.json => util-kg-knowledge_retrieval.json} | 2 +- ...iquid => util-kg-knowledge_retrieval.fn.liquid} | 2 +- 7 files changed, 21 insertions(+), 9 deletions(-) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-knowledge-knowledge_retrieval.json => util-kg-knowledge_retrieval.json} (86%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/{util-knowledge-knowledge_retrieval.fn.liquid => util-kg-knowledge_retrieval.fn.liquid} (63%) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 77a5d076..69537fba 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -23,8 +23,6 @@ - - @@ -46,10 +44,10 @@ PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs index de90430d..e4518344 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs @@ -2,5 +2,5 @@ namespace BotSharp.Plugin.KnowledgeBase.Enum; public class UtilityName { - public const string KnowledgeRetrieval = "knowledge.knowledge-retrieval"; + public const string KnowledgeRetrieval = "kg.knowledge-base"; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f3f8cbf1..940c3834 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { - public string Name => "util-knowledge-knowledge_retrieval"; + public string Name => "util-kg-knowledge_retrieval"; public string Indication => "searching my brain"; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index 700b8457..84254841 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook { - private static string PREFIX = "util-knowledge-"; + private static string PREFIX = "util-kg-"; private static string KNOWLEDGE_RETRIEVAL_FN = $"{PREFIX}knowledge_retrieval"; public void AddUtilities(List utilities) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index da4add53..90aa0114 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -40,6 +40,20 @@ public class KnowledgeHook : IKnowledgeHook var result = await _knowledgeService.SearchGraphKnowledge(text, options); results.Add(result.Result); } + else if (knowledgeBase.Type == "document") + { + var options = new VectorSearchOptions + { + Fields = null, + Limit = 5, + Confidence = 0.25f, + WithVector = true + }; + var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) + .Select(x => x.Data["text"].ToString()) + .Where(x => x != null)!); + } else { var options = new VectorSearchOptions diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json similarity index 86% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json index 088f4ea6..cb79298a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json @@ -1,5 +1,5 @@ { - "name": "util-knowledge-knowledge_retrieval", + "name": "util-kg-knowledge_retrieval", "description": "Retrieve related domain knowledge to handle user request", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid similarity index 63% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid index f0bc60c8..e7d61b7f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid @@ -1,3 +1,3 @@ -Call function util-knowledge-knowledge_retrieval to retrieve related domain knowledge to handle user request. +Call function util-kg-knowledge_retrieval to retrieve related domain knowledge to handle user request. You must retrieve existing KnowledgeBase to get prerequisite informations before you writing SQL query; You must retrieve existing API specification from KnowledgeBase before calling a Web API; From 51d933e3db1616f50b7266f33f5aa4a62ead54b1 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 24 Feb 2025 22:35:22 -0600 Subject: [PATCH 06/84] lazy routing (WIP) --- .../Infrastructures/Enums/StateConst.cs | 3 ++ .../Routing/IRoutingContext.cs | 8 ++-- .../Routing/Models/FallbackArgs.cs | 10 +++++ .../BotSharp.Core/BotSharp.Core.csproj | 14 ++++++- .../ConversationService.SendMessage.cs | 15 +++++++- .../Routing/Functions/FallbackToRouterFn.cs | 29 +++------------ .../Routing/Hooks/RoutingUtilityHook.cs | 20 ++++++++++ .../Routing/Reasoning/NaiveReasoner.cs | 2 +- .../BotSharp.Core/Routing/RoutingContext.cs | 37 ++++++++++++++++--- .../BotSharp.Core/Routing/RoutingPlugin.cs | 2 + .../Routing/RoutingService.InvokeAgent.cs | 3 +- .../functions/route_to_agent.json | 31 ++++++++++++++++ .../instructions/instruction.liquid | 2 + .../util-routing-fallback_to_router.json | 18 +++++++++ .../util-routing-fallback_to_router.fn.liquid | 1 + 15 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..f2e0450a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -8,6 +8,9 @@ public class StateConst public const string NEXT_ACTION_REASON = "next_action_reason"; public const string USER_GOAL_AGENT = "user_goal_agent"; public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; + // lazy or eager + public const string ROUTING_MODE = "routing_mode"; + public const string LAZY_ROUTING_AGENT_ID = "lazy_routing_agent_id"; public const string LANGUAGE = "language"; diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs index da7d5c01..337d63c6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs @@ -13,10 +13,10 @@ public interface IRoutingContext bool IsEmpty { get; } string IntentName { get; set; } int AgentCount { get; } - void Push(string agentId, string? reason = null); - void Pop(string? reason = null); - void PopTo(string agentId, string reason); - void Replace(string agentId, string? reason = null); + void Push(string agentId, string? reason = null, bool updateLazyRouting = true); + void Pop(string? reason = null, bool updateLazyRouting = true); + void PopTo(string agentId, string reason, bool updateLazyRouting = true); + void Replace(string agentId, string? reason = null, bool updateLazyRouting = true); void Empty(string? reason = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs new file mode 100644 index 00000000..6fc2ce03 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Routing.Models; + +public class FallbackArgs +{ + [JsonPropertyName("fallback_reason")] + public string Reason { get; set; } = null!; + + [JsonPropertyName("user_question")] + public string Question { get; set; } = null; +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6bc7596c..c8ded3cd 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -66,6 +66,8 @@ + + @@ -82,6 +84,7 @@ + @@ -146,6 +149,15 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index f60d32cf..3967aaaa 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Routing.Settings; @@ -36,7 +37,17 @@ public partial class ConversationService // Enqueue receiving agent first in case it stop completion by OnMessageReceived var routing = _services.GetRequiredService(); routing.Context.SetMessageId(_conversationId, message.MessageId); - routing.Context.Push(agent.Id, reason: "request started"); + + // Check the routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false); + + if (routingMode == "lazy") + { + message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId); + routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false); + } // Save payload in order to assign the payload before hook is invoked if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) @@ -77,7 +88,7 @@ public partial class ConversationService { agent = await agentService.LoadAgent(message.CurrentAgentId); } - + if (agent.Type == AgentType.Routing) { response = await routing.InstructLoop(message, dialogs); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs index 592a2c29..58e43819 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs @@ -5,8 +5,9 @@ namespace BotSharp.Core.Routing.Functions; public class FallbackToRouterFn : IFunctionCallback { - public string Name => "fallback_to_router"; + public string Name => "util-routing-fallback_to_router"; private readonly IServiceProvider _services; + public FallbackToRouterFn(IServiceProvider services) { _services = services; @@ -14,30 +15,10 @@ public class FallbackToRouterFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - var agentService = _services.GetRequiredService(); - var agents = await agentService.GetAgents(new AgentFilter - { - AgentNames = [args.AgentName] - }); - var targetAgent = agents.Items.FirstOrDefault(); - if (targetAgent == null) - { - message.Content = $"Can't find routing agent {args.AgentName}"; - return false; - } - - var conv = _services.GetRequiredService(); - var dialogs = conv.GetDialogHistory(); - + var args = JsonSerializer.Deserialize(message.FunctionArgs); var routing = _services.GetRequiredService(); - routing.Context.Replace(targetAgent.Id); - message.CurrentAgentId = targetAgent.Id; - - var response = await routing.InstructLoop(message, dialogs); - - message.Content = response.Content; - message.StopCompletion = true; + routing.Context.PopTo(routing.Context.EntryAgentId, "pop to entry agent"); + message.Content = args.Question; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs new file mode 100644 index 00000000..e6fedc05 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Core.Routing.Hooks; + +public class RoutingUtilityHook : IAgentUtilityHook +{ + private static string PREFIX = "util-routing-"; + private static string REDIRECT_TO_AGENT = $"{PREFIX}redirect_to_agent"; + private static string FALLBACK_TO_ROUTER = $"{PREFIX}fallback_to_router"; + + public void AddUtilities(List utilities) + { + var utility = new AgentUtility + { + Name = "routing.tools", + Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")], + Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")] + }; + + utilities.Add(utility); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs index 931b53e8..8980246e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs @@ -73,7 +73,7 @@ public class NaiveReasoner : IRoutingReasoner }; var response = await completion.GetChatCompletions(router, dialogs); - inst = response.Content.JsonContent(); + inst = (response.FunctionArgs ?? response.Content).JsonContent(); break; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index cf604c55..47e47631 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Routing.Settings; namespace BotSharp.Core.Routing; @@ -79,7 +80,7 @@ public class RoutingContext : IRoutingContext /// /// Id or Name /// - public void Push(string agentId, string? reason = null) + public void Push(string agentId, string? reason = null, bool updateLazyRouting = true) { // Convert id to name if (!Guid.TryParse(agentId, out _)) @@ -99,13 +100,15 @@ public class RoutingContext : IRoutingContext HookEmitter.Emit(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason) ).Wait(); + + UpdateLazyRoutingAgent(updateLazyRouting); } } /// /// Pop current agent /// - public void Pop(string? reason = null) + public void Pop(string? reason = null, bool updateLazyRouting = true) { if (_stack.Count == 0) { @@ -149,15 +152,17 @@ public class RoutingContext : IRoutingContext _stack.Push(agentId); } } + + UpdateLazyRoutingAgent(updateLazyRouting); } - public void PopTo(string agentId, string reason) + public void PopTo(string agentId, string reason, bool updateLazyRouting = true) { var currentAgentId = GetCurrentAgentId(); while (!string.IsNullOrEmpty(currentAgentId) && currentAgentId != agentId) { - Pop(reason); + Pop(reason, updateLazyRouting: updateLazyRouting); currentAgentId = GetCurrentAgentId(); } } @@ -181,7 +186,7 @@ public class RoutingContext : IRoutingContext return _stack.ToArray().Contains(agentId); } - public void Replace(string agentId, string? reason = null) + public void Replace(string agentId, string? reason = null, bool updateLazyRouting = true) { var fromAgent = agentId; var toAgent = agentId; @@ -200,6 +205,8 @@ public class RoutingContext : IRoutingContext await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason) ).Wait(); } + + UpdateLazyRoutingAgent(updateLazyRouting); } public void Empty(string? reason = null) @@ -275,4 +282,24 @@ public class RoutingContext : IRoutingContext { _dialogs = []; } + + private void UpdateLazyRoutingAgent(bool updateLazyRouting) + { + if (!updateLazyRouting) + { + return; + } + + // Set next handling agent for lazy routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + if (routingMode == "lazy") + { + var agentId = GetCurrentAgentId(); + if (agentId != BuiltInAgentId.Fallback) + { + states.SetState(StateConst.LAZY_ROUTING_AGENT_ID, agentId); + } + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs index c5317ba4..2e409fcf 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs @@ -37,5 +37,7 @@ public class RoutingPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index a4d4b37a..273fc996 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -53,7 +53,8 @@ public partial class RoutingService // Handle output routing exception. if (agent.Type == AgentType.Routing) { - response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; + // Forgot about what situation needs to handle in this way + // response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; } message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json new file mode 100644 index 00000000..31c8400f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -0,0 +1,31 @@ +{ + "name": "route_to_agent", + "description": "Route request to appropriate AI agent.", + "visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "next_action_agent": { + "type": "string", + "description": "Agent for next action based on user latest response" + }, + "next_action_reason": { + "type": "string", + "description": "The reason why route to this agent." + }, + "user_goal_agent": { + "type": "string", + "description": "Agent who can acheive user initial task." + }, + "conversation_end": { + "type": "boolean", + "description": "User is ending the conversation." + }, + "args": { + "type": "object", + "description": "Required parameters of next action agent" + } + }, + "required": [ "next_action_agent", "user_goal_agent", "args" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index bc614f81..997012db 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -14,6 +14,7 @@ Follow these steps to handle user request: {%- endfor %} {% endif %} +{% if routing_mode != 'lazy' %} [FUNCTIONS] {% for handler in routing_handlers -%} # {{ handler.description}} @@ -26,6 +27,7 @@ Parameters: {%- endif %} {{ "\r\n" }} {%- endfor %} +{% endif %} [AGENTS] {% for agent in routing_agents -%} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json new file mode 100644 index 00000000..cd9549c3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -0,0 +1,18 @@ +{ + "name": "util-routing-fallback_to_router", + "description": "Get the appropriate agent who can handle the user request.", + "parameters": { + "type": "object", + "properties": { + "fallback_reason": { + "type": "string", + "description": "The reason why you need to reach out to other agent." + }, + "user_question": { + "type": "string", + "description": "User question or statement." + } + }, + "required": [ "user_question" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid new file mode 100644 index 00000000..ab3863f8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid @@ -0,0 +1 @@ +"If you're unsure whether you understand the user's request or if the user brings up an unrelated topic, call the function `util-routing-fallback_to_router` to get the appropriate agent from the router." \ No newline at end of file From 2428349247fff8b80e085abe4d16bf3d90a64f52 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 25 Feb 2025 19:50:11 -0600 Subject: [PATCH 07/84] remove limit --- .../FileRepository/FileRepository.Conversation.cs | 10 ---------- .../Repository/MongoRepository.Conversation.cs | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index ec4848d7..7bd1dd15 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -500,16 +500,6 @@ public partial class FileRepository batchSize = batchLimit; } - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - foreach (var d in Directory.GetDirectories(dir)) { var convFile = Path.Combine(d, CONVERSATION_FILE); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 06512fb7..a78db60c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -466,16 +466,6 @@ public partial class MongoRepository batchSize = batchLimit; } - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - while (true) { var skip = (page - 1) * batchSize; From e68f245f2a1d82fd51ffbc2ec9e41c061e93ddc2 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 11:41:52 -0600 Subject: [PATCH 08/84] Add Twilio HangupPhoneCallFn --- .../Infrastructures/Enums/StateConst.cs | 1 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 3 +- .../Realtime/RealTimeCompletionProvider.cs | 10 +- .../BotSharp.Plugin.Twilio.csproj | 15 ++- .../Enums/UtilityName.cs | 2 +- .../Functions/HandleOutboundPhoneCallFn.cs | 106 --------------- .../Functions/HangupPhoneCallFn.cs | 35 +++++ .../Functions/OutboundPhoneCallFn.cs | 125 ++++++++++++++++++ .../OutboundPhoneCallHandlerUtilityHook.cs | 15 ++- .../LlmContexts/LlmContextIn.cs | 3 + .../util-twilio-hangup_phone_call.json | 15 +++ ...n => util-twilio-outbound_phone_call.json} | 7 +- .../util-twilio-hangup_phone_call.fn.liquid | 1 + .../util-twilio-outbound_phone_call.fn.liquid | 2 + ...wilio-twilio_outbound_phone_call.fn.liquid | 2 - 15 files changed, 218 insertions(+), 124 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json rename src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-twilio-twilio_outbound_phone_call.json => util-twilio-outbound_phone_call.json} (76%) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..a21271df 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -12,4 +12,5 @@ public class StateConst public const string LANGUAGE = "language"; public const string SUB_CONVERSATION_ID = "sub_conversation_id"; + public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index c44f74d0..150959bb 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -4,6 +4,7 @@ using System; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Enums; namespace BotSharp.Core.Realtime; @@ -118,7 +119,7 @@ public class RealtimeHub : IRealtimeHub foreach (var message in messages) { // Invoke function - if (message.MessageType == "function_call") + if (message.MessageType == MessageTypeName.FunctionCall) { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 9b981cb1..efa96547 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; @@ -204,12 +205,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { + if (_webSocket.State != WebSocketState.Open) + { + return; + } + if (message is not string data) { data = JsonSerializer.Serialize(message); } var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } @@ -559,7 +566,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion CurrentAgentId = conn.EntryAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, - ToolCallId = output.CallId + ToolCallId = output.CallId, + MessageType = MessageTypeName.FunctionCall }); } else if (output.Type == "message") diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 91c81e90..ece4e6f6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -9,15 +9,16 @@ - - - - - - + PreserveNewest - + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs index 63252a57..b7244e7d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs @@ -2,6 +2,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Enums { public class UtilityName { - public const string OutboundPhoneCall = "twilio.twilio-outbound-phone-call"; + public const string OutboundPhoneCall = "phone.twilio-phone-call"; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs deleted file mode 100644 index 12b827af..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ /dev/null @@ -1,106 +0,0 @@ -using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Abstraction.Options; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.Twilio.Interfaces; -using BotSharp.Plugin.Twilio.Models; -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; -using Twilio.Rest.Api.V2010.Account; -using Twilio.Types; - -namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions -{ - public class HandleOutboundPhoneCallFn : IFunctionCallback - { - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly BotSharpOptions _options; - private readonly TwilioSetting _twilioSetting; - - public string Name => "util-twilio-twilio_outbound_phone_call"; - public string Indication => "Dialing the number"; - - public HandleOutboundPhoneCallFn( - IServiceProvider services, - ILogger logger, - BotSharpOptions options, - TwilioSetting twilioSetting) - { - _services = services; - _logger = logger; - _options = options; - _twilioSetting = twilioSetting; - } - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); - if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) - { - var error = $"Invalid phone number format: {args.PhoneNumber}"; - _logger.LogError(error); - message.Content = error; - return false; - } - - if (string.IsNullOrWhiteSpace(args.InitialMessage)) - { - _logger.LogError("Initial message is empty."); - message.Content = "There is an error when generating phone message."; - return false; - } - - var convService = _services.GetRequiredService(); - var convStorage = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var sessionManager = _services.GetRequiredService(); - var states = _services.GetRequiredService(); - - // Fork conversation - var entryAgentId = routing.EntryAgentId; - var newConv = await convService.NewConversation(new Abstraction.Conversations.Models.Conversation - { - AgentId = entryAgentId, - Channel = ConversationChannel.Phone - }); - var conversationId = newConv.Id; - convStorage.Append(conversationId, new List - { - new RoleDialogModel(AgentRole.User, "Hi") - { - CurrentAgentId = entryAgentId - }, - new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) - { - CurrentAgentId = entryAgentId - } - }); - states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); - - // Generate audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - - // Call phone number - /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage - { - Content = args.InitialMessage, - SpeechFileName = fileName - });*/ - - var call = await CallResource.CreateAsync( - // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"), - to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber)); - - message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; - message.StopCompletion = true; - return true; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs new file mode 100644 index 00000000..b4c7bca3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -0,0 +1,35 @@ +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class HangupPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public string Name => "util-twilio-hangup_phone_call"; + public string Indication => "Hangup"; + + public HangupPhoneCallFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + // Have to find the SID by the phone number + var call = CallResource.Update( + status: CallResource.UpdateStatusEnum.Completed, + pathSid: args.CallSid + ); + + message.Content = "The call has ended."; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs new file mode 100644 index 00000000..bc9e6203 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -0,0 +1,125 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; +using Twilio.Types; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class OutboundPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-outbound_phone_call"; + public string Indication => "Dialing the phone number"; + + public OutboundPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) + { + var error = $"Invalid phone number format: {args.PhoneNumber}"; + _logger.LogError(error); + message.Content = error; + return false; + } + + if (string.IsNullOrWhiteSpace(args.InitialMessage)) + { + _logger.LogError("Initial message is empty."); + message.Content = "There is an error when generating phone message."; + return false; + } + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + + // Fork conversation + var newConversationId = Guid.NewGuid().ToString(); + states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + + // Generate initial assistant audio + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + var fileName = $"intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, fileName, data); + + // Make outbound call + var call = await CallResource.CreateAsync( + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={newConversationId}&init_audio_file={fileName}"), + to: new PhoneNumber(args.PhoneNumber), + from: new PhoneNumber(_twilioSetting.PhoneNumber)); + + var convService = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var originConversationId = convService.ConversationId; + var entryAgentId = routing.EntryAgentId; + + await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); + + message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]"; + message.StopCompletion = true; + return true; + } + + private async Task ForkConversation(LlmContextIn args, + string entryAgentId, + string originConversationId, + string newConversationId, + CallResource resource) + { + // new scope service for isolated conversation + using var scope = _services.CreateScope(); + var services = scope.ServiceProvider; + var convService = services.GetRequiredService(); + var convStorage = services.GetRequiredService(); + + var newConv = await convService.NewConversation(new Conversation + { + Id = newConversationId, + AgentId = entryAgentId, + Channel = ConversationChannel.Phone, + ChannelId = resource.Sid, + Title = args.InitialMessage + }); + + convStorage.Append(newConversationId, new List + { + new RoleDialogModel(AgentRole.User, $"[Calling Phone To: {resource.ToFormatted}, Call SID: {resource.Sid}.]") + { + CurrentAgentId = entryAgentId + }, + new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) + { + CurrentAgentId = entryAgentId + } + }); + + convService.SetConversationId(newConversationId, + [ + new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId), + new MessageState("phone_number", resource.To) + ]); + convService.SaveStates(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index 32639c9a..692610ff 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -6,15 +6,24 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; - private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}twilio_outbound_phone_call"; + private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; public void AddUtilities(List utilities) { var utility = new AgentUtility { Name = UtilityName.OutboundPhoneCall, - Functions = [new($"{OUTBOUND_PHONE_CALL_FN}")], - Templates = [new($"{OUTBOUND_PHONE_CALL_FN}.fn")] + Functions = + [ + new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{HANGUP_PHONE_CALL_FN}") + ], + Templates = + [ + new($"{OUTBOUND_PHONE_CALL_FN}.fn"), + new($"{HANGUP_PHONE_CALL_FN}.fn") + ] }; utilities.Add(utility); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index cde85aa9..b56db84e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -9,4 +9,7 @@ public class LlmContextIn [JsonPropertyName("initial_message")] public string InitialMessage { get; set; } + + [JsonPropertyName("call_sid")] + public string CallSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json new file mode 100644 index 00000000..63dd4327 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -0,0 +1,15 @@ +{ + "name": "util-twilio-hangup_phone_call", + "description": "Call this function if the user wants to end the phone call", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "call_sid": { + "type": "string", + "description": "The unique string SID that we created to identify this Call resource." + } + }, + "required": [ "call_sid" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json similarity index 76% rename from src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json rename to src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json index 768c6fde..a93ec3cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json @@ -1,15 +1,16 @@ { - "name": "util-twilio-twilio_outbound_phone_call", + "name": "util-twilio-outbound_phone_call", "description": "If the user wants to initiate a phone call, you need to capture the phone number and compose the message the users wants to send. Then call this function to make an outbound call via Twilio.", + "visibility_expression": "{% if states.channel != 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { "phone_number": { - "to_read": "string", + "type": "string", "description": "The phone number which will be dialed. It needs to be a valid phone number starting with +1." }, "initial_message": { - "to_read": "string", + "type": "string", "description": "The initial message which will be sent." } }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid new file mode 100644 index 00000000..cc8360be --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -0,0 +1 @@ +** Please call util-twilio-hangup_phone_call if user wants to end the phone call. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid new file mode 100644 index 00000000..ada72fb9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -0,0 +1,2 @@ +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. +** For outbound calls, phone number format is "XXX-XXX-XXXX". \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid deleted file mode 100644 index 3bfb7dc2..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please take a look at the conversation and decide whether user wants to make an outbound call. -** Please call util-twilio-twilio_outbound_phone_call if user wants to make an outbound call. \ No newline at end of file From 1468c01f4671f48825e2bc8019e7c415899dc087 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 13:16:57 -0600 Subject: [PATCH 09/84] Remove call_sid from function util-twilio-hangup_phone_call --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 2 +- .../Controllers/TwilioStreamController.cs | 4 +++- .../Functions/HangupPhoneCallFn.cs | 15 ++++++++++++--- .../Functions/OutboundPhoneCallFn.cs | 2 +- .../LlmContexts/LlmContextIn.cs | 3 --- .../functions/util-twilio-hangup_phone_call.json | 6 +----- .../util-twilio-outbound_phone_call.fn.liquid | 3 +-- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 150959bb..51c1bd33 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -139,7 +139,7 @@ public class RealtimeHub : IRealtimeHub if (!string.IsNullOrEmpty(message.Content)) { - await hook.OnMessageReceived(message); + await hook.OnResponseGenerated(message); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e6de9fc5..774d014b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -93,6 +93,7 @@ public class TwilioStreamController : TwilioController Id = request.CallSid, AgentId = _settings.AgentId, Channel = ConversationChannel.Phone, + ChannelId = request.CallSid, Title = $"Phone call from {request.From}", Tags = [], }; @@ -103,7 +104,8 @@ public class TwilioStreamController : TwilioController var states = new List { new("channel", ConversationChannel.Phone), - new("calling_phone", request.From) + new("calling_phone", request.From), + new("twilio_call_sid", request.CallSid), }; convService.SetConversationId(conversation.Id, states); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index b4c7bca3..f54c1b42 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -21,14 +20,24 @@ public class HangupPhoneCallFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); + var states = _services.GetRequiredService(); + var callSid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(callSid)) + { + message.Content = "The call has not been initiated."; + _logger.LogError(message.Content); + return false; + } + // Have to find the SID by the phone number var call = CallResource.Update( status: CallResource.UpdateStatusEnum.Completed, - pathSid: args.CallSid + pathSid: callSid ); message.Content = "The call has ended."; + message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index bc9e6203..df97cd44 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -105,7 +105,7 @@ public class OutboundPhoneCallFn : IFunctionCallback convStorage.Append(newConversationId, new List { - new RoleDialogModel(AgentRole.User, $"[Calling Phone To: {resource.ToFormatted}, Call SID: {resource.Sid}.]") + new RoleDialogModel(AgentRole.User, "Hi") { CurrentAgentId = entryAgentId }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index b56db84e..cde85aa9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -9,7 +9,4 @@ public class LlmContextIn [JsonPropertyName("initial_message")] public string InitialMessage { get; set; } - - [JsonPropertyName("call_sid")] - public string CallSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 63dd4327..76b0e841 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -5,11 +5,7 @@ "parameters": { "type": "object", "properties": { - "call_sid": { - "type": "string", - "description": "The unique string SID that we created to identify this Call resource." - } }, - "required": [ "call_sid" ] + "required": [] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid index ada72fb9..d8fa9131 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -1,2 +1 @@ -** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. -** For outbound calls, phone number format is "XXX-XXX-XXXX". \ No newline at end of file +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. \ No newline at end of file From 9d570311d10538b721cd7eb5b23636488a18b075 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 22:40:50 -0600 Subject: [PATCH 10/84] realtime route to agent --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 23 ++++++++++++++----- .../Realtime/RealTimeCompletionProvider.cs | 4 ++-- .../Controllers/TwilioStreamController.cs | 18 +++++++-------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 8e1d11d5..bd958805 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -24,7 +24,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateInitialSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); Task TriggerModelInference(string? instructions = null); Task> OnResponsedDone(RealtimeHubConnection conn, string response); diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 51c1bd33..c33f28a2 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -1,9 +1,7 @@ using BotSharp.Abstraction.Realtime; using System.Net.WebSockets; -using System; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Enums; namespace BotSharp.Core.Realtime; @@ -12,6 +10,7 @@ public class RealtimeHub : IRealtimeHub { private readonly IServiceProvider _services; private readonly ILogger _logger; + public RealtimeHub(IServiceProvider services, ILogger logger) { _services = services; @@ -82,8 +81,7 @@ public class RealtimeHub : IRealtimeHub onModelReady: async () => { // Control initial session - await completer.UpdateInitialSession(conn); - + await completer.UpdateSession(conn); // Add dialog history foreach (var item in dialogs) @@ -123,8 +121,21 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); + if (message.FunctionName == "route_to_agent") + { + var routedAgentId = routing.Context.GetCurrentAgentId(); + if (conn.EntryAgentId != routedAgentId) + { + conn.EntryAgentId = routedAgentId; + await completer.UpdateSession(conn); + await completer.TriggerModelInference("Reply based on the function's output."); + } + } + else + { + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } } else { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index efa96547..834cdd17 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -254,13 +254,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateInitialSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conv.AgentId); + var agent = await agentService.LoadAgent(conn.EntryAgentId); var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 774d014b..cd71ba01 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -1,13 +1,12 @@ using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Twilio.TwiML.Voice; using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -52,10 +51,6 @@ public class TwilioStreamController : TwilioController { request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; } - else - { - request.ConversationId = request.CallSid; - } await HookEmitter.Emit(_services, async hook => { @@ -65,7 +60,7 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - await InitConversation(request); + request.ConversationId = await InitConversation(request); var twilio = _services.GetRequiredService(); @@ -82,7 +77,7 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - private async Task InitConversation(ConversationalVoiceRequest request) + private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); @@ -90,11 +85,10 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - Id = request.CallSid, AgentId = _settings.AgentId, Channel = ConversationChannel.Phone, ChannelId = request.CallSid, - Title = $"Phone call from {request.From}", + Title = $"Incoming phone call from {request.From}", Tags = [], }; @@ -106,9 +100,13 @@ public class TwilioStreamController : TwilioController new("channel", ConversationChannel.Phone), new("calling_phone", request.From), new("twilio_call_sid", request.CallSid), + // Enable lazy routing mode to optimize realtime experience + new(StateConst.ROUTING_MODE, "lazy"), }; convService.SetConversationId(conversation.Id, states); convService.SaveStates(); + + return conversation.Id; } } From 419b4717d6c333265a6283b8c0c0d09d8b19b027 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 27 Feb 2025 12:52:14 +0800 Subject: [PATCH 11/84] Support AgentId for twilio stream --- .../Controllers/TwilioStreamController.cs | 2 +- .../Models/ConversationalVoiceRequest.cs | 3 +++ .../agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json | 5 +++-- .../agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json | 3 ++- .../agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json | 5 +++-- .../agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json | 3 ++- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index cd71ba01..dcb33959 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -85,7 +85,7 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - AgentId = _settings.AgentId, + AgentId = request.AgentId ?? _settings.AgentId, Channel = ConversationChannel.Phone, ChannelId = request.CallSid, Title = $"Incoming phone call from {request.From}", diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 1fe78116..e2b6ee13 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -4,6 +4,9 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceRequest : VoiceRequest { + [FromQuery(Name = "agent-id")] + public string AgentId { get; set; } + [FromRoute] public string ConversationId { get; set; } diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json index 94746011..3a28b3e0 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json @@ -1,7 +1,7 @@ { "id": "8970b1e5-d260-4e2c-90b1-f1415a257c18", "name": "Pizza Bot", - "description": "AI assistant that can help customer place pizza order.", + "description": "AI assistant that can help customer place pizza order, make payment or inquiry order status.", "type": "routing", "inheritAgentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", "createdDateTime": "2023-08-18T10:39:32.2349685Z", @@ -10,10 +10,11 @@ "disabled": false, "isPublic": true, "profiles": [ "pizza" ], + "labels": [ "experiment" ], "routingRules": [ { "type": "reasoner", - "field": "NaiveReasoner" + "field": "Naive Reasoner" } ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json index 4f606251..09f7489b 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json @@ -6,5 +6,6 @@ "id": "b284db86-e9c2-4c25-a59e-4649797dd130", "disabled": false, "isPublic": true, - "profiles": [ "pizza" ] + "profiles": [ "pizza" ], + "labels": [ "experiment" ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json index 62f069bb..40988cb2 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json @@ -1,10 +1,11 @@ { "name": "Ordering", - "description": "Provide types of pizza available, unit price and total cost. Place the order and returned the order number.", + "description": "Provide types of pizza available, unit price, total cost and place the order.", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd", "disabled": false, "isPublic": true, - "profiles": [ "pizza" ] + "profiles": [ "pizza" ], + "labels": [ "experiment" ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json index 1dab7257..ef47f34d 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json @@ -1,12 +1,13 @@ { "name": "Payment", - "description": "Make payment when user wants to pay for the order", + "description": "Make payment when user confirmed the price and going to pay for the order", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "fe8c60aa-b114-4ef3-93cb-a8efeac80f75", "disabled": false, "isPublic": true, "profiles": [ "pizza" ], + "labels": [ "experiment" ], "routingRules": [ { "field": "order_number", From 3e0d1a4d2a48130b528c2af6dac5f4d05e69dbaa Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 27 Feb 2025 16:19:43 -0600 Subject: [PATCH 12/84] add try catch --- .../Hooks/ChatHubConversationHook.cs | 101 +++++++++++++----- .../Hooks/ChatHubCrontabHook.cs | 24 ++++- .../Hooks/StreamingLogHook.cs | 68 +++++++++--- .../Hooks/WelcomeHook.cs | 32 ++++-- .../BotSharp.Plugin.ChatHub/SignalRHub.cs | 24 +++-- 5 files changed, 189 insertions(+), 60 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 9f2c1008..99b734e6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -7,6 +7,7 @@ public class ChatHubConversationHook : ConversationHookBase { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly BotSharpOptions _options; private readonly ChatHubSettings _settings; @@ -23,12 +24,14 @@ public class ChatHubConversationHook : ConversationHookBase public ChatHubConversationHook( IServiceProvider services, IHubContext chatHub, + ILogger logger, BotSharpOptions options, ChatHubSettings settings, IUserIdentity user) { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _options = options; _settings = settings; @@ -177,74 +180,122 @@ public class ChatHubConversationHook : ConversationHookBase private async Task InitClientConversation(string conversationId, ConversationViewModel conversation) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + _logger.LogWarning($"Failed to init client conversation in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task ReceiveClientMessage(string conversationId, ChatResponseModel model) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task ReceiveAssistantMessage(string conversationId, string? json) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } - + } private async Task GenerateSenderAction(string conversationId, ConversationSenderActionModel action) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + _logger.LogWarning($"Failed to generate sender action in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task DeleteMessage(string conversationId, ChatResponseModel model) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(DELETE_MESSAGE, model); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(DELETE_MESSAGE, model); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(DELETE_MESSAGE, model); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(DELETE_MESSAGE, model); + _logger.LogWarning($"Failed to delete message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task GenerateNotification(string conversationId, string? json) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_NOTIFICATION, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_NOTIFICATION, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_NOTIFICATION, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_NOTIFICATION, json); + _logger.LogWarning($"Failed to generate notification in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } #endregion diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs index a00dae90..f4b4266f 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs @@ -8,6 +8,7 @@ public class ChatHubCrontabHook : ICrontabHook { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; @@ -19,6 +20,7 @@ public class ChatHubCrontabHook : ICrontabHook public ChatHubCrontabHook(IServiceProvider services, IHubContext chatHub, + ILogger logger, IUserIdentity user, IConversationStorage storage, BotSharpOptions options, @@ -26,6 +28,7 @@ public class ChatHubCrontabHook : ICrontabHook { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _storage = storage; _options = options; @@ -48,13 +51,26 @@ public class ChatHubCrontabHook : ICrontabHook } }, _options.JsonSerializerOptions); - if (_settings.EventDispatchBy == EventDispatchType.Group) + await SendEvent(item, json); + } + + private async Task SendEvent(CrontabItem item, string json) + { + try { - await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); + } + else + { + await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); + _logger.LogWarning($"Failed to send event in {nameof(ChatHubCrontabHook)} (conversation id: {item.ConversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index a2f527f4..351bb3a3 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -12,6 +12,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR private readonly ChatHubSettings _settings; private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IConversationStateService _state; private readonly IUserIdentity _user; private readonly IAgentService _agentService; @@ -30,6 +31,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR ChatHubSettings settings, IServiceProvider serivces, IHubContext chatHub, + ILogger logger, IConversationStateService state, IUserIdentity user, IAgentService agentService, @@ -40,6 +42,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR _settings = settings; _services = serivces; _chatHub = chatHub; + _logger = logger; _state = state; _user = user; _agentService = agentService; @@ -439,54 +442,85 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR #region Private methods private async Task SendContentLog(string conversationId, ContentLogInputModel input) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + _logger.LogWarning($"Failed to send content log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendStateLog(string conversationId, string agentId, Dictionary states, RoleDialogModel message) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + _logger.LogWarning($"Failed to send state log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendAgentQueueLog(string conversationId, string log) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + _logger.LogWarning($"Failed to send agent queue log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendStateChange(string conversationId, StateChangeModel stateChange) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + _logger.LogWarning($"Failed to send state change in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } - private string BuildContentLog(ContentLogInputModel input) { var output = new ContentLogOutputModel diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index c7aedd83..70545305 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -6,6 +6,7 @@ public class WelcomeHook : ConversationHookBase { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; @@ -17,6 +18,7 @@ public class WelcomeHook : ConversationHookBase public WelcomeHook(IServiceProvider services, IHubContext chatHub, + ILogger logger, IUserIdentity user, IConversationStorage storage, BotSharpOptions options, @@ -24,6 +26,7 @@ public class WelcomeHook : ConversationHookBase { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _storage = storage; _options = options; @@ -78,17 +81,30 @@ public class WelcomeHook : ConversationHookBase _storage.Append(conversation.Id, dialog); - if (_settings.EventDispatchBy == EventDispatchType.Group) - { - await _chatHub.Clients.Group(conversation.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); - } - else - { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); - } + await SendEvent(conversation.Id, json); } } await base.OnUserAgentConnectedInitially(conversation); } + + private async Task SendEvent(string conversationId, string json) + { + try + { + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Failed to send event in {nameof(WelcomeHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs index 240b5993..63ded25a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs @@ -33,12 +33,7 @@ public class SignalRHub : Hub if (!string.IsNullOrEmpty(conversationId)) { _logger.LogInformation($"Connection {Context.ConnectionId} is with conversation {conversationId}"); - - var settings = _services.GetRequiredService(); - if (settings.EventDispatchBy == EventDispatchType.Group) - { - await Groups.AddToGroupAsync(Context.ConnectionId, conversationId); - } + await AddGroup(conversationId); var conv = await convService.GetConversation(conversationId); if (conv != null) @@ -56,4 +51,21 @@ public class SignalRHub : Hub await base.OnConnectedAsync(); } + + private async Task AddGroup(string conversationId) + { + try + { + var settings = _services.GetRequiredService(); + if (settings.EventDispatchBy == EventDispatchType.Group) + { + await Groups.AddToGroupAsync(Context.ConnectionId, conversationId); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Failed to add chat group in {nameof(SignalRHub)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + } + } } From a15821061f15e21fb148f2b027f9e13a03147558 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 27 Feb 2025 21:19:20 -0600 Subject: [PATCH 13/84] Upgrade package --- Directory.Packages.props | 20 +++++++++---------- .../BotSharp.Core/Realtime/RealtimeHub.cs | 4 ++++ .../Realtime/RealTimeCompletionProvider.cs | 7 ++++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index f6b1545e..42521480 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,24 +1,24 @@ 8.0.0 + 2.3.0 - - - - - - - + + + + + + + - - - + + diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index c33f28a2..83a251ae 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -75,6 +75,10 @@ public class RealtimeHub : IRealtimeHub var routing = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); + if (dialogs.Count == 0) + { + dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); + } routing.Context.SetDialogs(dialogs); await completer.Connect(conn, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 834cdd17..d4b74771 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -212,9 +212,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (message is not string data) { - data = JsonSerializer.Serialize(message); + data = JsonSerializer.Serialize(message, options: new JsonSerializerOptions + { + WriteIndented = true + }); } + _logger.LogInformation($"SendEventToModel:\r\n{data}"); + var buffer = Encoding.UTF8.GetBytes(data); await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); From 31fe5757863c39e68207ce5697e76509daa3552b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 28 Feb 2025 11:27:23 +0800 Subject: [PATCH 14/84] Fix SemanticKernel compile issue. --- .../BotSharp.Plugin.SemanticKernel.UnitTests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index c47fcead..ee159e86 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -10,7 +10,6 @@ - From 2ab73ead56f8a0eabb1d29b0a5ce1398f06da4d9 Mon Sep 17 00:00:00 2001 From: Haiping Date: Fri, 28 Feb 2025 09:51:48 -0600 Subject: [PATCH 15/84] Update agent.json --- .../data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json index c54a67e0..3200865e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json @@ -1,7 +1,7 @@ { "id": "01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d", "name": "Fallback Agent", - "description": "Don't have sufficient confidence to trigger any of existing agent.", + "description": "Handle initiated conversation without specific task given yet or don't have sufficient confidence to handle user task.", "type": "task", "createdDateTime": "2024-05-07T10:00:00Z", "updatedDateTime": "2024-05-07T10:00:00Z", From 152811cc722a1ab5bbbafd51182de48e749cb183 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 10:00:58 -0600 Subject: [PATCH 16/84] add agent id in resource key --- .../BotSharp.Core/Conversations/Services/TokenStatistics.cs | 5 +++-- .../BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 9fa5c718..8387e187 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -60,12 +60,13 @@ public class TokenStatistics : ITokenStatistics stat.SetState("llm_total_cost", total_cost, isNeedVersion: false, source: StateSource.Application); // Save stats + var agentId = message.CurrentAgentId ?? string.Empty; var globalStats = _services.GetRequiredService(); var body = new BotSharpStatsInput { Metric = StatsMetric.AgentLlmCost, Dimension = "agent", - DimRefVal = message.CurrentAgentId, + DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ @@ -75,7 +76,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats("global-llm-cost", body); + globalStats.UpdateStats($"global-llm-cost-{agentId}", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 2cc2c9fb..34fda963 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -25,17 +25,18 @@ public class GlobalStatsConversationHook : IContentGeneratingHook // record agent call var globalStats = _services.GetRequiredService(); + var agentId = message.CurrentAgentId ?? string.Empty; var body = new BotSharpStatsInput { Metric = StatsMetric.AgentCall, Dimension = "agent", - DimRefVal = message.CurrentAgentId ?? string.Empty, + DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats("global-agent-call", body); + globalStats.UpdateStats($"global-agent-call-{agentId}", body); } } From 14b6d41ce521f624fed85df7e7a0aa707ddbef10 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 12:44:59 -0600 Subject: [PATCH 17/84] OnSessionUpdated log --- .../Loggers/IContentGeneratingHook.cs | 11 ++++ .../Options/BotSharpOptions.cs | 2 +- .../Realtime/Models/RealtimeHubConnection.cs | 2 +- .../Conversations/Services/TokenStatistics.cs | 4 ++ .../BotSharp.Core/Realtime/RealtimeHub.cs | 20 +++---- .../Routing/RoutingService.InvokeAgent.cs | 2 +- .../util-routing-fallback_to_router.json | 2 +- .../util-routing-fallback_to_router.fn.liquid | 2 +- .../Hooks/StreamingLogHook.cs | 27 +++++++++ .../BotSharp.Plugin.OpenAI.csproj | 1 + .../Realtime/RealTimeCompletionProvider.cs | 59 ++++++++++++------- .../Providers/Text/TextCompletionProvider.cs | 13 +--- 12 files changed, 100 insertions(+), 45 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs index 9d0662db..c4f63450 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Abstraction.Loggers; /// @@ -37,4 +39,13 @@ public interface IContentGeneratingHook /// /// Task OnRenderingTemplate(Agent agent, string name, string content) => Task.CompletedTask; + + /// + /// Realtime session updated + /// + /// + /// + /// + /// + Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs index 90168872..b8609422 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Options; public class BotSharpOptions { - private readonly static JsonSerializerOptions defaultJsonOptions = new JsonSerializerOptions() + public readonly static JsonSerializerOptions defaultJsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 60fec1dc..ec521637 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -4,7 +4,7 @@ public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; - public string EntryAgentId { get; set; } = null!; + public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; public string Model { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 9fa5c718..2be594f5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -110,6 +110,10 @@ public class TokenStatistics : ITokenStatistics public void StopTimer() { + if (_timer == null) + { + return; + } _timer.Stop(); } } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 83a251ae..ee59fb62 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -71,9 +71,11 @@ public class RealtimeHub : IRealtimeHub var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conversation.AgentId); - conn.EntryAgentId = agent.Id; + conn.CurrentAgentId = agent.Id; var routing = _services.GetRequiredService(); + routing.Context.Push(agent.Id); + var dialogs = convService.GetDialogHistory(); if (dialogs.Count == 0) { @@ -125,21 +127,19 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - if (message.FunctionName == "route_to_agent") + if (message.FunctionName == "route_to_agent" || + message.FunctionName == "util-routing-fallback_to_router") { var routedAgentId = routing.Context.GetCurrentAgentId(); - if (conn.EntryAgentId != routedAgentId) + if (conn.CurrentAgentId != routedAgentId) { - conn.EntryAgentId = routedAgentId; + conn.CurrentAgentId = routedAgentId; await completer.UpdateSession(conn); - await completer.TriggerModelInference("Reply based on the function's output."); } } - else - { - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); - } + + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); } else { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 273fc996..bec44bc4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -54,7 +54,7 @@ public partial class RoutingService if (agent.Type == AgentType.Routing) { // Forgot about what situation needs to handle in this way - // response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; + response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; } message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json index cd9549c3..77f0dbfd 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -1,6 +1,6 @@ { "name": "util-routing-fallback_to_router", - "description": "Get the appropriate agent who can handle the user request.", + "description": "Return to the Router to find the appropriate agent who can handle the user's request.", "parameters": { "type": "object", "properties": { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid index ab3863f8..a1bd9ceb 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid @@ -1 +1 @@ -"If you're unsure whether you understand the user's request or if the user brings up an unrelated topic, call the function `util-routing-fallback_to_router` to get the appropriate agent from the router." \ No newline at end of file +Carefully consider whether the current user request is related to your responsibilities. Only when it is not relevant should you consider Return to the Router. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 351bb3a3..22ec7d4d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -85,6 +85,33 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR await SendContentLog(conversationId, input); } + public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) + { + var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + + // Agent queue log + var log = $"{instruction}"; + if (functions.Length > 0) + { + log += $"\r\n\r\n[FUNCTIONS]:\r\n\r\n{string.Join("\r\n\r\n", functions.Select(x => JsonSerializer.Serialize(x, BotSharpOptions.defaultJsonOptions)))}"; + } + _logger.LogInformation(log); + + var message = new RoleDialogModel(AgentRole.Assistant, log) + { + MessageId = _routingCtx.MessageId + }; + var input = new ContentLogInputModel(conversationId, message) + { + Name = agent.Name, + AgentId = agent.Id, + Source = ContentLogSource.Prompt, + Log = log + }; + await SendContentLog(conversationId, input); + } + public async Task OnRenderingTemplate(Agent agent, string name, string content) { if (!_convSettings.ShowVerboseLog) return; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 9a9c57fb..0a509b1a 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -18,6 +18,7 @@ + \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index d4b74771..722972f8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,7 +1,9 @@ using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; using System.Net.WebSockets; @@ -212,14 +214,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (message is not string data) { - data = JsonSerializer.Serialize(message, options: new JsonSerializerOptions - { - WriteIndented = true - }); + data = JsonSerializer.Serialize(message, BotSharpOptions.defaultJsonOptions); } - _logger.LogInformation($"SendEventToModel:\r\n{data}"); - var buffer = Encoding.UTF8.GetBytes(data); await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); @@ -265,13 +262,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var conv = await convService.GetConversation(conn.ConversationId); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conn.EntryAgentId); + var agent = await agentService.LoadAgent(conn.CurrentAgentId); var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, []); var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + var functions = options.Tools.Select(x => + { + var fn = new FunctionDef + { + Name = x.FunctionName, + Description = x.FunctionDescription + }; + fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); + return fn; + }).ToArray(); var sessionUpdate = new { @@ -287,21 +294,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Voice = "alloy", Instructions = instruction, ToolChoice = "auto", - Tools = options.Tools.Select(x => - { - var fn = new FunctionDef - { - Name = x.FunctionName, - Description = x.FunctionDescription - }; - fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); - return fn; - }).ToArray(), + Tools = functions, Modalities = [ "text", "audio" ], Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) } }; + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionUpdated(agent, instruction, functions); + }); + await SendEventToModel(sessionUpdate); } @@ -568,7 +571,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { outputs.Add(new RoleDialogModel(output.Role, output.Arguments) { - CurrentAgentId = conn.EntryAgentId, + CurrentAgentId = conn.CurrentAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, ToolCallId = output.CallId, @@ -581,11 +584,27 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(output.Role, content.Transcript) { - CurrentAgentId = conn.EntryAgentId + CurrentAgentId = conn.CurrentAgentId }); } } + var contentHooks = _services.GetServices().ToList(); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } + return outputs; } @@ -594,7 +613,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) { - CurrentAgentId = conn.EntryAgentId + CurrentAgentId = conn.CurrentAgentId }; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index c2180076..097a32b7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -3,6 +3,7 @@ using System.Net.Http; using System.Net.Mime; using System.Text.Json; using System.Text; +using BotSharp.Abstraction.Options; namespace BotSharp.Plugin.OpenAI.Providers.Text; @@ -13,14 +14,6 @@ public class TextCompletionProvider : ITextCompletion private readonly OpenAiSettings _settings; protected string _model; - protected readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - PropertyNameCaseInsensitive = true, - WriteIndented = true, - AllowTrailingCommas = true, - }; - public virtual string Provider => "openai"; public TextCompletionProvider( @@ -110,7 +103,7 @@ public class TextCompletionProvider : ITextCompletion MaxTokens = maxTokens, Temperature = temperature }; - var data = JsonSerializer.Serialize(request, _jsonOptions); + var data = JsonSerializer.Serialize(request, BotSharpOptions.defaultJsonOptions); var httpRequest = new HttpRequestMessage { Method = HttpMethod.Post, @@ -121,7 +114,7 @@ public class TextCompletionProvider : ITextCompletion var httpResponse = await httpClient.SendAsync(httpRequest); httpResponse.EnsureSuccessStatusCode(); var responseStr = await httpResponse.Content.ReadAsStringAsync(); - var response = JsonSerializer.Deserialize(responseStr, _jsonOptions); + var response = JsonSerializer.Deserialize(responseStr, BotSharpOptions.defaultJsonOptions); return response; } catch (Exception ex) From 606026130a07f7b5e3d326e481abbb357d1f8ca8 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 15:26:45 -0600 Subject: [PATCH 18/84] rollback reset states --- .../Conversations/Services/ConversationStateService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 4d229866..5d19e2d0 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -220,7 +220,6 @@ public class ConversationStateService : IConversationStateService { if (_conversationId == null || _sidecar?.IsEnabled() == true) { - Reset(); return; } @@ -253,7 +252,6 @@ public class ConversationStateService : IConversationStateService } _db.UpdateConversationStates(_conversationId, states); - Reset(); _logger.LogInformation($"Saved states of conversation {_conversationId}"); } From 2851851657df96f3b3d9da04315ca92507fddd75 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 15:34:53 -0600 Subject: [PATCH 19/84] refine lock key --- .../Conversations/Services/TokenStatistics.cs | 8 +++++--- .../BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 8387e187..68f5ecf0 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -60,12 +60,14 @@ public class TokenStatistics : ITokenStatistics stat.SetState("llm_total_cost", total_cost, isNeedVersion: false, source: StateSource.Application); // Save stats + var metric = StatsMetric.AgentLlmCost; + var dim = "agent"; var agentId = message.CurrentAgentId ?? string.Empty; var globalStats = _services.GetRequiredService(); var body = new BotSharpStatsInput { - Metric = StatsMetric.AgentLlmCost, - Dimension = "agent", + Metric = metric, + Dimension = dim, DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, @@ -76,7 +78,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats($"global-llm-cost-{agentId}", body); + globalStats.UpdateStats($"global-llm-cost-{metric}-{dim}-{agentId}", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 34fda963..66b0e8ef 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -25,11 +25,13 @@ public class GlobalStatsConversationHook : IContentGeneratingHook // record agent call var globalStats = _services.GetRequiredService(); + var metric = StatsMetric.AgentCall; + var dim = "agent"; var agentId = message.CurrentAgentId ?? string.Empty; var body = new BotSharpStatsInput { - Metric = StatsMetric.AgentCall, - Dimension = "agent", + Metric = metric, + Dimension = dim, DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, @@ -37,6 +39,6 @@ public class GlobalStatsConversationHook : IContentGeneratingHook new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats($"global-agent-call-{agentId}", body); + globalStats.UpdateStats($"global-agent-call-{metric}-{dim}-{agentId}", body); } } From 64831f5b3a16cf54338cea425e578b90be7728fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 15:35:46 -0600 Subject: [PATCH 20/84] minor change --- .../BotSharp.Core/Conversations/Services/TokenStatistics.cs | 2 +- .../BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 388c814c..0698c0cb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -78,7 +78,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats($"global-llm-cost-{metric}-{dim}-{agentId}", body); + globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 66b0e8ef..8ff9be86 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -39,6 +39,6 @@ public class GlobalStatsConversationHook : IContentGeneratingHook new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats($"global-agent-call-{metric}-{dim}-{agentId}", body); + globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body); } } From 235d972ca1f9af8fb2a11fb91f524b79d5482dd8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 16:47:03 -0600 Subject: [PATCH 21/84] remove data --- .../Conversations/Models/Conversation.cs | 7 +------ .../Conversations/Services/ConversationStorage.cs | 15 +++++---------- .../Models/DialogMongoElement.cs | 7 ++----- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index 07258402..ac1efb04 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -58,17 +58,13 @@ public class DialogElement [JsonPropertyName("payload")] public string? Payload { get; set; } - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("data")] - public object? Data { get; set; } - public DialogElement() { } public DialogElement(DialogMetaData meta, string content, string? richContent = null, - string? secondaryContent = null, string? secondaryRichContent = null, string? payload = null, object? data = null) + string? secondaryContent = null, string? secondaryRichContent = null, string? payload = null) { MetaData = meta; Content = content; @@ -76,7 +72,6 @@ public class DialogElement SecondaryContent = secondaryContent; SecondaryRichContent = secondaryRichContent; Payload = payload; - Data = data; } public override string ToString() diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 89e9fcaa..75d1a501 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -52,8 +52,7 @@ public class ConversationStorage : IConversationStorage MetaData = meta, Content = dialog.Content, SecondaryContent = dialog.SecondaryContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } else @@ -84,8 +83,7 @@ public class ConversationStorage : IConversationStorage SecondaryContent = dialog.SecondaryContent, RichContent = richContent, SecondaryRichContent = secondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } @@ -123,8 +121,7 @@ public class ConversationStorage : IConversationStorage MetaData = meta, Content = dialog.Content, SecondaryContent = dialog.SecondaryContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } else @@ -155,8 +152,7 @@ public class ConversationStorage : IConversationStorage SecondaryContent = dialog.SecondaryContent, RichContent = richContent, SecondaryRichContent = secondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } } @@ -200,8 +196,7 @@ public class ConversationStorage : IConversationStorage RichContent = richContent, SecondaryContent = secondaryContent, SecondaryRichContent = secondaryRichContent, - Payload = payload, - Data = dialog.Data + Payload = payload }; results.Add(record); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 87022130..030a93ab 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -11,7 +11,6 @@ public class DialogMongoElement public string? RichContent { get; set; } public string? SecondaryRichContent { get; set; } public string? Payload { get; set; } - public object? Data { get; set; } public static DialogMongoElement ToMongoElement(DialogElement dialog) { @@ -22,8 +21,7 @@ public class DialogMongoElement SecondaryContent = dialog.SecondaryContent, RichContent = dialog.RichContent, SecondaryRichContent = dialog.SecondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }; } @@ -36,8 +34,7 @@ public class DialogMongoElement SecondaryContent = dialog.SecondaryContent, RichContent = dialog.RichContent, SecondaryRichContent = dialog.SecondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }; } } From e3036ef3ba3630608d32c43955a02040066ff555 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 18:19:44 -0600 Subject: [PATCH 22/84] add welcome for realtime --- .../MLTasks/IRealTimeCompletion.cs | 2 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 40 +++++++++++++------ .../functions/route_to_agent.json | 2 +- .../instructions/instruction.liquid | 2 + .../util-routing-fallback_to_router.json | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 32 ++++++++++++++- 6 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index bd958805..be6f8821 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -26,7 +26,9 @@ public interface IRealTimeCompletion Task CreateSession(Agent agent, List conversations); Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); + Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); + Task CancelModelResponse(); Task> OnResponsedDone(RealtimeHubConnection conn, string response); Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index ee59fb62..eeac2ab3 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -3,6 +3,7 @@ using System.Net.WebSockets; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Core.Realtime; @@ -97,7 +98,7 @@ public class RealtimeHub : IRealtimeHub if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { @@ -127,19 +128,32 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - if (message.FunctionName == "route_to_agent" || - message.FunctionName == "util-routing-fallback_to_router") - { - var routedAgentId = routing.Context.GetCurrentAgentId(); - if (conn.CurrentAgentId != routedAgentId) - { - conn.CurrentAgentId = routedAgentId; - await completer.UpdateSession(conn); - } - } - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); + if (message.FunctionName == "route_to_agent") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + message.Content = $"Connected to agent of {inst.AgentName}"; + conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await completer.UpdateSession(conn); + await completer.InsertConversationItem(message); + await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}."); + } + else if (message.FunctionName == "util-routing-fallback_to_router") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + message.Content = $"Returned to Router due to {inst.Reason}"; + conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await completer.UpdateSession(conn); + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply user request."); + } + else + { + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } } else { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json index 31c8400f..08f938ec 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -26,6 +26,6 @@ "description": "Required parameters of next action agent" } }, - "required": [ "next_action_agent", "user_goal_agent", "args" ] + "required": [ "next_action_agent", "user_goal_agent", "next_action_reason", "args" ] } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index 997012db..7ee8f56e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -5,7 +5,9 @@ Follow these steps to handle user request: 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. +{% if routing_mode != 'lazy' %} 5. Response must be in JSON format. +{% endif %} {% if routing_requirements and routing_requirements != empty %} [REQUIREMENTS] diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json index 77f0dbfd..3902d8ae 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -13,6 +13,6 @@ "description": "User question or statement." } }, - "required": [ "user_question" ] + "required": [ "fallback_reason" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 722972f8..2be07da8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -102,6 +102,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } + public async Task CancelModelResponse() + { + await SendEventToModel(new + { + type = "response.cancel" + }); + } + + public async Task RemoveConversationItem(string itemId) + { + await SendEventToModel(new + { + type = "conversation.item.delete", + item_id = itemId + }); + } + private async Task ReceiveMessage(RealtimeHubConnection conn, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, @@ -169,7 +186,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - await Task.Delay(1000); var messages = await OnResponsedDone(conn, receivedText); onModelResponseDone(messages); } @@ -296,7 +312,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) + Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), + MaxResponseOutputTokens = 512, + TurnDetection = new RealtimeSessionTurnDetection + { + Threshold = 0.8f, + SilenceDuration = 800 + } } }; @@ -565,6 +587,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var outputs = new List(); var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + return []; + } + foreach (var output in data.Outputs) { if (output.Type == "function_call") @@ -575,6 +602,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion FunctionName = output.Name, FunctionArgs = output.Arguments, ToolCallId = output.CallId, + MessageId = output.Id, MessageType = MessageTypeName.FunctionCall }); } From c096b6365682386c320fa7993ffc33df6589310a Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 20:42:03 -0600 Subject: [PATCH 23/84] Output input audio transcript. --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index eeac2ab3..7af86a45 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -137,7 +137,7 @@ public class RealtimeHub : IRealtimeHub await completer.UpdateSession(conn); await completer.InsertConversationItem(message); - await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}."); + await completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures."); } else if (message.FunctionName == "util-routing-fallback_to_router") { @@ -147,7 +147,7 @@ public class RealtimeHub : IRealtimeHub await completer.UpdateSession(conn); await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply user request."); + await completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}"); } else { @@ -157,7 +157,7 @@ public class RealtimeHub : IRealtimeHub } else { - // append transcript to conversation + // append output audio transcript to conversation storage.Append(conn.ConversationId, message); dialogs.Add(message); @@ -166,10 +166,7 @@ public class RealtimeHub : IRealtimeHub hook.SetAgent(agent) .SetConversation(conversation); - if (!string.IsNullOrEmpty(message.Content)) - { - await hook.OnResponseGenerated(message); - } + await hook.OnResponseGenerated(message); } } } @@ -180,9 +177,17 @@ public class RealtimeHub : IRealtimeHub }, onInputAudioTranscriptionCompleted: async message => { - // append transcript to conversation + // append input audio transcript to conversation storage.Append(conn.ConversationId, message); dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnMessageReceived(message); + } }, onUserInterrupted: async () => { From 61e3d2f04ad49d760a81d0b03152597f8c84d58d Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 1 Mar 2025 11:44:52 -0800 Subject: [PATCH 24/84] more projects --- Directory.Packages.props | 35 +++++++++++++++++-- .../BotSharp.Plugin.MetaMessenger.csproj | 5 +-- ...tSharp.Plugin.MicrosoftExtensionsAI.csproj | 5 +-- .../BotSharp.Plugin.MongoStorage.csproj | 3 +- .../BotSharp.Plugin.OpenAI.csproj | 7 ++-- .../BotSharp.Plugin.PaddleSharp.csproj | 19 +++++----- .../BotSharp.Plugin.PythonInterpreter.csproj | 3 +- .../BotSharp.Plugin.Qdrant.csproj | 3 +- .../BotSharp.Plugin.RoutingSpeeder.csproj | 5 +-- .../BotSharp.Plugin.Selenium.csproj | 5 +-- .../BotSharp.Plugin.SemanticKernel.csproj | 7 ++-- .../BotSharp.Plugin.SparkDesk.csproj | 3 +- .../BotSharp.Plugin.SqlDriver.csproj | 7 ++-- .../BotSharp.Plugin.TencentCos.csproj | 3 +- .../BotSharp.Plugin.Twilio.csproj | 9 ++--- .../BotSharp.Plugin.WeChat.csproj | 5 +-- .../BotSharp.Plugin.WebDriver.csproj | 5 +-- 17 files changed, 88 insertions(+), 41 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 42521480..00c01c68 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -11,7 +11,7 @@ - + @@ -43,7 +43,8 @@ - + + @@ -51,6 +52,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj index 225d7a90..4d871e0b 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj @@ -7,6 +7,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -14,8 +15,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj index b2667f35..3af661c8 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj @@ -9,11 +9,12 @@ $(GeneratePackageOnBuild) $(SolutionDir)packages $(NoWarn);NU5104 + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj index 5ac884e0..a7673b93 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 0a509b1a..22e90ee6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -8,12 +8,13 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 75f0246a..b17038ff 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -8,18 +8,19 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - - - - - - - - + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj index c5e3d4ff..8ea26cf7 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -25,7 +26,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj index fb19756b..74f28534 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj +++ b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj index 488e0db5..7088307f 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -29,8 +30,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj index 2a4e3e08..33316a95 100644 --- a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj +++ b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj index b8c4a8e9..92a061a0 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj @@ -8,12 +8,13 @@ $(GeneratePackageOnBuild) $(GeneratePackageOnBuild) $(SolutionDir)packages + true - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj index 04969796..c74b04f5 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index dbd3228d..f03955fe 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -100,9 +101,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj index b52e930a..d6082ce7 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj +++ b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index ece4e6f6..9a84f462 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -6,6 +6,7 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages + true @@ -24,10 +25,10 @@ - - - - + + + + diff --git a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj index 4a513a97..b21581f0 100644 --- a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj +++ b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj @@ -7,6 +7,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -27,8 +28,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index c5436fe6..e6ec3e17 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + From 885b16f220d77dcf5c9cd62109d5f545e3dd5508 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 1 Mar 2025 11:57:38 -0800 Subject: [PATCH 25/84] more projects --- Directory.Packages.props | 28 +++++++++++++++++-- src/BotSharp.AppHost/BotSharp.AppHost.csproj | 5 ++-- .../BotSharp.ServiceDefaults.csproj | 23 +++++++-------- src/WebStarter/WebStarter.csproj | 25 +++++++++-------- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 00c01c68..188c7ee5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,7 +1,7 @@ 8.0.0 - 2.3.0 + 2.3.0 @@ -82,8 +82,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + - diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 2a2705db..5fc59246 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -7,11 +7,12 @@ enable true e4bd9862-7edd-45f0-9a89-b8e3b3339c71 + true - - + + diff --git a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj index 53f9559f..2b3ad27d 100644 --- a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj +++ b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj @@ -5,21 +5,22 @@ enable enable true + true - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index b2fcd816..e78b8bef 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -6,24 +6,25 @@ enable OutOfProcess 4fb8c9df-7975-4926-ba73-46c8ca440691 + true - - - - - - - - - - + + + + + + + + + + - - + + From 059942e0a188660d2a028b282475a44813d55d35 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 1 Mar 2025 12:06:26 -0800 Subject: [PATCH 26/84] unit test projects --- Directory.Packages.props | 9 +++++++++ ...tSharp.Plugin.SemanticKernel.UnitTests.csproj | 16 ++++++++-------- tests/UnitTest/UnitTest.csproj | 14 +++++++------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 188c7ee5..8c1a3bf1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -95,6 +95,15 @@ + + + + + + + + + diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index ee159e86..fa03bec6 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -4,21 +4,21 @@ $(TargetFramework) enable enable - false + true - - - - - - + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/UnitTest/UnitTest.csproj b/tests/UnitTest/UnitTest.csproj index 67e7d504..3590e075 100644 --- a/tests/UnitTest/UnitTest.csproj +++ b/tests/UnitTest/UnitTest.csproj @@ -4,18 +4,18 @@ $(TargetFramework) enable enable - false true + true - - - - - - + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 2397f756cd203f4a623b1585dcdbca8b9f90c49a Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 1 Mar 2025 19:44:55 -0800 Subject: [PATCH 27/84] added back missing version for Microsoft.Extensions.Configuration --- Directory.Packages.props | 1 + 1 file changed, 1 insertion(+) diff --git a/Directory.Packages.props b/Directory.Packages.props index 8c1a3bf1..7ca80c18 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,6 +7,7 @@ + From c0a15f87e3c714e037102dece5837ea019fe9891 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 1 Mar 2025 19:57:50 -0800 Subject: [PATCH 28/84] remove ManagePackageVersionsCentrally from individual projects and add it in Directory.Packages.props --- Directory.Packages.props | 1 + src/BotSharp.AppHost/BotSharp.AppHost.csproj | 1 - src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj | 1 - .../BotSharp.Abstraction/BotSharp.Abstraction.csproj | 1 - .../BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj | 1 - src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj | 1 - src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 1 - .../BotSharp.Plugin.AnthropicAI.csproj | 1 - .../BotSharp.Plugin.AudioHandler.csproj | 1 - .../BotSharp.Plugin.AzureOpenAI.csproj | 1 - .../BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj | 1 - .../BotSharp.Plugin.EmailHandler.csproj | 1 - .../BotSharp.Plugin.ExcelHandler.csproj | 1 - .../BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj | 1 - .../BotSharp.Plugin.HuggingFace.csproj | 1 - .../BotSharp.Plugin.JavaScriptInterpreter.csproj | 1 - .../BotSharp.Plugin.KnowledgeBase.csproj | 1 - .../BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj | 1 - .../BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj | 1 - src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj | 1 - .../BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj | 1 - .../BotSharp.Plugin.MetaMessenger.csproj | 1 - .../BotSharp.Plugin.MicrosoftExtensionsAI.csproj | 1 - .../BotSharp.Plugin.MongoStorage.csproj | 1 - src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj | 1 - .../BotSharp.Plugin.PaddleSharp.csproj | 1 - .../BotSharp.Plugin.PythonInterpreter.csproj | 1 - src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj | 1 - .../BotSharp.Plugin.RoutingSpeeder.csproj | 1 - .../BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj | 1 - .../BotSharp.Plugin.SemanticKernel.csproj | 1 - .../BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj | 1 - .../BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj | 1 - .../BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj | 1 - src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj | 1 - src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj | 1 - .../BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj | 1 - src/WebStarter/WebStarter.csproj | 1 - .../BotSharp.Plugin.SemanticKernel.UnitTests.csproj | 1 - tests/UnitTest/UnitTest.csproj | 1 - 40 files changed, 1 insertion(+), 39 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 7ca80c18..f553f838 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -2,6 +2,7 @@ 8.0.0 2.3.0 + true diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 5fc59246..68673f7d 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -7,7 +7,6 @@ enable true e4bd9862-7edd-45f0-9a89-b8e3b3339c71 - true diff --git a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj index 2b3ad27d..d19f2eea 100644 --- a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj +++ b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj @@ -5,7 +5,6 @@ enable enable true - true diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index f189d323..ba9b65c9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -8,7 +8,6 @@ Icon.png $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj index b4bcaf84..ff6c2368 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj +++ b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj @@ -5,7 +5,6 @@ $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) - true $(SolutionDir)packages enable enable diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index c8ded3cd..044149f5 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -6,7 +6,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 58157b2d..e7338958 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -8,7 +8,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj index 4360a7a9..7309b457 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index e7459219..abc9749a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -9,7 +9,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 61758e2e..494326fa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 598e1295..2f7e326a 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index a7f4edcf..c9ceb1e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index 3b6db5d3..51355e9c 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -4,7 +4,6 @@ net8.0 enable enable - true diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index 9e153d76..08d1f499 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj index 2b8854af..0a9efc58 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj index 6abacd91..95d1c014 100644 --- a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj @@ -4,7 +4,6 @@ net8.0 enable enable - true diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 5cb7342f..aad7d494 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index d5581998..af7a6237 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj index 92509518..7ed224d3 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj +++ b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 2c247292..231b4b3f 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj index d1bcb1b4..4e60cc00 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj index 4d871e0b..284a0b4b 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj @@ -7,7 +7,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj index 3af661c8..29dcc453 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj @@ -9,7 +9,6 @@ $(GeneratePackageOnBuild) $(SolutionDir)packages $(NoWarn);NU5104 - true diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj index a7673b93..26843035 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 22e90ee6..fb8af0d3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index b17038ff..27a51abf 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj index 8ea26cf7..391593b3 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj index 74f28534..67ee3934 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj +++ b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj index 7088307f..a9ff968b 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj index 33316a95..e11e847c 100644 --- a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj +++ b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj index 92a061a0..ebc46a11 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj index c74b04f5..51f26f87 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index f03955fe..ec2d91c1 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj index d6082ce7..4d78adc0 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj +++ b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 9a84f462..3f820090 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -6,7 +6,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj index b21581f0..b7491db8 100644 --- a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj +++ b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj @@ -7,7 +7,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index e6ec3e17..2f391853 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index e78b8bef..59cac76a 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -6,7 +6,6 @@ enable OutOfProcess 4fb8c9df-7975-4926-ba73-46c8ca440691 - true diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index fa03bec6..c088a873 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -5,7 +5,6 @@ enable enable false - true diff --git a/tests/UnitTest/UnitTest.csproj b/tests/UnitTest/UnitTest.csproj index 3590e075..f7686736 100644 --- a/tests/UnitTest/UnitTest.csproj +++ b/tests/UnitTest/UnitTest.csproj @@ -6,7 +6,6 @@ enable false true - true From b4862837da96dd7f10e17c6ed5d25000a742b90c Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 3 Mar 2025 15:30:06 -0600 Subject: [PATCH 29/84] Mak initial audio is not interruptable --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 23 +++++++------ .../Realtime/RealTimeCompletionProvider.cs | 32 ++++++++++++------- .../Services/TwilioService.cs | 9 +++++- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 7af86a45..36397b0a 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -21,28 +21,22 @@ public class RealtimeHub : IRealtimeHub public async Task Listen(WebSocket userWebSocket, Func onUserMessageReceived) { - var buffer = new byte[1024 * 4]; + var buffer = new byte[1024 * 16]; WebSocketReceiveResult result; - var llmProviderService = _services.GetRequiredService(); - var model = llmProviderService.GetProviderModel("openai", "gpt-4", - realTime: true).Name; - var completer = _services.GetServices().First(x => x.Provider == "openai"); - completer.SetModelName(model); do { result = await userWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); - _logger.LogDebug($"Received from user: {receivedText}"); + if (string.IsNullOrEmpty(receivedText)) { continue; } var conn = onUserMessageReceived(receivedText); - conn.Model = model; if (conn.Event == "user_connected") { @@ -74,6 +68,17 @@ public class RealtimeHub : IRealtimeHub var agent = await agentService.LoadAgent(conversation.AgentId); conn.CurrentAgentId = agent.Id; + // Set model + var model = agent.LlmConfig.Model; + if (!model.Contains("-realtime-")) + { + var llmProviderService = _services.GetRequiredService(); + model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name; + } + + completer.SetModelName(model); + conn.Model = model; + var routing = _services.GetRequiredService(); routing.Context.Push(agent.Id); @@ -98,7 +103,7 @@ public class RealtimeHub : IRealtimeHub if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 2be07da8..7abefbbb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -128,9 +128,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var buffer = new byte[1024 * 1024 * 1]; + var buffer = new byte[1024 * 256]; WebSocketReceiveResult result; - string lastAssistantItem = ""; + string? lastAssistantItem = null; + do { result = await _webSocket.ReceiveAsync( @@ -166,15 +167,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); + await Task.Delay(1000); onAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { var audio = JsonSerializer.Deserialize(receivedText); - lastAssistantItem = audio?.ItemId ?? ""; + lastAssistantItem = audio?.ItemId; if (audio != null && audio.Delta != null) { + _logger.LogDebug($"{response.Type}: {receivedText}"); onModelAudioDeltaReceived(audio.Delta); } } @@ -204,16 +207,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; // handle use interuption - var truncateEvent = new + if (!string.IsNullOrEmpty(lastAssistantItem)) { - type = "conversation.item.truncate", - item_id = lastAssistantItem, - content_index = 0, - audio_end_ms = 100 - }; + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = lastAssistantItem, + content_index = 0, + audio_end_ms = 300 + }; - await SendEventToModel(truncateEvent); - onUserInterrupted(); + await SendEventToModel(truncateEvent); + onUserInterrupted(); + } } } while (!result.CloseStatus.HasValue); @@ -612,7 +618,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(output.Role, content.Transcript) { - CurrentAgentId = conn.CurrentAgentId + CurrentAgentId = conn.CurrentAgentId, + MessageId = output.Id, + MessageType = MessageTypeName.Plain }); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 1ba7120c..6f0364bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -189,7 +189,14 @@ public class TwilioService { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + if (speechPath.StartsWith("twilio/")) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + } } } var connect = new Connect(); From e0fffe191c0c8ea92e107c349a950926c1eb9a28 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 4 Mar 2025 17:25:35 -0600 Subject: [PATCH 30/84] add latest state --- .../Instructs/IInstructHook.cs | 1 + .../Instructs/InstructHookBase.cs | 5 + .../Instructs/Models/InstructLogFilter.cs | 16 ++ .../Instructs/Models/InstructResponseModel.cs | 8 + .../Loggers/Models/InstructionLogModel.cs | 24 ++ .../BotSharp.Abstraction/Models/KeyValue.cs | 14 ++ .../Repositories/IBotSharpRepository.cs | 9 + .../Utilities/StringExtensions.cs | 16 ++ .../FileRepository.Conversation.cs | 209 +++++++++++++++--- .../FileRepository/FileRepository.Log.cs | 35 +++ .../FileRepository/FileRepository.cs | 3 +- .../BotSharpLoggerExtensions.cs | 1 + .../Hooks/InstructionLogHook.cs | 96 ++++++++ src/Infrastructure/BotSharp.Logger/Using.cs | 1 + .../Controllers/InstructModeController.cs | 12 + .../Collections/ConversationDocument.cs | 1 + .../Collections/InstructionLogBetaDocument.cs | 38 ++++ .../MongoDbContext.cs | 2 + .../MongoRepository.Conversation.cs | 93 ++++++-- .../Repository/MongoRepository.Log.cs | 100 +++++++++ .../Repository/MongoRepository.cs | 6 +- 21 files changed, 637 insertions(+), 53 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs create mode 100644 src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs index 8eaee99b..c9fa1cd7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs @@ -7,4 +7,5 @@ public interface IInstructHook string SelfId { get; } Task BeforeCompletion(Agent agent, RoleDialogModel message); Task AfterCompletion(Agent agent, InstructResult result); + Task OnResponseGenerated(InstructResponseModel response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs index b59e4640..f19cca5c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs @@ -15,4 +15,9 @@ public class InstructHookBase : IInstructHook { return; } + + public virtual async Task OnResponseGenerated(InstructResponseModel response) + { + return; + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs new file mode 100644 index 00000000..d8cafcee --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructLogFilter : Pagination +{ + public List? AgentIds { get; set; } + public List? Providers { get; set; } + public List? Models { get; set; } + public List? States { get; set; } + + public static InstructLogFilter Empty() + { + return new InstructLogFilter(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs new file mode 100644 index 00000000..e2ee794d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructResponseModel +{ + public string? AgentId { get; set; } + public string Provider { get; set; } + public string Model { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs new file mode 100644 index 00000000..f77b08fe --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Loggers.Models; + +public class InstructionLogModel +{ + [JsonPropertyName("agent_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentId { get; set; } + + [JsonPropertyName("provider")] + public string Provider { get; set; } = default!; + + [JsonPropertyName("model")] + public string Model { get; set; } = default!; + + [JsonPropertyName("user_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserId { get; set; } + + [JsonPropertyName("states")] + public Dictionary States { get; set; } = []; + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs index 81286489..b1dfa734 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -13,3 +13,17 @@ public class KeyValue return $"Key: {Key}, Value: {Value}"; } } + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public T? Value { get; set; } + + public override string ToString() + { + return $"Key: {Key}, Value: {Value}"; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 235c449b..6f8de07d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; @@ -171,6 +172,14 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); #endregion + #region Instruction Log + bool SaveInstructionLogs(IEnumerable logs) + => throw new NotImplementedException(); + + PagedItems GetInstructionLogs(InstructLogFilter filter) + => throw new NotImplementedException(); + #endregion + #region Statistics BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index 65f7b78e..af222d0d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -92,4 +92,20 @@ public static class StringExtensions return JsonSerializer.Deserialize(text, options); } + + public static bool IsPrimitiveValue(this string value) + { + return int.TryParse(value, out _) || + long.TryParse(value, out _) || + double.TryParse(value, out _) || + float.TryParse(value, out _) || + bool.TryParse(value, out _) || + char.TryParse(value, out _) || + byte.TryParse(value, out _) || + sbyte.TryParse(value, out _) || + short.TryParse(value, out _) || + ushort.TryParse(value, out _) || + uint.TryParse(value, out _) || + ulong.TryParse(value, out _); + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 7bd1dd15..5769320b 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -33,13 +33,19 @@ public partial class FileRepository var stateFile = Path.Combine(dir, STATE_FILE); if (!File.Exists(stateFile)) { - File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); + File.WriteAllText(stateFile, "[]"); + } + + var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE); + if (!File.Exists(latestStateFile)) + { + File.WriteAllText(latestStateFile, "{}"); } var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); if (!File.Exists(breakpointFile)) { - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); + File.WriteAllText(breakpointFile, "[]"); } } @@ -300,14 +306,21 @@ public partial class FileRepository if (states.IsNullOrEmpty()) return; var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (string.IsNullOrEmpty(convDir)) return; + + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) { - var stateFile = Path.Combine(convDir, STATE_FILE); - if (File.Exists(stateFile)) - { - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateFile, stateStr); - } + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); + } + + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) + { + var latestStates = BuildLatestStates(states); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateFile, stateStr); } } @@ -427,25 +440,57 @@ public partial class FileRepository } // Check states - if (filter != null && !filter.States.IsNullOrEmpty()) + if (matched && filter != null && !filter.States.IsNullOrEmpty()) { - var stateFile = Path.Combine(d, STATE_FILE); - var convStates = CollectConversationStates(stateFile); - foreach (var pair in filter.States) + var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE); + var convStates = CollectConversationLatestStates(latestStateFile); + + if (convStates.IsNullOrEmpty()) { - if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - - var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); - if (foundState == null) + matched = false; + } + else + { + foreach (var pair in filter.States) { - matched = false; - break; - } + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - if (!string.IsNullOrWhiteSpace(pair.Value)) - { - var curValue = foundState.Values.LastOrDefault()?.Data; - matched = matched && pair.Value.IsEqualTo(curValue); + var components = pair.Key.Split(".").ToList(); + var primaryKey = components[0]; + if (convStates.TryGetValue(primaryKey, out var doc)) + { + var elem = doc.RootElement.GetProperty("data"); + if (components.Count < 2) + { + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + if (elem.ValueKind == JsonValueKind.Array) + { + matched = elem.EnumerateArray().Select(x => x.ToString()).Contains(pair.Value); + } + else if (elem.ValueKind == JsonValueKind.String) + { + matched = elem.GetString() == pair.Value; + } + else + { + matched = elem.GetRawText() == pair.Value; + } + } + } + else + { + var paths = components.Where((_, idx) => idx > 0); + var found = FindState(elem, paths, pair.Value); + matched = found != null; + } + } + else + { + matched = false; + } + + if (!matched) break; } } } @@ -575,8 +620,9 @@ public partial class FileRepository // Handle truncated states var refTime = dialogs.ElementAt(foundIdx).MetaData.CreatedTime; var stateDir = Path.Combine(convDir, STATE_FILE); + var latestStateDir = Path.Combine(convDir, CONV_LATEST_STATE_FILE); var states = CollectConversationStates(stateDir); - isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); + isSaved = HandleTruncatedStates(stateDir, latestStateDir, states, messageId, refTime); // Handle truncated breakpoints var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); @@ -703,7 +749,7 @@ public partial class FileRepository return isSaved; } - private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) + private bool HandleTruncatedStates(string stateDir, string latestStateDir, List states, string refMsgId, DateTime refTime) { var truncatedStates = new List(); foreach (var state in states) @@ -724,6 +770,10 @@ public partial class FileRepository } var isSaved = SaveTruncatedStates(stateDir, truncatedStates); + if (isSaved) + { + SaveTruncatedLatestStates(latestStateDir, truncatedStates); + } return isSaved; } @@ -794,6 +844,17 @@ public partial class FileRepository return true; } + private bool SaveTruncatedLatestStates(string latestStateDir, List states) + { + if (string.IsNullOrEmpty(latestStateDir) || states == null) return false; + if (!File.Exists(latestStateDir)) File.Create(latestStateDir); + + var latestStates = BuildLatestStates(states); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateDir, stateStr); + return true; + } + private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) { if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; @@ -804,22 +865,98 @@ public partial class FileRepository return true; } - private string? EncodeText(string? text) + private Dictionary CollectConversationLatestStates(string latestStateDir) { - if (string.IsNullOrEmpty(text)) return text; + if (string.IsNullOrEmpty(latestStateDir) || !File.Exists(latestStateDir)) return []; - var bytes = Encoding.UTF8.GetBytes(text); - var encoded = Convert.ToBase64String(bytes); - return encoded; + var str = File.ReadAllText(latestStateDir); + var states = JsonSerializer.Deserialize>(str, _options); + return states ?? []; } - private string? DecodeText(string? text) + private Dictionary BuildLatestStates(List states) { - if (string.IsNullOrEmpty(text)) return text; + var endNodes = new Dictionary(); + foreach (var pair in states) + { + var value = pair.Values?.LastOrDefault(); + if (value == null || !value.Active) continue; - var decoded = Convert.FromBase64String(text); - var origin = Encoding.UTF8.GetString(decoded); - return origin; + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _options); + var json = JsonDocument.Parse(jsonStr); + endNodes[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = value.Data }, _options); + var json = JsonDocument.Parse(str); + endNodes[pair.Key] = json; + } + } + + return endNodes; + } + + private JsonElement? FindState(JsonElement? root, IEnumerable paths, string? targetValue) + { + JsonElement? elem = null; + + if (root == null || paths.IsNullOrEmpty()) + { + return elem; + } + + elem = root; + for (int i = 0; i < paths.Count(); i++) + { + var field = paths.ElementAt(i); + if (elem.Value.ValueKind == JsonValueKind.Array) + { + if (elem.Value.EnumerateArray().IsNullOrEmpty()) + { + elem = null; + break; + } + else + { + foreach (var item in elem.Value.EnumerateArray()) + { + var subPaths = paths.Where((_, idx) => idx >= i); + elem = FindState(item, subPaths, targetValue); + if (elem != null) + { + return elem; + } + } + } + } + else if (elem.Value.TryGetProperty(field, out var prop)) + { + elem = prop; + } + } + + if (elem != null && !string.IsNullOrWhiteSpace(targetValue)) + { + if (elem.Value.ValueKind == JsonValueKind.Array) + { + var isInArray = elem.Value.EnumerateArray().Select(x => x.ToString()).Contains(targetValue); + return isInArray ? elem : null; + } + else if ((elem.Value.ValueKind == JsonValueKind.String && elem.Value.GetString() == targetValue) + || (elem.Value.ValueKind != JsonValueKind.String && elem.Value.GetRawText() == targetValue)) + { + return elem; + } + else + { + return null; + } + } + + return elem; } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index f82cb6fa..9f7ca305 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using System.IO; @@ -122,6 +123,40 @@ namespace BotSharp.Core.Repository } #endregion + #region Instruction Log + public bool SaveInstructionLogs(IEnumerable logs) + { + if (logs.IsNullOrEmpty()) return false; + + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + Directory.CreateDirectory(baseDir); + } + + foreach (var log in logs) + { + var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.log"); + var text = JsonSerializer.Serialize(log, _options); + File.WriteAllText(file, text); + } + return true; + } + + public PagedItems GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + return new PagedItems + { + + }; + } + #endregion + #region Private methods private int GetNextLogIndex(string logDir, string id) { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 78e90e4b..f26e8bc0 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -6,7 +6,6 @@ using BotSharp.Abstraction.Users.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; - namespace BotSharp.Core.Repository; public partial class FileRepository : IBotSharpRepository @@ -34,6 +33,7 @@ public partial class FileRepository : IBotSharpRepository private const string DIALOG_FILE = "dialogs.json"; private const string STATE_FILE = "state.json"; private const string BREAKPOINT_FILE = "breakpoint.json"; + private const string CONV_LATEST_STATE_FILE = "latest-state.json"; private const string TRANSLATION_MEMORY_FILE = "memory.json"; private const string USERS_FOLDER = "users"; @@ -54,6 +54,7 @@ public partial class FileRepository : IBotSharpRepository private const string STATS_FILE = "stats.json"; private const string CRON_FILE = "cron.json"; + private const string INSTRUCTION_LOG_FOLDER = "instruction-logs"; public FileRepository( IServiceProvider services, diff --git a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs index 6ccd70ac..0404ceb4 100644 --- a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs +++ b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs @@ -15,6 +15,7 @@ public static class BotSharpLoggerExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); return services; } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs new file mode 100644 index 00000000..03e1f8ff --- /dev/null +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -0,0 +1,96 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Users; +using System.Text.Json; + +namespace BotSharp.Logger.Hooks; + +public class InstructionLogHook : InstructHookBase +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly IUserIdentity _user; + + public InstructionLogHook( + IServiceProvider services, + ILogger logger, + IUserIdentity user, + BotSharpOptions options) + { + _services = services; + _logger = logger; + _user = user; + _options = options; + } + + public override async Task OnResponseGenerated(InstructResponseModel response) + { + if (response == null) return; + + var db = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + var user = db.GetUserById(_user.Id); + db.SaveInstructionLogs(new List + { + new InstructionLogModel + { + AgentId = response.AgentId, + Provider = response.Provider, + Model = response.Model, + States = state.GetStates(), + UserId = user?.Id + } + }); + return; + } + + private Dictionary CollectStates() + { + var res = new Dictionary(); + var state = _services.GetRequiredService(); + var curStates = state.GetStates(); + + curStates["test"] = JsonSerializer.Serialize(new + { + Number = "789", + Dummy = new + { + Id = 123, + Name = "name", + Score = 12.123 + }, + Items = new List + { + new + { + Name = "image", + Label = "before-service", + Attribute = new + { + Location = "Chicago", + Time = "afternoon" + } + }, + new + { + Name = "pdf", + Label = "after-service", + Attribute = new + { + Location = "New York", + Time = "morning" + } + }, + }, + Lists = new List + { + "abc", + "bcd" + } + }, _options.JsonSerializerOptions); + return curStates; + } +} diff --git a/src/Infrastructure/BotSharp.Logger/Using.cs b/src/Infrastructure/BotSharp.Logger/Using.cs index 5115e776..9c975007 100644 --- a/src/Infrastructure/BotSharp.Logger/Using.cs +++ b/src/Infrastructure/BotSharp.Logger/Using.cs @@ -12,4 +12,5 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Instructs; global using BotSharp.Logger.Hooks; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index dbce9598..45054223 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -75,6 +75,18 @@ public class InstructModeController : ControllerBase { new RoleDialogModel(AgentRole.User, input.Text) }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = input.AgentId, + Provider = input.Provider, + Model = input.Model + }); + } + return message.Content; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index ace9472d..8fb85239 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -14,4 +14,5 @@ public class ConversationDocument : MongoBase public List Tags { get; set; } = []; public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } + public Dictionary LatestStates { get; set; } = new(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs new file mode 100644 index 00000000..615e93af --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs @@ -0,0 +1,38 @@ +using BotSharp.Abstraction.Loggers.Models; +using System.Text.Json; + +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class InstructionLogBetaDocument : MongoBase +{ + public string? AgentId { get; set; } + public string Provider { get; set; } = default!; + public string Model { get; set; } = default!; + public string? UserId { get; set; } + public Dictionary States { get; set; } = new(); + public DateTime CreatedTime { get; set; } + + public static InstructionLogBetaDocument ToMongoModel(InstructionLogModel log) + { + return new InstructionLogBetaDocument + { + AgentId = log.AgentId, + Provider = log.Provider, + Model = log.Model, + UserId = log.UserId, + CreatedTime = log.CreatedTime + }; + } + + public static InstructionLogModel ToDomainModel(InstructionLogBetaDocument log) + { + return new InstructionLogModel + { + AgentId = log.AgentId, + Provider = log.Provider, + Model = log.Model, + UserId = log.UserId, + CreatedTime = log.CreatedTime + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 4aa82b2e..9adc04b9 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -193,4 +193,6 @@ public class MongoDbContext public IMongoCollection GlobalStatistics => GetCollectionOrCreate("GlobalStatistics"); + public IMongoCollection InstructionLogs + => GetCollectionOrCreate("InstructionLogsBeta"); } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index a78db60c..1c0eeb2f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -23,7 +24,8 @@ public partial class MongoRepository Status = conversation.Status, Tags = conversation.Tags ?? new(), CreatedTime = utcNow, - UpdatedTime = utcNow + UpdatedTime = utcNow, + LatestStates = [] }; var dialogDoc = new ConversationDialogDocument @@ -72,8 +74,9 @@ public partial class MongoRepository var cronDeleted = _dc.CrontabItems.DeleteMany(conbTabItems); var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0 - || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0; + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 + || promptLogDeleted.DeletedCount > 0 || contentLogDeleted.DeletedCount > 0 + || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0; } [SideCar] @@ -266,6 +269,14 @@ public partial class MongoRepository .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); + + // Update latest states + var endNodes = BuildLatestStates(saveStates); + var filter = Builders.Filter.Eq(x => x.Id, conversationId); + var update = Builders.Update.Set(x => x.LatestStates, endNodes) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Conversations.UpdateOne(filter, update); } public void UpdateConversationStatus(string conversationId, string status) @@ -371,21 +382,47 @@ public partial class MongoRepository } // Filter states - var stateFilters = new List>(); if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty()) { foreach (var pair in filter.States) { - var elementFilters = new List> { Builders.Filter.Eq(x => x.Key, pair.Key) }; - if (!string.IsNullOrEmpty(pair.Value)) - { - elementFilters.Add(Builders.Filter.Eq("Values.Data", pair.Value)); - } - stateFilters.Add(Builders.Filter.ElemMatch(x => x.States, Builders.Filter.And(elementFilters))); - } + if (string.IsNullOrWhiteSpace(pair.Key)) continue; - var targetConvIds = _dc.ConversationStates.Find(Builders.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList(); - convFilters.Add(convBuilder.In(x => x.Id, targetConvIds)); + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "LatestStates"); + var formattedKey = string.Join(".", keys); + + if (string.IsNullOrWhiteSpace(pair.Value)) + { + convFilters.Add(convBuilder.Exists(formattedKey)); + } + else if (bool.TryParse(pair.Value, out var boolValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, boolValue)); + } + else if (int.TryParse(pair.Value, out var intValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, intValue)); + } + else if (decimal.TryParse(pair.Value, out var decimalValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, decimalValue)); + } + else if (float.TryParse(pair.Value, out var floatValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, floatValue)); + } + else if (double.TryParse(pair.Value, out var doubleValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, doubleValue)); + } + else + { + convFilters.Add(convBuilder.Eq(formattedKey, pair.Value)); + } + } } // Sort and paginate @@ -527,6 +564,7 @@ public partial class MongoRepository var stateFilter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault(); + var endNodes = new Dictionary(); if (foundStates != null) { // Truncate states @@ -550,6 +588,7 @@ public partial class MongoRepository truncatedStates.Add(state); } foundStates.States = truncatedStates; + endNodes = BuildLatestStates(truncatedStates); } // Truncate breakpoints @@ -573,6 +612,7 @@ public partial class MongoRepository // Update conversation var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.LatestStates, endNodes) .Set(x => x.DialogCount, truncatedDialogs.Count); _dc.Conversations.UpdateOne(convFilter, updateConv); @@ -621,8 +661,6 @@ public partial class MongoRepository return keys; } - - private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); @@ -640,4 +678,29 @@ public partial class MongoRepository return pascalCase.ToString(); } + + private Dictionary BuildLatestStates(List states) + { + var endNodes = new Dictionary(); + foreach (var pair in states) + { + var value = pair.Values?.LastOrDefault(); + if (value == null || !value.Active) continue; + + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + endNodes[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = value.Data }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(str); + endNodes[pair.Key] = json; + } + } + + return endNodes; + } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index e4c64527..c90960dd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; +using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -110,4 +112,102 @@ public partial class MongoRepository return logs; } #endregion + + #region Instruction Log + public bool SaveInstructionLogs(IEnumerable logs) + { + if (logs.IsNullOrEmpty()) return false; + + var docs = new List(); + foreach (var log in logs) + { + var doc = InstructionLogBetaDocument.ToMongoModel(log); + foreach (var pair in log.States) + { + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + doc.States[pair.Key] = json; + } + catch + { + var jsonStr = JsonSerializer.Serialize(new { Data = pair.Value }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + doc.States[pair.Key] = json; + } + } + docs.Add(doc); + } + + _dc.InstructionLogs.InsertMany(docs); + return true; + } + + public PagedItems GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var builder = Builders.Filter; + var filters = new List>() { builder.Empty }; + + // Filter logs + if (!filter.AgentIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + } + if (!filter.Providers.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.Provider, filter.Providers)); + } + if (!filter.Models.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.Model, filter.Models)); + } + + if (!filter.States.IsNullOrEmpty()) + { + foreach (var pair in filter.States) + { + if (string.IsNullOrWhiteSpace(pair.Key)) continue; + + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "States"); + var formattedKey = string.Join(".", keys); + + if (pair.Value == null) + { + filters.Add(builder.Exists(formattedKey)); + } + else + { + filters.Add(builder.Eq(formattedKey, pair.Value)); + } + } + } + + var filterDef = builder.And(filters); + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); + var count = _dc.InstructionLogs.CountDocuments(filterDef); + + var logs = docs.Select(x => + { + var log = InstructionLogBetaDocument.ToDomainModel(x); + log.States = x.States.ToDictionary(x => x.Key, x => x.Value.GetElement("data").Value.ToString() ?? string.Empty); + return log; + }).ToList(); + + return new PagedItems + { + Items = logs, + Count = (int)count + }; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 258c1883..c91a291a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Options; using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -7,16 +8,19 @@ public partial class MongoRepository : IBotSharpRepository private readonly MongoDbContext _dc; private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly BotSharpOptions _botSharpOptions; private UpdateOptions _options; public MongoRepository( MongoDbContext dc, IServiceProvider services, - ILogger logger) + ILogger logger, + BotSharpOptions botSharpOptions) { _dc = dc; _services = services; _logger = logger; + _botSharpOptions = botSharpOptions; _options = new UpdateOptions { IsUpsert = true, From 263ec44517a6030db533320c7934adc1685d4a85 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 4 Mar 2025 23:25:12 -0600 Subject: [PATCH 31/84] refine --- .../FileRepository.Conversation.cs | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 5769320b..08a534d4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -464,9 +464,15 @@ public partial class FileRepository { if (!string.IsNullOrWhiteSpace(pair.Value)) { - if (elem.ValueKind == JsonValueKind.Array) + if (elem.ValueKind == JsonValueKind.Null) { - matched = elem.EnumerateArray().Select(x => x.ToString()).Contains(pair.Value); + matched = false; + } + else if (elem.ValueKind == JsonValueKind.Array) + { + matched = elem.EnumerateArray().Where(x => x.ValueKind != JsonValueKind.Null) + .Select(x => x.ToString()) + .Any(x => x == pair.Value); } else if (elem.ValueKind == JsonValueKind.String) { @@ -901,25 +907,21 @@ public partial class FileRepository private JsonElement? FindState(JsonElement? root, IEnumerable paths, string? targetValue) { - JsonElement? elem = null; + var elem = root; - if (root == null || paths.IsNullOrEmpty()) + if (elem == null || paths.IsNullOrEmpty()) { - return elem; + return null; } - elem = root; for (int i = 0; i < paths.Count(); i++) { + if (elem == null) return null; + var field = paths.ElementAt(i); if (elem.Value.ValueKind == JsonValueKind.Array) { - if (elem.Value.EnumerateArray().IsNullOrEmpty()) - { - elem = null; - break; - } - else + if (!elem.Value.EnumerateArray().IsNullOrEmpty()) { foreach (var item in elem.Value.EnumerateArray()) { @@ -931,18 +933,32 @@ public partial class FileRepository } } } + else + { + return null; + } } - else if (elem.Value.TryGetProperty(field, out var prop)) + else if (elem.Value.ValueKind == JsonValueKind.Object && elem.Value.TryGetProperty(field, out var prop)) { elem = prop; } + else + { + return null; + } } if (elem != null && !string.IsNullOrWhiteSpace(targetValue)) { - if (elem.Value.ValueKind == JsonValueKind.Array) + if (elem.Value.ValueKind == JsonValueKind.Null) { - var isInArray = elem.Value.EnumerateArray().Select(x => x.ToString()).Contains(targetValue); + return null; + } + else if (elem.Value.ValueKind == JsonValueKind.Array) + { + var isInArray = elem.Value.EnumerateArray().Where(x => x.ValueKind != JsonValueKind.Null) + .Select(x => x.ToString()) + .Any(x => x == targetValue); return isInArray ? elem : null; } else if ((elem.Value.ValueKind == JsonValueKind.String && elem.Value.GetString() == targetValue) From d78c77141b4f534fc0cc135e305413bac7fa4e9a Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Wed, 5 Mar 2025 21:48:06 +0800 Subject: [PATCH 32/84] 1.hotfix timestamp,mark and interruption handling 2.Hotfix message persistence --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 19 +- .../Services/Stream/TwilioStreamMiddleware.cs | 216 ++++++++++++++---- 2 files changed, 174 insertions(+), 61 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 36397b0a..17c5162a 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -160,19 +160,16 @@ public class RealtimeHub : IRealtimeHub await completer.TriggerModelInference("Reply based on the function's output."); } } - else + // append output audio transcript to conversation + storage.Append(conn.ConversationId, message); + dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) { - // append output audio transcript to conversation - storage.Append(conn.ConversationId, message); - dialogs.Add(message); + hook.SetAgent(agent) + .SetConversation(conversation); - foreach (var hook in hookProvider.HooksOrderByPriority) - { - hook.SetAgent(agent) - .SetConversation(conversation); - - await hook.OnResponseGenerated(message); - } + await hook.OnResponseGenerated(message); } } }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 492b5dcf..08966396 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -5,20 +5,25 @@ using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; using System.Net.WebSockets; +using System.Text.Json; +using System.Collections.Concurrent; +using System.Text; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services.Stream; /// -/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js +/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js /// public class TwilioStreamMiddleware { private readonly RequestDelegate _next; + private readonly ILogger _logger; - public TwilioStreamMiddleware(RequestDelegate next) + public TwilioStreamMiddleware(RequestDelegate next, ILogger logger) { _next = next; + _logger = logger; } public async Task Invoke(HttpContext httpContext) @@ -43,15 +48,23 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); - + var convService = services.GetRequiredService(); + + // Session state var conn = new RealtimeHubConnection { ConversationId = conversationId }; - // load conversation and state - var convService = services.GetRequiredService(); - convService.SetConversationId(conversationId, []); + // Variables for timestamp and interruption handling + string streamSid = null; + long latestMediaTimestamp = 0; + string lastAssistantItem = null; + var markQueue = new ConcurrentQueue(); + long? responseStartTimestampTwilio = null; + + // Load session and state + convService.SetConversationId(conversationId, new List()); var hooks = services.GetServices(); foreach (var hook in hooks) { @@ -59,56 +72,159 @@ public class TwilioStreamMiddleware } convService.States.Save(); - await hub.Listen(webSocket, (receivedText) => + // Set up event handlers + conn.OnModelMessageReceived = message => { - var response = JsonSerializer.Deserialize(receivedText); - conn.StreamId = response.StreamSid; - conn.Event = response.Event switch + // Record last assistant item ID for interruption handling + if (!string.IsNullOrEmpty(conn.StreamId)) { - "start" => "user_connected", - "media" => "user_data_received", - "stop" => "user_disconnected", - _ => response.Event + lastAssistantItem = conn.StreamId; + } + + // If this is the first delta of a new response, set the start timestamp + if (!responseStartTimestampTwilio.HasValue) + { + responseStartTimestampTwilio = latestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {responseStartTimestampTwilio}ms"); + } + + // Add mark to queue + markQueue.Enqueue("responsePart"); + + return new + { + @event = "media", + streamSid = conn.StreamId, + media = new { payload = message } }; + }; - if (string.IsNullOrEmpty(conn.Event)) + conn.OnModelAudioResponseDone = () => + { + return new { + @event = "mark", + streamSid = conn.StreamId, + mark = new { name = "responsePart" } + }; + }; + + conn.OnModelUserInterrupted = () => + { + // Reset states + markQueue.Clear(); + lastAssistantItem = null; + responseStartTimestampTwilio = null; + + return new + { + @event = "clear", + streamSid = conn.StreamId + }; + }; + + try + { + await hub.Listen(webSocket, receivedText => + { + var response = JsonSerializer.Deserialize(receivedText); + if (response == null) + { + _logger.LogWarning("Failed to parse received WebSocket message"); + return conn; + } + + conn.StreamId = response.StreamSid; + + switch (response.Event) + { + case "start": + conn.Event = "user_connected"; + streamSid = response.StreamSid; + _logger.LogInformation($"Incoming stream started: {streamSid}"); + + // Reset start and media timestamps + responseStartTimestampTwilio = null; + latestMediaTimestamp = 0; + + var startResponse = JsonSerializer.Deserialize(receivedText); + if (startResponse?.Body?.CustomParameters != null) + { + conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); + } + break; + + case "media": + conn.Event = "user_data_received"; + var mediaResponse = JsonSerializer.Deserialize(receivedText); + if (mediaResponse?.Body != null) + { + conn.Data = mediaResponse.Body.Payload; + + // Update latest media timestamp + if (long.TryParse(mediaResponse.Body.Timestamp, out latestMediaTimestamp)) + { + _logger.LogDebug($"Received media message with timestamp: {latestMediaTimestamp}ms"); + } + + // Check if user started speaking (interruption handling) + if (markQueue.Count > 0 && responseStartTimestampTwilio.HasValue && + !string.IsNullOrEmpty(lastAssistantItem)) + { + // Detect voice activity - more complex logic can be added here + // e.g., check audio energy levels or use VAD (Voice Activity Detection) + + // If voice activity detected, handle interruption + if (ShouldHandleInterruption(mediaResponse.Body.Payload)) + { + conn.Event = "user_interrupted"; + long elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio.Value; + _logger.LogDebug($"Calculating elapsed time for truncation: {latestMediaTimestamp} - {responseStartTimestampTwilio} = {elapsedTime}ms"); + } + } + } + break; + + case "mark": + // Handle mark event + if (markQueue.TryDequeue(out _)) + { + _logger.LogDebug("Processing mark event, removing one mark from queue"); + } + break; + + case "stop": + conn.Event = "user_disconnected"; + break; + + default: + _logger.LogInformation($"Received non-media event: {response.Event}"); + break; + } + return conn; - } + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error in WebSocket communication"); + } + } - conn.OnModelMessageReceived = message => - new - { - @event = "media", - streamSid = response.StreamSid, - media = new { payload = message } - }; - conn.OnModelAudioResponseDone = () => - new - { - @event = "mark", - streamSid = response.StreamSid, - mark = new { name = "responsePart" } - }; - conn.OnModelUserInterrupted = () => - new - { - @event = "clear", - streamSid = response.StreamSid - }; - - if (response.Event == "start") - { - var startResponse = JsonSerializer.Deserialize(receivedText); - conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); - } - else if (response.Event == "media") - { - var mediaResponse = JsonSerializer.Deserialize(receivedText); - conn.Data = mediaResponse.Body.Payload; - } - - return conn; - }); + // Simple interruption detection logic - can be extended as needed + private bool ShouldHandleInterruption(string audioPayload) + { + // Here should implement actual voice activity detection logic + // e.g., analyze audio energy levels or use VAD algorithm + + // Simple example - should be replaced with real detection logic in production + if (!string.IsNullOrEmpty(audioPayload)) + { + // Check if audio payload contains sufficient energy + // This is just a placeholder - needs actual VAD implementation + return false; // Default to false to avoid false interruptions + } + + return false; } } From d69f19bd414b21d03ddb72979de8f46204b60f2b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 5 Mar 2025 12:29:28 -0600 Subject: [PATCH 33/84] refactor realtime code. --- .../Realtime/Models/RealtimeHubConnection.cs | 6 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 44 +++- .../Translation/TranslationResponseHook.cs | 6 + .../BotSharp.Logger/Hooks/VerboseLogHook.cs | 2 +- .../Models/Realtime/RealtimeSessionRequest.cs | 21 +- .../Realtime/RealTimeCompletionProvider.cs | 51 ++-- .../Services/Stream/TwilioStreamMiddleware.cs | 217 +++++------------- 7 files changed, 160 insertions(+), 187 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index ec521637..61a5cfe4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -1,9 +1,15 @@ +using System.Collections.Concurrent; + namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; + public string? LastAssistantItem { get; set; } = null!; + public long LatestMediaTimestamp { get; set; } + public long? ResponseStartTimestamp { get; set; } + public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 17c5162a..6517ef3c 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -112,8 +112,28 @@ public class RealtimeHub : IRealtimeHub }, onModelAudioDeltaReceived: async audioDeltaData => { + // If this is the first delta of a new response, set the start timestamp + if (!conn.ResponseStartTimestamp.HasValue) + { + conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms"); + } + var data = conn.OnModelMessageReceived(audioDeltaData); await SendEventToUser(userWebSocket, data); + + // Send mark messages to Media Streams so we know if and when AI response playback is finished + if (!string.IsNullOrEmpty(conn.StreamId)) + { + var markEvent = new + { + @event = "mark", + streamSid = conn.StreamId, + mark = new { name = "responsePart" } + }; + await SendEventToUser(userWebSocket, markEvent); + conn.MarkQueue.Enqueue("responsePart"); + } }, onModelAudioResponseDone: async () => { @@ -160,16 +180,19 @@ public class RealtimeHub : IRealtimeHub await completer.TriggerModelInference("Reply based on the function's output."); } } - // append output audio transcript to conversation - storage.Append(conn.ConversationId, message); - dialogs.Add(message); - - foreach (var hook in hookProvider.HooksOrderByPriority) + else { - hook.SetAgent(agent) - .SetConversation(conversation); + // append output audio transcript to conversation + storage.Append(conn.ConversationId, message); + dialogs.Add(message); - await hook.OnResponseGenerated(message); + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnResponseGenerated(message); + } } } }, @@ -193,6 +216,11 @@ public class RealtimeHub : IRealtimeHub }, onUserInterrupted: async () => { + // Reset states + conn.MarkQueue.Clear(); + conn.LastAssistantItem = null; + conn.ResponseStartTimestamp = null; + var data = conn.OnModelUserInterrupted(); await SendEventToUser(userWebSocket, data); }); diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs index 3df585f2..d5e61c12 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Translation; using System; @@ -26,6 +27,11 @@ namespace BotSharp.Logger.Hooks { return; } + + if (_states.GetState("channel") == ConversationChannel.Phone) + { + return; + } // Handle multi-language for output var agentService = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index 9740bb20..7affc3a4 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -35,7 +35,7 @@ public class VerboseLogHook : IContentGeneratingHook public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) { - if (!_convSettings.ShowVerboseLog) return; + if (!_convSettings.ShowVerboseLog || string.IsNullOrEmpty(tokenStats.Prompt)) return; var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs index b2125e99..cabf3d30 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -1,8 +1,27 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionCreationRequest : RealtimeSessionBody +public class RealtimeSessionCreationRequest { + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Model { get; set; } = null!; + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = ["audio", "text"]; + + [JsonPropertyName("instructions")] + public string Instructions { get; set; } = null!; + + [JsonPropertyName("tool_choice")] + public string ToolChoice { get; set; } = "auto"; + + [JsonPropertyName("tools")] + public FunctionDef[] Tools { get; set; } = []; + + [JsonPropertyName("turn_detection")] + public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); } /// diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 7abefbbb..5d7b2e80 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -128,9 +128,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var buffer = new byte[1024 * 256]; + var buffer = new byte[1024 * 16]; WebSocketReceiveResult result; - string? lastAssistantItem = null; do { @@ -173,7 +172,16 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.audio.delta") { var audio = JsonSerializer.Deserialize(receivedText); - lastAssistantItem = audio?.ItemId; + // Record last assistant item ID for interruption handling + if (conn.ResponseStartTimestamp.HasValue) + { + conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; + } + + if (!string.IsNullOrEmpty(conn.StreamId)) + { + conn.LastAssistantItem = audio?.ItemId; + } if (audio != null && audio.Delta != null) { @@ -205,19 +213,24 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "input_audio_buffer.speech_started") { - // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; - // handle use interuption - if (!string.IsNullOrEmpty(lastAssistantItem)) + // Handle user interuption + if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) { - var truncateEvent = new - { - type = "conversation.item.truncate", - item_id = lastAssistantItem, - content_index = 0, - audio_end_ms = 300 - }; + var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; + + if (!string.IsNullOrEmpty(conn.LastAssistantItem)) + { + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = conn.LastAssistantItem, + content_index = 0, + audio_end_ms = elapsedTime + }; + + await SendEventToModel(truncateEvent); + } - await SendEventToModel(truncateEvent); onUserInterrupted(); } } @@ -256,6 +269,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var args = new RealtimeSessionCreationRequest { + Model = _model, Instructions = instruction, ToolChoice = "auto", Tools = options.Tools.Select(x => @@ -271,7 +285,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, args.Model); + var settings = settingsService.GetSetting(Provider, args.Model ?? _model); var api = _services.GetRequiredService(); var session = await api.GetSessionAsync(args, settings.ApiKey); @@ -318,12 +332,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), + Temperature = Math.Max(options.Temperature ?? 0f, 0.8f), MaxResponseOutputTokens = 512, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.8f, - SilenceDuration = 800 + Threshold = 0.5f, + PrefixPadding = 300, + SilenceDuration = 500 } } }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 08966396..fbb8d794 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -37,7 +37,14 @@ public class TwilioStreamMiddleware var services = httpContext.RequestServices; var conversationId = request.Path.Value.Split("/").Last(); using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); - await HandleWebSocket(services, conversationId, webSocket); + try + { + await HandleWebSocket(services, conversationId, webSocket); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error in WebSocket communication: {ex.Message} for conversation {conversationId}"); + } return; } } @@ -48,23 +55,15 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); - var convService = services.GetRequiredService(); - // Session state var conn = new RealtimeHubConnection { ConversationId = conversationId }; - // Variables for timestamp and interruption handling - string streamSid = null; - long latestMediaTimestamp = 0; - string lastAssistantItem = null; - var markQueue = new ConcurrentQueue(); - long? responseStartTimestampTwilio = null; - - // Load session and state - convService.SetConversationId(conversationId, new List()); + // load conversation and state + var convService = services.GetRequiredService(); + convService.SetConversationId(conversationId, []); var hooks = services.GetServices(); foreach (var hook in hooks) { @@ -72,159 +71,59 @@ public class TwilioStreamMiddleware } convService.States.Save(); - // Set up event handlers - conn.OnModelMessageReceived = message => + await hub.Listen(webSocket, (receivedText) => { - // Record last assistant item ID for interruption handling - if (!string.IsNullOrEmpty(conn.StreamId)) + var response = JsonSerializer.Deserialize(receivedText); + conn.StreamId = response.StreamSid; + conn.Event = response.Event switch { - lastAssistantItem = conn.StreamId; - } - - // If this is the first delta of a new response, set the start timestamp - if (!responseStartTimestampTwilio.HasValue) - { - responseStartTimestampTwilio = latestMediaTimestamp; - _logger.LogDebug($"Setting start timestamp for new response: {responseStartTimestampTwilio}ms"); - } - - // Add mark to queue - markQueue.Enqueue("responsePart"); - - return new - { - @event = "media", - streamSid = conn.StreamId, - media = new { payload = message } + "start" => "user_connected", + "media" => "user_data_received", + "stop" => "user_disconnected", + _ => response.Event }; - }; - conn.OnModelAudioResponseDone = () => - { - return new + if (string.IsNullOrEmpty(conn.Event)) { - @event = "mark", - streamSid = conn.StreamId, - mark = new { name = "responsePart" } - }; - }; - - conn.OnModelUserInterrupted = () => - { - // Reset states - markQueue.Clear(); - lastAssistantItem = null; - responseStartTimestampTwilio = null; - - return new - { - @event = "clear", - streamSid = conn.StreamId - }; - }; - - try - { - await hub.Listen(webSocket, receivedText => - { - var response = JsonSerializer.Deserialize(receivedText); - if (response == null) - { - _logger.LogWarning("Failed to parse received WebSocket message"); - return conn; - } - - conn.StreamId = response.StreamSid; - - switch (response.Event) - { - case "start": - conn.Event = "user_connected"; - streamSid = response.StreamSid; - _logger.LogInformation($"Incoming stream started: {streamSid}"); - - // Reset start and media timestamps - responseStartTimestampTwilio = null; - latestMediaTimestamp = 0; - - var startResponse = JsonSerializer.Deserialize(receivedText); - if (startResponse?.Body?.CustomParameters != null) - { - conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); - } - break; - - case "media": - conn.Event = "user_data_received"; - var mediaResponse = JsonSerializer.Deserialize(receivedText); - if (mediaResponse?.Body != null) - { - conn.Data = mediaResponse.Body.Payload; - - // Update latest media timestamp - if (long.TryParse(mediaResponse.Body.Timestamp, out latestMediaTimestamp)) - { - _logger.LogDebug($"Received media message with timestamp: {latestMediaTimestamp}ms"); - } - - // Check if user started speaking (interruption handling) - if (markQueue.Count > 0 && responseStartTimestampTwilio.HasValue && - !string.IsNullOrEmpty(lastAssistantItem)) - { - // Detect voice activity - more complex logic can be added here - // e.g., check audio energy levels or use VAD (Voice Activity Detection) - - // If voice activity detected, handle interruption - if (ShouldHandleInterruption(mediaResponse.Body.Payload)) - { - conn.Event = "user_interrupted"; - long elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio.Value; - _logger.LogDebug($"Calculating elapsed time for truncation: {latestMediaTimestamp} - {responseStartTimestampTwilio} = {elapsedTime}ms"); - } - } - } - break; - - case "mark": - // Handle mark event - if (markQueue.TryDequeue(out _)) - { - _logger.LogDebug("Processing mark event, removing one mark from queue"); - } - break; - - case "stop": - conn.Event = "user_disconnected"; - break; - - default: - _logger.LogInformation($"Received non-media event: {response.Event}"); - break; - } - return conn; - }); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error in WebSocket communication"); - } - } + } - // Simple interruption detection logic - can be extended as needed - private bool ShouldHandleInterruption(string audioPayload) - { - // Here should implement actual voice activity detection logic - // e.g., analyze audio energy levels or use VAD algorithm - - // Simple example - should be replaced with real detection logic in production - if (!string.IsNullOrEmpty(audioPayload)) - { - // Check if audio payload contains sufficient energy - // This is just a placeholder - needs actual VAD implementation - return false; // Default to false to avoid false interruptions - } - - return false; + conn.OnModelMessageReceived = message => + new + { + @event = "media", + streamSid = response.StreamSid, + media = new { payload = message } + }; + conn.OnModelAudioResponseDone = () => + new + { + @event = "mark", + streamSid = response.StreamSid, + mark = new { name = "responsePart" } + }; + conn.OnModelUserInterrupted = () => + new + { + @event = "clear", + streamSid = response.StreamSid + }; + + if (response.Event == "start") + { + var startResponse = JsonSerializer.Deserialize(receivedText); + conn.LatestMediaTimestamp = 0; + conn.ResponseStartTimestamp = null; + conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); + } + else if (response.Event == "media") + { + var mediaResponse = JsonSerializer.Deserialize(receivedText); + conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp); + conn.Data = mediaResponse.Body.Payload; + } + + return conn; + }); } } From 9c50824369c6e55674f8389f6167cb3dda144ae4 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 5 Mar 2025 16:57:34 -0600 Subject: [PATCH 34/84] Support DTMF in realtime --- .../Realtime/Models/RealtimeHubConnection.cs | 1 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 53 +++++++++++++++++-- .../Realtime/RealTimeCompletionProvider.cs | 5 +- .../Models/Stream/StreamEventDtmfResponse.cs | 17 ++++++ .../Services/Stream/TwilioStreamMiddleware.cs | 14 +++++ 5 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 61a5cfe4..7feaf23c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -9,6 +9,7 @@ public class RealtimeHubConnection public string? LastAssistantItem { get; set; } = null!; public long LatestMediaTimestamp { get; set; } public long? ResponseStartTimestamp { get; set; } + public string KeypadInputBuffer { get; set; } = string.Empty; public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 6517ef3c..de0f01aa 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -4,6 +4,10 @@ using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Routing.Models; +using NetTopologySuite.Index.HPRtree; +using BotSharp.Abstraction.Agents.Models; +using Microsoft.Identity.Client.Extensions.Msal; +using Microsoft.AspNetCore.Cors.Infrastructure; namespace BotSharp.Core.Realtime; @@ -46,9 +50,14 @@ public class RealtimeHub : IRealtimeHub { await completer.AppenAudioBuffer(conn.Data); } + else if (conn.Event == "user_dtmf_received") + { + await HandleUserDtmfReceived(completer, conn); + } else if (conn.Event == "user_disconnected") { await completer.Disconnect(); + await HandleUserDisconnected(conn); } } while (!result.CloseStatus.HasValue); @@ -58,8 +67,6 @@ public class RealtimeHub : IRealtimeHub private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn) { var hookProvider = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - var convService = _services.GetRequiredService(); convService.SetConversationId(conn.ConversationId, []); var conversation = await convService.GetConversation(conn.ConversationId); @@ -183,7 +190,6 @@ public class RealtimeHub : IRealtimeHub else { // append output audio transcript to conversation - storage.Append(conn.ConversationId, message); dialogs.Add(message); foreach (var hook in hookProvider.HooksOrderByPriority) @@ -203,7 +209,6 @@ public class RealtimeHub : IRealtimeHub onInputAudioTranscriptionCompleted: async message => { // append input audio transcript to conversation - storage.Append(conn.ConversationId, message); dialogs.Add(message); foreach (var hook in hookProvider.HooksOrderByPriority) @@ -226,6 +231,46 @@ public class RealtimeHub : IRealtimeHub }); } + private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn) + { + var routing = _services.GetRequiredService(); + var hookProvider = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conn.CurrentAgentId); + var dialogs = routing.Context.GetDialogs(); + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(conn.ConversationId); + + var message = new RoleDialogModel(AgentRole.User, conn.Data) + { + CurrentAgentId = routing.Context.GetCurrentAgentId() + }; + dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnMessageReceived(message); + } + + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the user input"); + } + + private async Task HandleUserDisconnected(RealtimeHubConnection conn) + { + // Save dialog history + var routing = _services.GetRequiredService(); + var storage = _services.GetRequiredService(); + var dialogs = routing.Context.GetDialogs(); + foreach (var item in dialogs) + { + storage.Append(conn.ConversationId, item); + } + } + private async Task SendEventToUser(WebSocket webSocket, object message) { var data = JsonSerializer.Serialize(message); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 5d7b2e80..f8c5b2e9 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -75,7 +75,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Disconnect() { - await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + if (_webSocket.State == WebSocketState.Open) + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } } public async Task AppenAudioBuffer(string message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs new file mode 100644 index 00000000..c74eca98 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs @@ -0,0 +1,17 @@ +using BotSharp.Plugin.Twilio.Models.Stream; +using System.Text.Json.Serialization; + +public class StreamEventDtmfResponse : StreamEventResponse +{ + [JsonPropertyName("dtmf")] + public StreamEventDtmfBody Body { get; set; } +} + +public class StreamEventDtmfBody +{ + [JsonPropertyName("track")] + public string Track { get; set; } + + [JsonPropertyName("digit")] + public string Digit { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index fbb8d794..c9dcc567 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -122,6 +122,20 @@ public class TwilioStreamMiddleware conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp); conn.Data = mediaResponse.Body.Payload; } + else if (response.Event == "dtmf") + { + var dtmfResponse = JsonSerializer.Deserialize(receivedText); + if (dtmfResponse.Body.Digit == "#") + { + conn.Event = "user_dtmf_received"; + conn.Data = conn.KeypadInputBuffer; + conn.KeypadInputBuffer = string.Empty; + } + else + { + conn.KeypadInputBuffer += dtmfResponse.Body.Digit; + } + } return conn; }); From 89fc99f4e189a90952be17e64c0cb05bcd6ed458 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 5 Mar 2025 16:57:58 -0600 Subject: [PATCH 35/84] prevent initial response interruption --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 10 +++++++--- .../Models/Realtime/RealtimeSessionBody.cs | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 14 ++++++++++---- .../Services/Stream/TwilioStreamMiddleware.cs | 4 ---- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index be6f8821..61c52e33 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -24,7 +24,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index de0f01aa..a6e3ed0a 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -99,8 +99,8 @@ public class RealtimeHub : IRealtimeHub await completer.Connect(conn, onModelReady: async () => { - // Control initial session - await completer.UpdateSession(conn); + // Control initial session, prevent initial response interruption + await completer.UpdateSession(conn, turnDetection: false); // Add dialog history foreach (var item in dialogs) @@ -110,12 +110,16 @@ public class RealtimeHub : IRealtimeHub if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { await completer.TriggerModelInference("Reply based on the conversation context."); } + + // Start turn detection + await Task.Delay(1000 * 8); + await completer.UpdateSession(conn, turnDetection: true); }, onModelAudioDeltaReceived: async audioDeltaData => { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index 1aca181b..0d0947e5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -47,7 +47,7 @@ public class RealtimeSessionBody public FunctionDef[] Tools { get; set; } = []; [JsonPropertyName("turn_detection")] - public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); + public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new(); } public class RealtimeSessionTurnDetection diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index f8c5b2e9..d3264e29 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -59,10 +59,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (_webSocket.State == WebSocketState.Open) { - onModelReady(); - // Receive a message _ = ReceiveMessage(conn, + onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, onAudioTranscriptDone, @@ -122,7 +121,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } - private async Task ReceiveMessage(RealtimeHubConnection conn, + private async Task ReceiveMessage(RealtimeHubConnection conn, + Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, @@ -156,6 +156,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "session.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); + onModelReady(); } else if (response.Type == "session.updated") { @@ -295,7 +296,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -346,6 +347,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; + if (!turnDetection) + { + sessionUpdate.session.TurnDetection = null; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index c9dcc567..8091736a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,13 +1,9 @@ using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; using System.Net.WebSockets; -using System.Text.Json; -using System.Collections.Concurrent; -using System.Text; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services.Stream; From 366e1339c6190616b435bdbdd508a651fc672d13 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 5 Mar 2025 17:22:46 -0600 Subject: [PATCH 36/84] refactor instruction --- .../BotSharp.Abstraction.csproj | 2 +- .../Conversations/Models/RoleDialogModel.cs | 3 +- .../Instructs/Models/InstructLogFilter.cs | 2 +- .../Instructs/Models/InstructResponseModel.cs | 8 +- .../Loggers/Models/InstructionLogModel.cs | 28 ++++- .../MLTasks/IAudioCompletion.cs | 2 + .../MLTasks/IChatCompletion.cs | 2 + .../MLTasks/IImageCompletion.cs | 2 + .../MLTasks/IRealTimeCompletion.cs | 1 - .../MLTasks/ITextCompletion.cs | 1 + .../MLTasks/ITextEmbedding.cs | 7 +- .../Instruct/FileInstructService.Image.cs | 112 +++++++++++++++++- .../Instruct/FileInstructService.Pdf.cs | 24 +++- .../Instructs/InsturctionPlugin.cs | 11 +- .../Services/InstructService.Execute.cs | 20 ++++ .../FileRepository/FileRepository.Log.cs | 85 ++++++++++++- .../Hooks/InstructionLogHook.cs | 60 ++-------- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 2 +- .../Controllers/InstructModeController.cs | 42 ++++++- .../Controllers/LoggerController.cs | 16 +++ .../Instructs/InstructMessageModel.cs | 1 + .../Providers/ChatCompletionProvider.cs | 9 +- .../Provider/NativeWhisperProvider.cs | 4 + .../Audio/AudioCompletionProvider.cs | 2 + .../Providers/Chat/ChatCompletionProvider.cs | 26 +++- .../Embedding/TextEmbeddingProvider.cs | 2 + .../Image/ImageCompletionProvider.cs | 1 + .../Providers/Text/TextCompletionProvider.cs | 1 + .../Providers/Chat/ChatCompletionProvider.cs | 24 +++- .../Providers/Text/TextCompletionProvider.cs | 1 + .../Chat/GeminiChatCompletionProvider.cs | 9 +- .../Chat/PalmChatCompletionProvider.cs | 8 +- .../Text/GeminiTextCompletionProvider.cs | 2 +- .../Text/PalmTextCompletionProvider.cs | 1 + .../Providers/ChatCompletionProvider.cs | 2 + .../Providers/ChatCompletionProvider.cs | 8 +- .../Providers/TextCompletionProvider.cs | 1 + .../Providers/TextEmbeddingProvider.cs | 3 +- .../Providers/ChatCompletionProvider.cs | 2 + .../Providers/TextCompletionProvider.cs | 2 + .../Providers/fastTextEmbeddingProvider.cs | 2 + .../Providers/ChatCompletionProvider.cs | 10 +- ...osoftExtensionsAIChatCompletionProvider.cs | 7 +- ...osoftExtensionsAITextCompletionProvider.cs | 1 + ...rosoftExtensionsAITextEmbeddingProvider.cs | 1 + .../Collections/InstructionLogBetaDocument.cs | 14 ++- .../MongoDbContext.cs | 2 +- .../Repository/MongoRepository.Log.cs | 39 +++--- .../Audio/AudioCompletionProvider.cs | 2 + .../Providers/Chat/ChatCompletionProvider.cs | 24 +++- .../Embedding/TextEmbeddingProvider.cs | 1 + .../Image/ImageCompletionProvider.cs | 1 + .../Providers/Text/TextCompletionProvider.cs | 1 + .../SemanticKernelChatCompletionProvider.cs | 4 +- .../SemanticKernelTextCompletionProvider.cs | 1 + .../SemanticKernelTextEmbeddingProvider.cs | 2 + .../Providers/ChatCompletionProvider.cs | 22 +++- 57 files changed, 537 insertions(+), 136 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index f189d323..2266e975 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 585112a2..6f96eec5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -108,7 +108,8 @@ public class RoleDialogModel : ITrackableMessage [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); - + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public string RenderedInstruction { get; set; } = string.Empty; private RoleDialogModel() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs index d8cafcee..9d154052 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs @@ -7,7 +7,7 @@ public class InstructLogFilter : Pagination public List? AgentIds { get; set; } public List? Providers { get; set; } public List? Models { get; set; } - public List? States { get; set; } + public List? TemplateNames { get; set; } public static InstructLogFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs index e2ee794d..62752bf4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs @@ -3,6 +3,10 @@ namespace BotSharp.Abstraction.Instructs.Models; public class InstructResponseModel { public string? AgentId { get; set; } - public string Provider { get; set; } - public string Model { get; set; } + public string Provider { get; set; } = default!; + public string Model { get; set; } = default!; + public string? TemplateName { get; set; } + public string UserMessage { get; set; } = default!; + public string? SystemInstruction { get; set; } + public string CompletionText { get; set; } = default!; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs index f77b08fe..844d7942 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -1,24 +1,50 @@ +using System.Text.Json; + namespace BotSharp.Abstraction.Loggers.Models; public class InstructionLogModel { + [JsonPropertyName("id")] + public string Id { get; set; } = default!; + [JsonPropertyName("agent_id")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? AgentId { get; set; } + [JsonPropertyName("agent_name")] + [JsonIgnore] + public string? AgentName { get; set; } + [JsonPropertyName("provider")] public string Provider { get; set; } = default!; [JsonPropertyName("model")] public string Model { get; set; } = default!; + [JsonPropertyName("template_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TemplateName { get; set; } + + [JsonPropertyName("user_message")] + public string UserMessage { get; set; } = string.Empty; + + [JsonPropertyName("system_instruction")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SystemInstruction { get; set; } + + [JsonPropertyName("completion_text")] + public string CompletionText { get; set; } = string.Empty; + [JsonPropertyName("user_id")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UserId { get; set; } - [JsonPropertyName("states")] + [JsonIgnore] public Dictionary States { get; set; } = []; + [JsonPropertyName("states")] + public Dictionary InnerStates { get; set; } = []; + [JsonPropertyName("created_time")] public DateTime CreatedTime { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs index 54ed2d7d..85fc84f2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -6,6 +6,8 @@ public interface IAudioCompletion { string Provider { get; } + string Model { get; } + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); Task GenerateAudioFromTextAsync(string text); diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 306e72e3..7cf52fe4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -7,6 +7,8 @@ public interface IChatCompletion /// string Provider { get; } + string Model { get; } + /// /// Set model name, one provider can consume different model or version(s) /// diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs index 323fe643..2b74a337 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs @@ -9,6 +9,8 @@ public interface IImageCompletion /// string Provider { get; } + string Model { get; } + /// /// Set model name, one provider can consume different model or version(s) /// diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index be6f8821..c5103c18 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -6,7 +6,6 @@ public interface IRealTimeCompletion { string Provider { get; } string Model { get; } - void SetModelName(string model); Task Connect(RealtimeHubConnection conn, diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs index 63a15940..8fc549f7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextCompletion.cs @@ -6,6 +6,7 @@ public interface ITextCompletion /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI /// string Provider { get; } + string Model { get; } /// /// Set model name, one provider can consume different model or version(s) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs index 79b3aff8..a96dfec9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs @@ -6,9 +6,14 @@ public interface ITextEmbedding /// The Embedding provider like Microsoft Azure, OpenAI, ClaudAI /// string Provider { get; } + string Model { get; } + + void SetModelName(string model); + + Task GetVectorAsync(string text); Task> GetVectorsAsync(List texts); - void SetModelName(string model); + void SetDimension(int dimension); int GetDimension(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 4de657c3..2976ab47 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Instructs; using System.IO; namespace BotSharp.Core.Files.Services; @@ -6,10 +8,11 @@ public partial class FileInstructService { public async Task ReadImages(string? provider, string? model, string text, IEnumerable images, string? agentId = null) { + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { - Id = agentId ?? Guid.Empty.ToString(), + Id = innerAgentId, }, new List { new RoleDialogModel(AgentRole.User, text) @@ -17,16 +20,55 @@ public partial class FileInstructService Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() } }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text, string? agentId = null) { + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); var message = await completion.GetImageGeneration(new Agent() { - Id = agentId ?? Guid.Empty.ToString(), + Id = innerAgentId, }, new RoleDialogModel(AgentRole.User, text)); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message; } @@ -37,6 +79,7 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image url or data!"); } + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var bytes = await DownloadFile(image); using var stream = new MemoryStream(); @@ -46,10 +89,29 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { - Id = agentId ?? Guid.Empty.ToString() + Id = innerAgentId }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = string.Empty, + CompletionText = message.Content + }); + } + return message; } @@ -60,6 +122,7 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image url or data!"); } + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var bytes = await DownloadFile(image); using var stream = new MemoryStream(); @@ -69,10 +132,29 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = agentId ?? Guid.Empty.ToString() + Id = innerAgentId }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message; } @@ -84,6 +166,7 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image/mask url or data"); } + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var imageBytes = await DownloadFile(image); var maskBytes = await DownloadFile(mask); @@ -100,11 +183,30 @@ public partial class FileInstructService var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = agentId ?? Guid.Empty.ToString() + Id = innerAgentId }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = message.Content + }); + } + return message; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index e1362455..9f2132a1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Files.Converters; +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Instructs; namespace BotSharp.Core.Files.Services; @@ -23,11 +25,12 @@ public partial class FileInstructService var images = await ConvertPdfToImages(pdfFiles); if (images.IsNullOrEmpty()) return content; + var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model, modelId: modelId ?? "gpt-4", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { - Id = agentId ?? Guid.Empty.ToString(), + Id = innerAgentId, }, new List { new RoleDialogModel(AgentRole.User, prompt) @@ -35,6 +38,25 @@ public partial class FileInstructService Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = prompt, + CompletionText = message.Content + }); + } + return message.Content; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs index 7b4596ff..663c710a 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Plugins.Models; +using BotSharp.Abstraction.Users.Enums; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Instructs; @@ -17,7 +18,15 @@ public class InsturctionPlugin : IBotSharpPlugin public bool AttachMenu(List menu) { var section = menu.First(x => x.Label == "Apps"); - menu.Add(new PluginMenuDef("Instruction", link: "page/instruction", icon: "bx bx-book-content", weight: section.Weight + 5)); + menu.Add(new PluginMenuDef("Instruction", icon: "bx bx-book-content", weight: section.Weight + 5) + { + SubMenu = new List + { + new PluginMenuDef("Instruction", link: "page/instruction"), + new PluginMenuDef("Log", link: "page/instruction/log") { Roles = [UserRole.Root, UserRole.Admin] } + } + }); + return true; } } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index 1d9dd1d0..36f5b523 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -43,6 +43,9 @@ public partial class InstructService } } + var provider = string.Empty; + var model = string.Empty; + // Render prompt var prompt = string.IsNullOrEmpty(templateName) ? agentService.RenderedInstruction(agent) : @@ -57,11 +60,18 @@ public partial class InstructService }; if (completer is ITextCompletion textCompleter) { + instruction = null; + provider = textCompleter.Provider; + model = textCompleter.Model; + var result = await textCompleter.GetCompletion(prompt, agentId, message.MessageId); response.Text = result; } else if (completer is IChatCompletion chatCompleter) { + provider = chatCompleter.Provider; + model = chatCompleter.Model; + if (instruction == "#TEMPLATE#") { instruction = prompt; @@ -93,6 +103,16 @@ public partial class InstructService } await hook.AfterCompletion(agent, response); + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = agentId, + Provider = provider, + Model = model, + TemplateName = templateName, + UserMessage = prompt, + SystemInstruction = instruction, + CompletionText = response.Text + }); } return response; diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 9f7ca305..ea1f27b5 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -136,7 +136,8 @@ namespace BotSharp.Core.Repository foreach (var log in logs) { - var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.log"); + var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.json"); + log.InnerStates = BuildLogStates(log.States); var text = JsonSerializer.Serialize(log, _options); File.WriteAllText(file, text); } @@ -150,9 +151,67 @@ namespace BotSharp.Core.Repository filter = InstructLogFilter.Empty(); } + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + return new(); + } + + var logs = new List(); + var files = Directory.GetFiles(baseDir); + foreach (var file in files) + { + var json = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(json, _options); + if (log == null) continue; + + var matched = true; + if (!filter.AgentIds.IsNullOrEmpty()) + { + matched = matched && filter.AgentIds.Contains(log.AgentId); + } + if (!filter.Providers.IsNullOrEmpty()) + { + matched = matched && filter.Providers.Contains(log.Provider); + } + if (!filter.Models.IsNullOrEmpty()) + { + matched = matched && filter.Models.Contains(log.Model); + } + if (!filter.TemplateNames.IsNullOrEmpty()) + { + matched = matched && filter.TemplateNames.Contains(log.TemplateName); + } + + if (!matched) continue; + + log.Id = Path.GetFileNameWithoutExtension(file); + logs.Add(log); + } + + var records = logs.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size); + var agentIds = records.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); + var agents = GetAgents(new AgentFilter + { + AgentIds = agentIds + }); + + records = records.Select(x => + { + var states = x.InnerStates.ToDictionary(p => p.Key, p => + { + var data = p.Value.RootElement.GetProperty("data"); + return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; + }); + x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + x.States = states ?? []; + return x; + }).ToList(); + return new PagedItems { - + Items = records, + Count = logs.Count() }; } #endregion @@ -176,6 +235,28 @@ namespace BotSharp.Core.Repository return logIndexes.IsNullOrEmpty() ? 0 : logIndexes.Max() + 1; } + + private Dictionary BuildLogStates(Dictionary states) + { + var dic = new Dictionary(); + foreach (var pair in states) + { + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _options); + var json = JsonDocument.Parse(jsonStr); + dic[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = pair.Value }, _options); + var json = JsonDocument.Parse(str); + dic[pair.Key] = json; + } + } + + return dic; + } #endregion } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs index 03e1f8ff..265b631a 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Users; -using System.Text.Json; namespace BotSharp.Logger.Hooks; @@ -10,19 +8,18 @@ public class InstructionLogHook : InstructHookBase { private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly BotSharpOptions _options; private readonly IUserIdentity _user; + public override string SelfId => string.Empty; + public InstructionLogHook( IServiceProvider services, ILogger logger, - IUserIdentity user, - BotSharpOptions options) + IUserIdentity user) { _services = services; _logger = logger; _user = user; - _options = options; } public override async Task OnResponseGenerated(InstructResponseModel response) @@ -40,57 +37,14 @@ public class InstructionLogHook : InstructHookBase AgentId = response.AgentId, Provider = response.Provider, Model = response.Model, + TemplateName = response.TemplateName, + UserMessage = response.UserMessage, + SystemInstruction = response.SystemInstruction, + CompletionText = response.CompletionText, States = state.GetStates(), UserId = user?.Id } }); return; } - - private Dictionary CollectStates() - { - var res = new Dictionary(); - var state = _services.GetRequiredService(); - var curStates = state.GetStates(); - - curStates["test"] = JsonSerializer.Serialize(new - { - Number = "789", - Dummy = new - { - Id = 123, - Name = "name", - Score = 12.123 - }, - Items = new List - { - new - { - Name = "image", - Label = "before-service", - Attribute = new - { - Location = "Chicago", - Time = "afternoon" - } - }, - new - { - Name = "pdf", - Label = "after-service", - Attribute = new - { - Location = "New York", - Time = "morning" - } - }, - }, - Lists = new List - { - "abc", - "bcd" - } - }, _options.JsonSerializerOptions); - return curStates; - } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 58157b2d..38ec30db 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 45054223..6a1c3705 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -52,8 +52,29 @@ public class InstructModeController : ControllerBase .SetState("model", input.Model, source: StateSource.External) .SetState("model_id", input.ModelId, source: StateSource.External); + var agentId = input.AgentId ?? Guid.Empty.ToString(); var textCompletion = CompletionProvider.GetTextCompletion(_services); - return await textCompletion.GetCompletion(input.Text, input.AgentId ?? Guid.Empty.ToString(), Guid.NewGuid().ToString()); + var response = await textCompletion.GetCompletion(input.Text, agentId, Guid.NewGuid().ToString()); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = agentId, + Provider = textCompletion.Provider, + Model = textCompletion.Model, + TemplateName = input.Template, + UserMessage = input.Text, + CompletionText = response + }); + } + return response; } #region Chat @@ -66,10 +87,11 @@ public class InstructModeController : ControllerBase .SetState("model", input.Model, source: StateSource.External) .SetState("model_id", input.ModelId, source: StateSource.External); + var agentId = input.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services); var message = await completion.GetChatCompletions(new Agent() { - Id = input.AgentId ?? Guid.Empty.ToString(), + Id = agentId, Instruction = input.Instruction }, new List { @@ -79,14 +101,22 @@ public class InstructModeController : ControllerBase var hooks = _services.GetServices(); foreach (var hook in hooks) { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + { + continue; + } + await hook.OnResponseGenerated(new InstructResponseModel { - AgentId = input.AgentId, - Provider = input.Provider, - Model = input.Model + AgentId = agentId, + Provider = completion.Provider, + Model = completion.Model, + TemplateName = input.Template, + UserMessage = input.Text, + SystemInstruction = message.RenderedInstruction, + CompletionText = message.Content }); } - return message.Content; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index de07a482..e7ba7fd7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,4 +1,8 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; +using BotSharp.OpenAPI.ViewModels.Logs; +using EntityFrameworkCore.BootKit; using Microsoft.AspNetCore.Hosting; namespace BotSharp.OpenAPI.Controllers; @@ -45,4 +49,16 @@ public class LoggerController : ControllerBase var conversationService = _services.GetRequiredService(); return await conversationService.GetConversationStateLogs(conversationId); } + + [HttpGet("/logger/instruction/log")] + public async Task> GetInstructionLogs([FromQuery] InstructLogFilter request) + { + var db = _services.GetRequiredService(); + var logs = db.GetInstructionLogs(request); + return new PagedItems + { + Items = logs.Items.Select(x => InstructionLogViewModel.From(x)), + Count = logs.Count + }; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs index 3b0303fd..1ad3aead 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs @@ -15,4 +15,5 @@ public class IncomingInstructRequest : IncomingMessageModel { public string? AgentId { get; set; } public string? Instruction { get; set; } + public string? Template { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index b71306cf..178bc67b 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -9,10 +9,12 @@ namespace BotSharp.Plugin.AnthropicAI.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "anthropic"; + public string Model => _model; protected readonly AnthropicSettings _settings; protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private List renderedInstructions = []; protected string _model; @@ -57,6 +59,7 @@ public class ChatCompletionProvider : IChatCompletion ToolCallId = toolResult.Id, FunctionName = toolResult.Name, FunctionArgs = JsonSerializer.Serialize(toolResult.Input), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } else @@ -66,6 +69,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -98,12 +102,15 @@ public class ChatCompletionProvider : IChatCompletion private (string, MessageParameters) PrepareOptions(Agent agent, List conversations, LlmModelSetting settings) { var instruction = ""; + renderedInstructions = []; var agentService = _services.GetRequiredService(); if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { - instruction += agentService.RenderedInstruction(agent); + var text = agentService.RenderedInstruction(agent); + instruction += text; + renderedInstructions.Add(text); } /*var routing = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 1947ebc4..00d1da58 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -15,6 +15,8 @@ public class NativeWhisperProvider : IAudioCompletion private readonly ILogger _logger; public string Provider => "native-whisper"; + public string Model => _model; + private string _model; public NativeWhisperProvider( BotSharpDatabaseSettings dbSettings, @@ -56,11 +58,13 @@ public class NativeWhisperProvider : IAudioCompletion { if (Enum.TryParse(model, true, out GgmlType ggmlType)) { + _model = model; LoadWhisperModel(ggmlType); } else { _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + _model = "Tiny"; LoadWhisperModel(GgmlType.Tiny); } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs index 8cc75ad8..2948703e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -5,6 +5,8 @@ public partial class AudioCompletionProvider : IAudioCompletion private readonly IServiceProvider _services; public string Provider => "openai"; + public string Model => _model; + private string _model; public AudioCompletionProvider(IServiceProvider service) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 3fef7c0c..e49d91ce 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -9,10 +9,12 @@ public class ChatCompletionProvider : IChatCompletion protected readonly AzureOpenAiSettings _settings; protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private List renderedInstructions = []; protected string _model; public virtual string Provider => "azure-openai"; + public string Model => _model; public ChatCompletionProvider( AzureOpenAiSettings settings, @@ -58,7 +60,8 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -73,6 +76,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } } @@ -83,6 +87,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } catch (Exception ex) @@ -92,6 +97,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -136,7 +142,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -163,7 +170,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -199,7 +207,10 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; Console.Write(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); continue; } @@ -207,7 +218,10 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); } return true; @@ -221,6 +235,7 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; var messages = new List(); @@ -251,6 +266,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); + renderedInstructions.Add(instruction); messages.Add(new SystemChatMessage(instruction)); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 6878ca3c..b89e338c 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -13,6 +13,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected int _dimension; public virtual string Provider => "azure-openai"; + public string Model => _model; public TextEmbeddingProvider( AzureOpenAiSettings settings, @@ -49,6 +50,7 @@ public class TextEmbeddingProvider : ITextEmbedding _model = model; } + public void SetDimension(int dimension) { _dimension = dimension > 0 ? dimension : DEFAULT_DIMENSION; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 967f3e08..1b435d61 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -14,6 +14,7 @@ public partial class ImageCompletionProvider : IImageCompletion protected string _model; public virtual string Provider => "azure-openai"; + public string Model => _model; public ImageCompletionProvider( AzureOpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index 4953e114..2b6f3dfb 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -22,6 +22,7 @@ public class TextCompletionProvider : ITextCompletion }; public virtual string Provider => "azure-openai"; + public string Model => _model; public TextCompletionProvider( AzureOpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index ef4d0c8a..4c21dd03 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -9,9 +9,11 @@ public class ChatCompletionProvider : IChatCompletion { protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private List renderedInstructions = []; protected string _model; public virtual string Provider => "deepseek-ai"; + public string Model => _model; public ChatCompletionProvider( IServiceProvider services, @@ -51,7 +53,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -66,6 +69,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -107,7 +111,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -134,7 +139,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -170,7 +176,10 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; _logger.LogInformation(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); continue; } @@ -178,7 +187,10 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); } return true; @@ -197,6 +209,7 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; var messages = new List(); @@ -226,6 +239,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var text = agentService.RenderedInstruction(agent); + renderedInstructions.Add(text); messages.Add(new SystemChatMessage(text)); } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs index a1a88978..27198238 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs @@ -10,6 +10,7 @@ public class TextCompletionProvider : ITextCompletion protected string _model; public string Provider => "deepseek-ai"; + public string Model => _model; public TextCompletionProvider( IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 9567ae57..174efcf2 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -2,7 +2,6 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Loggers; -using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; using Mscc.GenerativeAI; @@ -12,10 +11,12 @@ public class GeminiChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public string Provider => "google-ai"; + public string Model => _model; public GeminiChatCompletionProvider( IServiceProvider services, @@ -53,7 +54,8 @@ public class GeminiChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = part.FunctionCall.Name, FunctionName = part.FunctionCall.Name, - FunctionArgs = part.FunctionCall.Args?.ToString() + FunctionArgs = part.FunctionCall.Args?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } else @@ -62,6 +64,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -98,6 +101,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var googleSettings = _services.GetRequiredService(); + renderedInstructions = []; // Add settings aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; @@ -117,6 +121,7 @@ public class GeminiChatCompletionProvider : IChatCompletion Role = AgentRole.User }); + renderedInstructions.Add(instruction); systemPrompts.Add(instruction); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index c62aebb8..4fe6ad17 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -13,10 +13,12 @@ public class PalmChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public string Provider => "google-palm"; + public string Model => _model; public PalmChatCompletionProvider( IServiceProvider services, @@ -61,7 +63,8 @@ public class PalmChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, FunctionName = llmResponse.FunctionName, - FunctionArgs = JsonSerializer.Serialize(llmResponse.Args) + FunctionArgs = JsonSerializer.Serialize(llmResponse.Args), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } else @@ -75,7 +78,8 @@ public class PalmChatCompletionProvider : IChatCompletion msg = new RoleDialogModel(llmResponse.Role, llmResponse.Content ?? message.Content) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index e14071f7..ebbca98a 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -14,6 +14,7 @@ public class GeminiTextCompletionProvider : ITextCompletion private string _model; public string Provider => "google-ai"; + public string Model => _model; public GeminiTextCompletionProvider( IServiceProvider services, @@ -74,7 +75,6 @@ public class GeminiTextCompletionProvider : ITextCompletion _model = model; } - private void PrepareOptions(GenerativeModel aiModel) { var settings = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs index ebaf6c5f..7df18bb7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -14,6 +14,7 @@ public class PalmTextCompletionProvider : ITextCompletion private string _model; public string Provider => "google-palm"; + public string Model => _model; public PalmTextCompletionProvider( IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 88b37dbb..460677ba 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -10,10 +10,12 @@ namespace BotSharp.Plugin.HuggingFace.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "huggingface"; + public string Model => _model; private readonly IServiceProvider _services; private readonly HuggingFaceSettings _settings; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 32f7220b..45f41e95 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -8,6 +8,7 @@ public class ChatCompletionProvider : IChatCompletion private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly LlamaSharpSettings _settings; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, @@ -20,6 +21,7 @@ public class ChatCompletionProvider : IChatCompletion } public string Provider => "llama-sharp"; + public string Model => _model; public async Task GetChatCompletions(Agent agent, List conversations) { @@ -64,7 +66,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = instruction }; // After chat completion hook @@ -146,7 +149,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = agent.Instruction }; // Text response received diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs index 2a98becc..e911a8ff 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs @@ -10,6 +10,7 @@ public class TextCompletionProvider : ITextCompletion private readonly ITokenStatistics _tokenStatistics; private string _model; public string Provider => "llama-sharp"; + public string Model => _model; public TextCompletionProvider(IServiceProvider services, ILogger logger, diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs index 1ec461ae..5c58f470 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs @@ -1,6 +1,4 @@ using System.IO; -using System.Xml.Linq; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.LLamaSharp.Providers; @@ -14,6 +12,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected int _dimension = DEFAULT_DIMENSION; public string Provider => "llama-sharp"; + public string Model => string.Empty; public TextEmbeddingProvider(IServiceProvider services, LlamaSharpSettings settings) { diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs index 61f66119..aa6b1175 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs @@ -20,6 +20,8 @@ namespace BotSharp.Plugin.VertexAI.Providers IServiceProvider services) : IChatCompletion { public string Provider => "vertexai"; + public string Model => _model; + private readonly VertexAIConfiguration _config = config; private readonly ChatSettings? _settings = settings; private readonly IServiceProvider _services = services; diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs index 46186452..70f0cbdc 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs @@ -19,6 +19,8 @@ namespace BotSharp.Plugin.VertexAI.Providers IServiceProvider services) : ITextCompletion { public string Provider => "vertexai"; + public string Model => _model; + private readonly VertexAIConfiguration _config = config; private readonly ChatSettings? _settings = settings; private readonly IServiceProvider _services = services; diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs index 7d9c6141..07b5b76e 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using BotSharp.Plugin.MetaAI.Settings; using FastText.NetWrapper; using Microsoft.Extensions.DependencyInjection; @@ -17,6 +18,7 @@ public class fastTextEmbeddingProvider : ITextEmbedding private int _dimension; public string Provider => "meta-ai"; + public string Model => string.Empty; public fastTextEmbeddingProvider(IServiceProvider services) { diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index f99293f4..b5b19671 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -5,11 +5,13 @@ namespace BotSharp.Plugin.MetaGLM.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "metaglm"; + public string Model => _model; private readonly MetaGLMSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly MetaGLMClient metaGLMClient; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, @@ -49,7 +51,8 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, response?.choices.FirstOrDefault()?.message.content) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.Last().MessageId, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -61,7 +64,8 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id, MessageId = conversations.Last().MessageId, FunctionName = toolcall.function.name, - FunctionArgs = toolcall.function.arguments + FunctionArgs = toolcall.function.arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -87,10 +91,12 @@ public class ChatCompletionProvider : IChatCompletion var agentService = _services.GetRequiredService(); List messages = new List(); List toolcalls = new List(); + renderedInstructions = []; if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); + renderedInstructions.Add(instruction); messages.Add(new MessageItem("system", instruction)); } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 8619ecad..37d3877c 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -27,6 +27,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio private readonly IChatClient _client; private readonly ILogger _logger; private readonly IServiceProvider _services; + private List renderedInstructions = []; private string? _model; /// @@ -45,6 +46,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio /// public string Provider => "microsoft.extensions.ai"; + public string Model => _model; /// public void SetModelName(string model) => _model = model; @@ -54,6 +56,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio { // Before chat completion hook var hooks = _services.GetServices().ToArray(); + renderedInstructions = []; await Task.WhenAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations))); // Configure options @@ -82,6 +85,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio if (_services.GetRequiredService().RenderedInstruction(agent) is string instruction && instruction.Length > 0) { + renderedInstructions.Add(instruction); messages.Add(new(ChatRole.System, instruction)); } @@ -143,7 +147,8 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio RoleDialogModel result = new(AgentRole.Assistant, string.Concat(completion.Message.Contents.OfType())) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; if (completion.Message.Contents.OfType().FirstOrDefault() is { } fcc) diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs index 94d26678..ed0e94d9 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs @@ -24,6 +24,7 @@ public sealed class MicrosoftExtensionsAITextCompletionProvider : ITextCompletio /// public string Provider => "microsoft-extensions-ai"; + public string Model => _model; /// /// Creates an instance of the class. diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs index 958c4694..1758468a 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs @@ -23,6 +23,7 @@ public sealed class MicrosoftExtensionsAITextEmbeddingProvider : ITextEmbedding /// public string Provider => "microsoft-extensions-ai"; + public string Model => _model; /// public async Task GetVectorAsync(string text) => diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs index 615e93af..4f70ad60 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Loggers.Models; -using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Collections; @@ -8,6 +7,10 @@ public class InstructionLogBetaDocument : MongoBase public string? AgentId { get; set; } public string Provider { get; set; } = default!; public string Model { get; set; } = default!; + public string? TemplateName { get; set; } + public string UserMessage { get; set; } = default!; + public string? SystemInstruction { get; set; } + public string CompletionText { get; set; } = default!; public string? UserId { get; set; } public Dictionary States { get; set; } = new(); public DateTime CreatedTime { get; set; } @@ -19,6 +22,10 @@ public class InstructionLogBetaDocument : MongoBase AgentId = log.AgentId, Provider = log.Provider, Model = log.Model, + TemplateName = log.TemplateName, + UserMessage = log.UserMessage, + SystemInstruction = log.SystemInstruction, + CompletionText = log.CompletionText, UserId = log.UserId, CreatedTime = log.CreatedTime }; @@ -28,9 +35,14 @@ public class InstructionLogBetaDocument : MongoBase { return new InstructionLogModel { + Id = log.Id, AgentId = log.AgentId, Provider = log.Provider, Model = log.Model, + TemplateName = log.TemplateName, + UserMessage = log.UserMessage, + SystemInstruction = log.SystemInstruction, + CompletionText = log.CompletionText, UserId = log.UserId, CreatedTime = log.CreatedTime }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 9adc04b9..277f9e25 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -194,5 +194,5 @@ public class MongoDbContext => GetCollectionOrCreate("GlobalStatistics"); public IMongoCollection InstructionLogs - => GetCollectionOrCreate("InstructionLogsBeta"); + => GetCollectionOrCreate("InstructionLogs"); } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index c90960dd..05d5a6a9 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories.Filters; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -167,28 +168,9 @@ public partial class MongoRepository { filters.Add(builder.In(x => x.Model, filter.Models)); } - - if (!filter.States.IsNullOrEmpty()) + if (!filter.TemplateNames.IsNullOrEmpty()) { - foreach (var pair in filter.States) - { - if (string.IsNullOrWhiteSpace(pair.Key)) continue; - - // Format key - var keys = pair.Key.Split(".").ToList(); - keys.Insert(1, "data"); - keys.Insert(0, "States"); - var formattedKey = string.Join(".", keys); - - if (pair.Value == null) - { - filters.Add(builder.Exists(formattedKey)); - } - else - { - filters.Add(builder.Eq(formattedKey, pair.Value)); - } - } + filters.Add(builder.In(x => x.TemplateName, filter.TemplateNames)); } var filterDef = builder.And(filters); @@ -196,10 +178,23 @@ public partial class MongoRepository var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); var count = _dc.InstructionLogs.CountDocuments(filterDef); + var agentIds = docs.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); + var agents = GetAgents(new AgentFilter + { + AgentIds = agentIds + }); + var logs = docs.Select(x => { var log = InstructionLogBetaDocument.ToDomainModel(x); - log.States = x.States.ToDictionary(x => x.Key, x => x.Value.GetElement("data").Value.ToString() ?? string.Empty); + log.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + log.States = x.States.ToDictionary(p => p.Key, p => + { + var jsonStr = p.Value.ToJson(); + var jsonDoc = JsonDocument.Parse(jsonStr); + var data = jsonDoc.RootElement.GetProperty("data"); + return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; + }); return log; }).ToList(); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs index 3311fd43..338affd9 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -7,6 +7,8 @@ public partial class AudioCompletionProvider : IAudioCompletion private readonly IServiceProvider _services; public string Provider => "openai"; + public string Model => _model; + private string _model; public AudioCompletionProvider(IServiceProvider service) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 15eb04c5..031fab9c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -10,8 +10,10 @@ public class ChatCompletionProvider : IChatCompletion protected readonly ILogger _logger; protected string _model; + private List renderedInstructions = []; public virtual string Provider => "openai"; + public string Model => _model; public ChatCompletionProvider( OpenAiSettings settings, @@ -53,7 +55,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -68,6 +71,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -112,7 +116,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -139,7 +144,8 @@ public class ChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments?.ToString() + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -175,7 +181,10 @@ public class ChatCompletionProvider : IChatCompletion var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; _logger.LogInformation(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); continue; } @@ -183,7 +192,10 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); } return true; @@ -198,6 +210,7 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; var messages = new List(); @@ -227,6 +240,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var text = agentService.RenderedInstruction(agent); + renderedInstructions.Add(text); messages.Add(new SystemChatMessage(text)); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 5b92d896..4131b814 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -13,6 +13,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected int _dimension = DEFAULT_DIMENSION; public virtual string Provider => "openai"; + public string Model => _model; public TextEmbeddingProvider( OpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 0e45e789..7cfda0f5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -14,6 +14,7 @@ public partial class ImageCompletionProvider : IImageCompletion protected string _model; public virtual string Provider => "openai"; + public string Model => _model; public ImageCompletionProvider( OpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index 097a32b7..91b39848 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -15,6 +15,7 @@ public class TextCompletionProvider : ITextCompletion protected string _model; public virtual string Provider => "openai"; + public string Model => _model; public TextCompletionProvider( OpenAiSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index 156f238c..3f742777 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -26,6 +26,7 @@ namespace BotSharp.Plugin.SemanticKernel /// public string Provider => "semantic-kernel"; + public string Model => _model; /// /// Create a new instance of @@ -74,7 +75,8 @@ namespace BotSharp.Plugin.SemanticKernel var response = chatMessageContent != null ? chatMessageContent.Content :string.Empty; var msg = new RoleDialogModel(AgentRole.Assistant, response) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = instruction }; // After chat completion hook diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs index 91dec640..8b03c011 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextCompletionProvider.cs @@ -23,6 +23,7 @@ namespace BotSharp.Plugin.SemanticKernel /// public string Provider => "semantic-kernel"; + public string Model => _model; /// /// Create a new instance of diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs index 5486d972..64915090 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel.Embeddings; using System.Collections.Generic; @@ -33,6 +34,7 @@ namespace BotSharp.Plugin.SemanticKernel protected int _dimension; public string Provider => "semantic-kernel"; + public string Model => string.Empty; /// public async Task GetVectorAsync(string text) diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 5edf4708..d5e64e7d 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -7,10 +7,12 @@ namespace BotSharp.Plugin.SparkDesk.Providers; public class ChatCompletionProvider : IChatCompletion { public string Provider => "sparkdesk"; + public string Model => _model; private readonly SparkDeskSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; + private List renderedInstructions = []; private string _model; public ChatCompletionProvider(IServiceProvider services, @@ -42,7 +44,8 @@ public class ChatCompletionProvider : IChatCompletion var responseMessage = new RoleDialogModel(AgentRole.Assistant, response.Text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.Last().MessageId, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; if (response.FunctionCall != null) @@ -52,7 +55,8 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id, MessageId = conversations.Last().MessageId, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments + FunctionArgs = response.FunctionCall.Arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -92,7 +96,8 @@ public class ChatCompletionProvider : IChatCompletion var msg = new RoleDialogModel(AgentRole.Assistant, response.Text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // After chat completion hook @@ -116,7 +121,8 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments + FunctionArgs = response.FunctionCall.Arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. @@ -150,14 +156,16 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments + FunctionArgs = response.FunctionCall.Arguments, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }); continue; } await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, response.Text) { - CurrentAgentId = agent.Id + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) }); } @@ -175,10 +183,12 @@ public class ChatCompletionProvider : IChatCompletion var functions = new List(); var agentService = _services.GetRequiredService(); var messages = new List(); + renderedInstructions = []; if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); + renderedInstructions.Add(instruction); messages.Add(ChatMessage.FromSystem(instruction)); } if (!string.IsNullOrEmpty(agent.Knowledges)) From f60bcda89699c5dcb1b8e0cad11a2d07e53de1e6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 5 Mar 2025 17:32:02 -0600 Subject: [PATCH 37/84] relocate --- .../Controllers/LoggerController.cs | 3 +- .../Instructs/InstructionLogViewModel.cs | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index e7ba7fd7..2c46b5b4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,8 +1,7 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories; -using BotSharp.OpenAPI.ViewModels.Logs; -using EntityFrameworkCore.BootKit; +using BotSharp.OpenAPI.ViewModels.Instructs; using Microsoft.AspNetCore.Hosting; namespace BotSharp.OpenAPI.Controllers; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs new file mode 100644 index 00000000..88fc27e2 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs @@ -0,0 +1,67 @@ +using BotSharp.Abstraction.Loggers.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructionLogViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } = default!; + + [JsonPropertyName("agent_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentId { get; set; } + + [JsonPropertyName("agent_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentName { get; set; } + + [JsonPropertyName("provider")] + public string Provider { get; set; } = default!; + + [JsonPropertyName("model")] + public string Model { get; set; } = default!; + + [JsonPropertyName("template_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TemplateName { get; set; } + + [JsonPropertyName("user_message")] + public string UserMessage { get; set; } = string.Empty; + + [JsonPropertyName("system_instruction")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SystemInstruction { get; set; } + + [JsonPropertyName("completion_text")] + public string CompletionText { get; set; } = string.Empty; + + [JsonPropertyName("user_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserId { get; set; } + + [JsonPropertyName("states")] + public Dictionary States { get; set; } = []; + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } = DateTime.UtcNow; + + public static InstructionLogViewModel From(InstructionLogModel log) + { + return new InstructionLogViewModel + { + Id = log.Id, + AgentId = log.AgentId, + AgentName = log.AgentName, + Provider = log.Provider, + Model = log.Model, + TemplateName = log.TemplateName, + UserMessage = log.UserMessage, + SystemInstruction = log.SystemInstruction, + CompletionText = log.CompletionText, + UserId = log.UserId, + States = log.States, + CreatedTime = log.CreatedTime + }; + } +} From 5d10f3d3050f5bb7b82badaddfebc9b86b747e43 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Thu, 6 Mar 2025 17:03:30 +0800 Subject: [PATCH 38/84] optimize RealTime --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Realtime/Models/RealtimeHubConnection.cs | 17 ++++- .../BotSharp.Core/Realtime/RealtimeHub.cs | 50 +++++++------ .../Realtime/RealTimeCompletionProvider.cs | 33 +++------ .../Services/Stream/TwilioStreamMiddleware.cs | 73 +++++++++---------- 5 files changed, 92 insertions(+), 83 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 1c4e847e..2d0150ec 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -10,7 +10,7 @@ public interface IRealTimeCompletion Task Connect(RealtimeHubConnection conn, Action onModelReady, - Action onModelAudioDeltaReceived, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, Action> onModelResponseDone, diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 7feaf23c..d1fc6a77 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -6,9 +6,9 @@ public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; - public string? LastAssistantItem { get; set; } = null!; + public string? LastAssistantItemId { get; set; } = null!; public long LatestMediaTimestamp { get; set; } - public long? ResponseStartTimestamp { get; set; } + public long? ResponseStartTimestampTwilio { get; set; } public string KeypadInputBuffer { get; set; } = string.Empty; public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; @@ -18,4 +18,17 @@ public class RealtimeHubConnection public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; public Func OnModelUserInterrupted { get; set; } = null!; + + public void ResetResponseState() + { + MarkQueue.Clear(); + LastAssistantItemId = null; + ResponseStartTimestampTwilio = null; + } + + public void ResetStreamState() + { + ResponseStartTimestampTwilio = null; + LatestMediaTimestamp = 0; + } } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index a6e3ed0a..6d2b10ec 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -121,30 +121,25 @@ public class RealtimeHub : IRealtimeHub await Task.Delay(1000 * 8); await completer.UpdateSession(conn, turnDetection: true); }, - onModelAudioDeltaReceived: async audioDeltaData => + onModelAudioDeltaReceived: async (audioDeltaData, itemId) => { - // If this is the first delta of a new response, set the start timestamp - if (!conn.ResponseStartTimestamp.HasValue) - { - conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; - _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms"); - } - var data = conn.OnModelMessageReceived(audioDeltaData); await SendEventToUser(userWebSocket, data); - // Send mark messages to Media Streams so we know if and when AI response playback is finished - if (!string.IsNullOrEmpty(conn.StreamId)) + // If this is the first delta of a new response, set the start timestamp + if (!conn.ResponseStartTimestampTwilio.HasValue) { - var markEvent = new - { - @event = "mark", - streamSid = conn.StreamId, - mark = new { name = "responsePart" } - }; - await SendEventToUser(userWebSocket, markEvent); - conn.MarkQueue.Enqueue("responsePart"); + conn.ResponseStartTimestampTwilio = conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestampTwilio}ms"); } + // Record last assistant item ID for interruption handling + if (!string.IsNullOrEmpty(itemId)) + { + conn.LastAssistantItemId = itemId; + } + + // Send mark messages to Media Streams so we know if and when AI response playback is finished + await SendMark(userWebSocket, conn); }, onModelAudioResponseDone: async () => { @@ -226,15 +221,28 @@ public class RealtimeHub : IRealtimeHub onUserInterrupted: async () => { // Reset states - conn.MarkQueue.Clear(); - conn.LastAssistantItem = null; - conn.ResponseStartTimestamp = null; + conn.ResetResponseState(); var data = conn.OnModelUserInterrupted(); await SendEventToUser(userWebSocket, data); }); } + private async Task SendMark(WebSocket userWebSocket, RealtimeHubConnection conn) + { + if (!string.IsNullOrEmpty(conn.StreamId)) + { + var markEvent = new + { + @event = "mark", + streamSid = conn.StreamId, + mark = new { name = "responsePart" } + }; + await SendEventToUser(userWebSocket, markEvent); + conn.MarkQueue.Enqueue("responsePart"); + } + } + private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn) { var routing = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index d3264e29..7f31c95e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -40,7 +40,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Connect(RealtimeHubConnection conn, Action onModelReady, - Action onModelAudioDeltaReceived, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, Action> onModelResponseDone, @@ -123,7 +123,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private async Task ReceiveMessage(RealtimeHubConnection conn, Action onModelReady, - Action onModelAudioDeltaReceived, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, Action> onModelResponseDone, @@ -176,21 +176,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.audio.delta") { var audio = JsonSerializer.Deserialize(receivedText); - // Record last assistant item ID for interruption handling - if (conn.ResponseStartTimestamp.HasValue) - { - conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; - } - - if (!string.IsNullOrEmpty(conn.StreamId)) - { - conn.LastAssistantItem = audio?.ItemId; - } - - if (audio != null && audio.Delta != null) + if (audio?.Delta != null) { _logger.LogDebug($"{response.Type}: {receivedText}"); - onModelAudioDeltaReceived(audio.Delta); + onModelAudioDeltaReceived(audio.Delta, audio.ItemId); } } else if (response.Type == "response.audio.done") @@ -218,16 +207,16 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) + if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestampTwilio != null) { - var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; + var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestampTwilio; - if (!string.IsNullOrEmpty(conn.LastAssistantItem)) + if (!string.IsNullOrEmpty(conn.LastAssistantItemId)) { var truncateEvent = new { type = "conversation.item.truncate", - item_id = conn.LastAssistantItem, + item_id = conn.LastAssistantItemId, content_index = 0, audio_end_ms = elapsedTime }; @@ -336,13 +325,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.8f), + Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), MaxResponseOutputTokens = 512, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.5f, + Threshold = 0.8f, PrefixPadding = 300, - SilenceDuration = 500 + SilenceDuration = 800 } } }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 8091736a..0e3a68c0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -71,17 +71,44 @@ public class TwilioStreamMiddleware { var response = JsonSerializer.Deserialize(receivedText); conn.StreamId = response.StreamSid; - conn.Event = response.Event switch - { - "start" => "user_connected", - "media" => "user_data_received", - "stop" => "user_disconnected", - _ => response.Event - }; - if (string.IsNullOrEmpty(conn.Event)) + switch (response.Event) { - return conn; + case "start": + conn.Event = "user_connected"; + var startResponse = JsonSerializer.Deserialize(receivedText); + conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); + conn.ResetStreamState(); + break; + case "media": + conn.Event = "user_data_received"; + var mediaResponse = JsonSerializer.Deserialize(receivedText); + conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp); + conn.Data = mediaResponse.Body.Payload; + break; + case "stop": + conn.Event = "user_disconnected"; + break; + case "mark": + conn.Event = "mark"; + if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _); + break; + case "dtmf": + var dtmfResponse = JsonSerializer.Deserialize(receivedText); + if (dtmfResponse.Body.Digit == "#") + { + conn.Event = "user_dtmf_received"; + conn.Data = conn.KeypadInputBuffer; + conn.KeypadInputBuffer = string.Empty; + } + else + { + conn.KeypadInputBuffer += dtmfResponse.Body.Digit; + } + break; + default: + conn.Event = response.Event; + break; } conn.OnModelMessageReceived = message => @@ -105,34 +132,6 @@ public class TwilioStreamMiddleware streamSid = response.StreamSid }; - if (response.Event == "start") - { - var startResponse = JsonSerializer.Deserialize(receivedText); - conn.LatestMediaTimestamp = 0; - conn.ResponseStartTimestamp = null; - conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); - } - else if (response.Event == "media") - { - var mediaResponse = JsonSerializer.Deserialize(receivedText); - conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp); - conn.Data = mediaResponse.Body.Payload; - } - else if (response.Event == "dtmf") - { - var dtmfResponse = JsonSerializer.Deserialize(receivedText); - if (dtmfResponse.Body.Digit == "#") - { - conn.Event = "user_dtmf_received"; - conn.Data = conn.KeypadInputBuffer; - conn.KeypadInputBuffer = string.Empty; - } - else - { - conn.KeypadInputBuffer += dtmfResponse.Body.Digit; - } - } - return conn; }); } From af01567fae0378ad18e12b3423385b35ac793800 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Thu, 6 Mar 2025 20:23:40 +0800 Subject: [PATCH 39/84] hotfix duplicate welcome message --- src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 6d2b10ec..219cd6ba 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -103,10 +103,10 @@ public class RealtimeHub : IRealtimeHub await completer.UpdateSession(conn, turnDetection: false); // Add dialog history - foreach (var item in dialogs) - { - await completer.InsertConversationItem(item); - } + //foreach (var item in dialogs) + //{ + // await completer.InsertConversationItem(item); + //} if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { From ae84088b5fb5f6afd6c07ad06e15770e61c2648f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 6 Mar 2025 11:11:45 -0600 Subject: [PATCH 40/84] refine instruction log --- .../Instructs/Models/InstructLogFilter.cs | 3 +- .../Loggers/Models/InstructionLogModel.cs | 3 + .../Loggers/Services/ILoggerService.cs | 9 +++ .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Instructs/InsturctionPlugin.cs | 5 +- .../BotSharp.Core/Loggers/LoggerPlugin.cs | 15 ++++ .../Loggers/Services/LoggerService.cs | 78 +++++++++++++++++++ .../FileRepository/FileRepository.Log.cs | 11 +-- src/Infrastructure/BotSharp.Core/Using.cs | 4 +- .../Controllers/LoggerController.cs | 13 +++- .../Instructs/InstructionLogViewModel.cs | 5 ++ .../Repository/MongoRepository.Log.cs | 7 -- 12 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs create mode 100644 src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs create mode 100644 src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs index 9d154052..6e192619 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs @@ -1,5 +1,3 @@ -using System.Collections.Generic; - namespace BotSharp.Abstraction.Instructs.Models; public class InstructLogFilter : Pagination @@ -8,6 +6,7 @@ public class InstructLogFilter : Pagination public List? Providers { get; set; } public List? Models { get; set; } public List? TemplateNames { get; set; } + public List? UserIds { get; set; } public static InstructLogFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs index 844d7942..de843023 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -39,6 +39,9 @@ public class InstructionLogModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UserId { get; set; } + [JsonIgnore] + public string? UserName { get; set; } + [JsonIgnore] public Dictionary States { get; set; } = []; diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs new file mode 100644 index 00000000..aa970bbd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -0,0 +1,9 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; + +namespace BotSharp.Abstraction.Loggers.Services; + +public interface ILoggerService +{ + Task> GetInstructionLogs(InstructLogFilter filter); +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 044149f5..00e1f30a 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs index 663c710a..47f6b245 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Users.Enums; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Instructs; @@ -22,8 +21,8 @@ public class InsturctionPlugin : IBotSharpPlugin { SubMenu = new List { - new PluginMenuDef("Instruction", link: "page/instruction"), - new PluginMenuDef("Log", link: "page/instruction/log") { Roles = [UserRole.Root, UserRole.Admin] } + new PluginMenuDef("Testing", link: "page/instruction/testing"), + new PluginMenuDef("Log", link: "page/instruction/log") } }); diff --git a/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs b/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs new file mode 100644 index 00000000..bc99b9c8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Loggers; + +public class LoggerPlugin : IBotSharpPlugin +{ + public string Id => "ea1aade7-7e29-4f13-a78b-2b1835aa4fea"; + public string Name => "Logger"; + public string Description => "Provide log service"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs new file mode 100644 index 00000000..fbd1d78a --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs @@ -0,0 +1,78 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users.Enums; +using BotSharp.Abstraction.Users.Models; + +namespace BotSharp.Core.Loggers.Services; + +public class LoggerService : ILoggerService +{ + private readonly IServiceProvider _services; + private readonly IUserIdentity _user; + private readonly ILogger _logger; + + public LoggerService( + IServiceProvider services, + IUserIdentity user, + ILogger logger) + { + _services = services; + _user = user; + _logger = logger; + } + + public async Task> GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(_user.Id); + var isAdmin = UserConstant.AdminRoles.Contains(user?.Role); + if (!isAdmin && user?.Id == null) return new(); + + filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : []; + + var agents = new List(); + var users = new List(); + + var db = _services.GetRequiredService(); + var logs = db.GetInstructionLogs(filter); + var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); + var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList(); + agents = db.GetAgents(new AgentFilter + { + AgentIds = agentIds, + Pager = new Pagination { Size = filter.Size } + }); + + if (isAdmin) + { + users = db.GetUserByIds(userIds); + } + + var items = logs.Items.Select(x => + { + x.AgentId = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + + if (!isAdmin) + { + x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null; + } + else + { + var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null; + x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null; + } + return x; + }).ToList(); + + return new PagedItems + { + Items = items, + Count = logs.Count + }; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index ea1f27b5..398fb03a 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -182,6 +182,10 @@ namespace BotSharp.Core.Repository { matched = matched && filter.TemplateNames.Contains(log.TemplateName); } + if (!filter.UserIds.IsNullOrEmpty()) + { + matched = matched && filter.UserIds.Contains(log.UserId); + } if (!matched) continue; @@ -190,12 +194,6 @@ namespace BotSharp.Core.Repository } var records = logs.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size); - var agentIds = records.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); - var agents = GetAgents(new AgentFilter - { - AgentIds = agentIds - }); - records = records.Select(x => { var states = x.InnerStates.ToDictionary(p => p.Key, p => @@ -203,7 +201,6 @@ namespace BotSharp.Core.Repository var data = p.Value.RootElement.GetProperty("data"); return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; }); - x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; x.States = states ?? []; return x; }).ToList(); diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 8655afb1..0afac090 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -39,6 +39,8 @@ global using BotSharp.Abstraction.SideCar.Attributes; global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Enums; global using BotSharp.Abstraction.Statistics.Services; +global using BotSharp.Abstraction.Loggers.Services; +global using BotSharp.Abstraction.Infrastructures.Events; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; @@ -46,4 +48,4 @@ global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; global using BotSharp.Core.Statistics.Services; -global using BotSharp.Abstraction.Infrastructures.Events; \ No newline at end of file +global using BotSharp.Core.Loggers.Services; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index 2c46b5b4..de6748db 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,6 +1,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Loggers.Services; using BotSharp.OpenAPI.ViewModels.Instructs; using Microsoft.AspNetCore.Hosting; @@ -11,10 +11,14 @@ namespace BotSharp.OpenAPI.Controllers; public class LoggerController : ControllerBase { private readonly IServiceProvider _services; + private readonly IUserIdentity _user; - public LoggerController(IServiceProvider services) + public LoggerController( + IServiceProvider services, + IUserIdentity user) { _services = services; + _user = user; } [HttpGet("/logger/full-log")] @@ -52,8 +56,9 @@ public class LoggerController : ControllerBase [HttpGet("/logger/instruction/log")] public async Task> GetInstructionLogs([FromQuery] InstructLogFilter request) { - var db = _services.GetRequiredService(); - var logs = db.GetInstructionLogs(request); + var logging = _services.GetRequiredService(); + var logs = await logging.GetInstructionLogs(request); + return new PagedItems { Items = logs.Items.Select(x => InstructionLogViewModel.From(x)), diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs index 88fc27e2..aca73f76 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs @@ -40,6 +40,10 @@ public class InstructionLogViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UserId { get; set; } + [JsonPropertyName("user_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserName { get; set; } + [JsonPropertyName("states")] public Dictionary States { get; set; } = []; @@ -60,6 +64,7 @@ public class InstructionLogViewModel SystemInstruction = log.SystemInstruction, CompletionText = log.CompletionText, UserId = log.UserId, + UserName = log.UserName, States = log.States, CreatedTime = log.CreatedTime }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 05d5a6a9..20cca2a7 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -178,16 +178,9 @@ public partial class MongoRepository var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); var count = _dc.InstructionLogs.CountDocuments(filterDef); - var agentIds = docs.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); - var agents = GetAgents(new AgentFilter - { - AgentIds = agentIds - }); - var logs = docs.Select(x => { var log = InstructionLogBetaDocument.ToDomainModel(x); - log.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; log.States = x.States.ToDictionary(p => p.Key, p => { var jsonStr = p.Value.ToJson(); From c4d2f5b0261245be62ad028168dd775b3f1785ac Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 6 Mar 2025 11:31:53 -0600 Subject: [PATCH 41/84] relocate --- .../Conversations/IConversationService.cs | 2 - .../Loggers/Services/ILoggerService.cs | 7 ++ .../Services/LoggerService.Conversation.cs} | 5 +- .../Services/LoggerService.Instruction.cs | 64 +++++++++++++++++++ .../Loggers/Services/LoggerService.cs | 62 +----------------- .../Controllers/LoggerController.cs | 8 +-- 6 files changed, 78 insertions(+), 70 deletions(-) rename src/Infrastructure/BotSharp.Core/{Conversations/Services/ConversationService.Log.cs => Loggers/Services/LoggerService.Conversation.cs} (83%) create mode 100644 src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 6d84103e..613940bf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -27,8 +27,6 @@ public interface IConversationService /// If not null, delete messages while input a new message; otherwise delete messages only /// Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null); - Task> GetConversationContentLogs(string conversationId); - Task> GetConversationStateLogs(string conversationId); /// /// Send message to LLM diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs index aa970bbd..23a4809f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -5,5 +5,12 @@ namespace BotSharp.Abstraction.Loggers.Services; public interface ILoggerService { + #region Conversation + Task> GetConversationContentLogs(string conversationId); + Task> GetConversationStateLogs(string conversationId); + #endregion + + #region Instruction Task> GetInstructionLogs(InstructLogFilter filter); + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs similarity index 83% rename from src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs rename to src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs index 7b583f3f..5ce057c3 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs @@ -1,9 +1,8 @@ using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Repositories; -namespace BotSharp.Core.Conversations.Services; +namespace BotSharp.Core.Loggers.Services; -public partial class ConversationService +public partial class LoggerService { public async Task> GetConversationContentLogs(string conversationId) { diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs new file mode 100644 index 00000000..18a0564b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs @@ -0,0 +1,64 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users.Enums; +using BotSharp.Abstraction.Users.Models; + +namespace BotSharp.Core.Loggers.Services; + +public partial class LoggerService +{ + public async Task> GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(_user.Id); + var isAdmin = UserConstant.AdminRoles.Contains(user?.Role); + if (!isAdmin && user?.Id == null) return new(); + + filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : []; + + var agents = new List(); + var users = new List(); + + var db = _services.GetRequiredService(); + var logs = db.GetInstructionLogs(filter); + var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); + var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList(); + agents = db.GetAgents(new AgentFilter + { + AgentIds = agentIds, + Pager = new Pagination { Size = filter.Size } + }); + + if (isAdmin) + { + users = db.GetUserByIds(userIds); + } + + var items = logs.Items.Select(x => + { + x.AgentId = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + + if (!isAdmin) + { + x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null; + } + else + { + var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null; + x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null; + } + return x; + }).ToList(); + + return new PagedItems + { + Items = items, + Count = logs.Count + }; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs index fbd1d78a..bd9a5147 100644 --- a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs @@ -1,11 +1,6 @@ -using BotSharp.Abstraction.Instructs.Models; -using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Users.Enums; -using BotSharp.Abstraction.Users.Models; - namespace BotSharp.Core.Loggers.Services; -public class LoggerService : ILoggerService +public partial class LoggerService : ILoggerService { private readonly IServiceProvider _services; private readonly IUserIdentity _user; @@ -20,59 +15,4 @@ public class LoggerService : ILoggerService _user = user; _logger = logger; } - - public async Task> GetInstructionLogs(InstructLogFilter filter) - { - if (filter == null) - { - filter = InstructLogFilter.Empty(); - } - - var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); - var isAdmin = UserConstant.AdminRoles.Contains(user?.Role); - if (!isAdmin && user?.Id == null) return new(); - - filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : []; - - var agents = new List(); - var users = new List(); - - var db = _services.GetRequiredService(); - var logs = db.GetInstructionLogs(filter); - var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); - var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList(); - agents = db.GetAgents(new AgentFilter - { - AgentIds = agentIds, - Pager = new Pagination { Size = filter.Size } - }); - - if (isAdmin) - { - users = db.GetUserByIds(userIds); - } - - var items = logs.Items.Select(x => - { - x.AgentId = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; - - if (!isAdmin) - { - x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null; - } - else - { - var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null; - x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null; - } - return x; - }).ToList(); - - return new PagedItems - { - Items = items, - Count = logs.Count - }; - } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index de6748db..d67ba8cd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -42,15 +42,15 @@ public class LoggerController : ControllerBase [HttpGet("/logger/conversation/{conversationId}/content-log")] public async Task> GetConversationContentLogs([FromRoute] string conversationId) { - var conversationService = _services.GetRequiredService(); - return await conversationService.GetConversationContentLogs(conversationId); + var logging = _services.GetRequiredService(); + return await logging.GetConversationContentLogs(conversationId); } [HttpGet("/logger/conversation/{conversationId}/state-log")] public async Task> GetConversationStateLogs([FromRoute] string conversationId) { - var conversationService = _services.GetRequiredService(); - return await conversationService.GetConversationStateLogs(conversationId); + var logging = _services.GetRequiredService(); + return await logging.GetConversationStateLogs(conversationId); } [HttpGet("/logger/instruction/log")] From c8a565f5853be08fa0df18d9c3179e5ab4f00d98 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 6 Mar 2025 12:01:43 -0600 Subject: [PATCH 42/84] Seperate realtime in a new project --- BotSharp.sln | 11 +++++++++++ .../Realtime/Models/RealtimeHubConnection.cs | 6 +++--- .../BotSharp.Core.Realtime.csproj | 13 +++++++++++++ .../BotSharp.Core.Realtime/RealtimePlugin.cs | 18 ++++++++++++++++++ .../Services}/RealtimeHub.cs | 19 ++++--------------- .../BotSharp.Core.Realtime/Using.cs | 17 +++++++++++++++++ .../BotSharp.Core/BotSharpCoreExtensions.cs | 3 --- .../Realtime/RealTimeCompletionProvider.cs | 4 ++-- src/WebStarter/WebStarter.csproj | 3 ++- src/WebStarter/appsettings.json | 1 + 10 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj create mode 100644 src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs rename src/Infrastructure/{BotSharp.Core/Realtime => BotSharp.Core.Realtime/Services}/RealtimeHub.cs (94%) create mode 100644 src/Infrastructure/BotSharp.Core.Realtime/Using.cs diff --git a/BotSharp.sln b/BotSharp.sln index d4890be1..b176747f 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -127,6 +127,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Rules", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.DeepSeekAI", "src\Plugins\BotSharp.Plugin.DeepSeekAI\BotSharp.Plugin.DeepSeekAI.csproj", "{AF329442-B48E-4B48-A18A-1C869D1BA6F5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "src\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj", "{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -519,6 +521,14 @@ Global {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|Any CPU.Build.0 = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.ActiveCfg = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.Build.0 = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.Build.0 = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.Build.0 = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.ActiveCfg = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -580,6 +590,7 @@ Global {F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AFD64412-4D6A-452E-82A2-79E5D8842E29} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA} + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index d1fc6a77..9fa3ee3b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -8,7 +8,7 @@ public class RealtimeHubConnection public string StreamId { get; set; } = null!; public string? LastAssistantItemId { get; set; } = null!; public long LatestMediaTimestamp { get; set; } - public long? ResponseStartTimestampTwilio { get; set; } + public long? ResponseStartTimestamp { get; set; } public string KeypadInputBuffer { get; set; } = string.Empty; public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; @@ -23,12 +23,12 @@ public class RealtimeHubConnection { MarkQueue.Clear(); LastAssistantItemId = null; - ResponseStartTimestampTwilio = null; + ResponseStartTimestamp = null; } public void ResetStreamState() { - ResponseStartTimestampTwilio = null; + ResponseStartTimestamp = null; LatestMediaTimestamp = 0; } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj new file mode 100644 index 00000000..b29ff39e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -0,0 +1,13 @@ + + + + net8.0 + enable + enable + + + + + + + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs new file mode 100644 index 00000000..e632dcd3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -0,0 +1,18 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Core.Realtime.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Realtime; + +public class RealtimePlugin : IBotSharpPlugin +{ + public string Id => "68c1c737-5c21-49de-b141-cd5c8d9bf978"; + public string Name => "Realtime Hub"; + public string? IconUrl => "https://thumbs.dreamstime.com/b/microphone-icon-sound-waves-voice-command-recording-message-sign-349007898.jpg"; + public string Description => "Build low-latency, multi-modal experiences with the Realtime API."; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs similarity index 94% rename from src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 219cd6ba..924a0e64 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,15 +1,4 @@ -using BotSharp.Abstraction.Realtime; -using System.Net.WebSockets; -using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Routing.Models; -using NetTopologySuite.Index.HPRtree; -using BotSharp.Abstraction.Agents.Models; -using Microsoft.Identity.Client.Extensions.Msal; -using Microsoft.AspNetCore.Cors.Infrastructure; - -namespace BotSharp.Core.Realtime; +namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub { @@ -127,10 +116,10 @@ public class RealtimeHub : IRealtimeHub await SendEventToUser(userWebSocket, data); // If this is the first delta of a new response, set the start timestamp - if (!conn.ResponseStartTimestampTwilio.HasValue) + if (!conn.ResponseStartTimestamp.HasValue) { - conn.ResponseStartTimestampTwilio = conn.LatestMediaTimestamp; - _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestampTwilio}ms"); + conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms"); } // Record last assistant item ID for interruption handling if (!string.IsNullOrEmpty(itemId)) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Using.cs b/src/Infrastructure/BotSharp.Core.Realtime/Using.cs new file mode 100644 index 00000000..68c2458a --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Using.cs @@ -0,0 +1,17 @@ +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.DependencyInjection; + +global using System.Net.WebSockets; +global using System.Text; +global using System.Text.Json; + +global using BotSharp.Abstraction.Realtime; +global using BotSharp.Abstraction.Realtime.Models; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Conversations.Enums; +global using BotSharp.Abstraction.Routing.Models; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Conversations.Models; diff --git a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs index 742b39e9..6b10ad81 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs @@ -16,7 +16,6 @@ using BotSharp.Abstraction.Templating; using BotSharp.Core.Templating; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Realtime; -using BotSharp.Core.Realtime; namespace BotSharp.Core; @@ -173,7 +172,5 @@ public static class BotSharpCoreExtensions }); services.AddSingleton(loader); - - services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 7f31c95e..6e314a99 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -207,9 +207,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestampTwilio != null) + if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) { - var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestampTwilio; + var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; if (!string.IsNullOrEmpty(conn.LastAssistantItemId)) { diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 59cac76a..78c4537c 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -36,6 +36,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index a0f02893..04216a1b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -391,6 +391,7 @@ "BotSharp.Core", "BotSharp.Core.SideCar", "BotSharp.Core.Crontab", + "BotSharp.Core.Realtime", "BotSharp.Logger", "BotSharp.Plugin.MongoStorage", "BotSharp.Plugin.Dashboard", From f9cc755444fbfbc2367574dd7432174383f52734 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 6 Mar 2025 15:10:17 -0600 Subject: [PATCH 43/84] RealtimeConversationHook --- .../Realtime/IRealtimeHub.cs | 9 +- .../Hooks/RealtimeConversationHook.cs | 64 ++++++++ .../BotSharp.Core.Realtime/RealtimePlugin.cs | 2 + .../Services/RealtimeHub.cs | 140 +++++++++--------- .../Realtime/RealTimeCompletionProvider.cs | 7 +- .../Services/Stream/TwilioStreamMiddleware.cs | 11 +- 6 files changed, 148 insertions(+), 85 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 67d0f18c..23ecb654 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime.Models; using System.Net.WebSockets; @@ -8,5 +9,11 @@ namespace BotSharp.Abstraction.Realtime; /// public interface IRealtimeHub { - Task Listen(WebSocket userWebSocket, Func onUserMessageReceived); + RealtimeHubConnection HubConn { get; } + RealtimeHubConnection SetHubConnection(string conversationId); + + IRealTimeCompletion Completer { get; } + IRealTimeCompletion SetCompleter(string provider); + + Task Listen(WebSocket userWebSocket, Action onUserMessageReceived); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs new file mode 100644 index 00000000..e7478e29 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -0,0 +1,64 @@ +using BotSharp.Abstraction.Utilities; + +namespace BotSharp.Core.Realtime.Hooks; + +public class RealtimeConversationHook : ConversationHookBase, IConversationHook +{ + private readonly IServiceProvider _services; + public RealtimeConversationHook(IServiceProvider services) + { + _services = services; + } + + public async Task OnFunctionExecuting(RoleDialogModel message) + { + var hub = _services.GetRequiredService(); + if (hub.HubConn == null) + { + return; + } + // Save states + var states = _services.GetRequiredService(); + states.SaveStateByArgs(message.FunctionArgs?.JsonContent() ?? JsonDocument.Parse("{}")); + } + + public async Task OnFunctionExecuted(RoleDialogModel message) + { + var hub = _services.GetRequiredService(); + if (hub.HubConn == null) + { + return; + } + var routing = _services.GetRequiredService(); + + message.Role = AgentRole.Function; + + if (message.FunctionName == "route_to_agent") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") ?? new(); + message.Content = $"Connected to agent of {inst.AgentName}"; + hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await hub.Completer.UpdateSession(hub.HubConn); + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures."); + } + else if (message.FunctionName == "util-routing-fallback_to_router") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") ?? new(); + message.Content = $"Returned to Router due to {inst.Reason}"; + hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await hub.Completer.UpdateSession(hub.HubConn); + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}"); + } + else + { + // Update session for changed states + await hub.Completer.UpdateSession(hub.HubConn); + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference("Reply based on the function's output."); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs index e632dcd3..a45325de 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Plugins; +using BotSharp.Core.Realtime.Hooks; using BotSharp.Core.Realtime.Services; using Microsoft.Extensions.Configuration; @@ -14,5 +15,6 @@ public class RealtimePlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 924a0e64..68636bd1 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Utilities; + namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub @@ -5,6 +7,12 @@ public class RealtimeHub : IRealtimeHub private readonly IServiceProvider _services; private readonly ILogger _logger; + private RealtimeHubConnection _conn; + public RealtimeHubConnection HubConn => _conn; + + private IRealTimeCompletion _completer; + public IRealTimeCompletion Completer => _completer; + public RealtimeHub(IServiceProvider services, ILogger logger) { _services = services; @@ -12,12 +20,12 @@ public class RealtimeHub : IRealtimeHub } public async Task Listen(WebSocket userWebSocket, - Func onUserMessageReceived) + Action onUserMessageReceived) { var buffer = new byte[1024 * 16]; WebSocketReceiveResult result; - var completer = _services.GetServices().First(x => x.Provider == "openai"); + do { @@ -29,40 +37,40 @@ public class RealtimeHub : IRealtimeHub continue; } - var conn = onUserMessageReceived(receivedText); + onUserMessageReceived(receivedText); - if (conn.Event == "user_connected") + if (_conn.Event == "user_connected") { - await ConnectToModel(completer, userWebSocket, conn); + await ConnectToModel(userWebSocket); } - else if (conn.Event == "user_data_received") + else if (_conn.Event == "user_data_received") { - await completer.AppenAudioBuffer(conn.Data); + await _completer.AppenAudioBuffer(_conn.Data); } - else if (conn.Event == "user_dtmf_received") + else if (_conn.Event == "user_dtmf_received") { - await HandleUserDtmfReceived(completer, conn); + await HandleUserDtmfReceived(); } - else if (conn.Event == "user_disconnected") + else if (_conn.Event == "user_disconnected") { - await completer.Disconnect(); - await HandleUserDisconnected(conn); + await _completer.Disconnect(); + await HandleUserDisconnected(); } } while (!result.CloseStatus.HasValue); await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); } - private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn) + private async Task ConnectToModel(WebSocket userWebSocket) { var hookProvider = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - convService.SetConversationId(conn.ConversationId, []); - var conversation = await convService.GetConversation(conn.ConversationId); + convService.SetConversationId(_conn.ConversationId, []); + var conversation = await convService.GetConversation(_conn.ConversationId); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conversation.AgentId); - conn.CurrentAgentId = agent.Id; + _conn.CurrentAgentId = agent.Id; // Set model var model = agent.LlmConfig.Model; @@ -72,8 +80,8 @@ public class RealtimeHub : IRealtimeHub model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name; } - completer.SetModelName(model); - conn.Model = model; + _completer.SetModelName(model); + _conn.Model = model; var routing = _services.GetRequiredService(); routing.Context.Push(agent.Id); @@ -85,54 +93,48 @@ public class RealtimeHub : IRealtimeHub } routing.Context.SetDialogs(dialogs); - await completer.Connect(conn, + await _completer.Connect(_conn, onModelReady: async () => { // Control initial session, prevent initial response interruption - await completer.UpdateSession(conn, turnDetection: false); - - // Add dialog history - //foreach (var item in dialogs) - //{ - // await completer.InsertConversationItem(item); - //} + await _completer.UpdateSession(_conn, turnDetection: false); if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { - await completer.TriggerModelInference("Reply based on the conversation context."); + await _completer.TriggerModelInference("Reply based on the conversation context."); } // Start turn detection await Task.Delay(1000 * 8); - await completer.UpdateSession(conn, turnDetection: true); + await _completer.UpdateSession(_conn, turnDetection: true); }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => { - var data = conn.OnModelMessageReceived(audioDeltaData); + var data = _conn.OnModelMessageReceived(audioDeltaData); await SendEventToUser(userWebSocket, data); // If this is the first delta of a new response, set the start timestamp - if (!conn.ResponseStartTimestamp.HasValue) + if (!_conn.ResponseStartTimestamp.HasValue) { - conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; - _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms"); + _conn.ResponseStartTimestamp = _conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {_conn.ResponseStartTimestamp}ms"); } // Record last assistant item ID for interruption handling if (!string.IsNullOrEmpty(itemId)) { - conn.LastAssistantItemId = itemId; + _conn.LastAssistantItemId = itemId; } // Send mark messages to Media Streams so we know if and when AI response playback is finished - await SendMark(userWebSocket, conn); + await SendMark(userWebSocket, _conn); }, onModelAudioResponseDone: async () => { - var data = conn.OnModelAudioResponseDone(); + var data = _conn.OnModelAudioResponseDone(); await SendEventToUser(userWebSocket, data); }, onAudioTranscriptDone: async transcript => @@ -144,36 +146,10 @@ public class RealtimeHub : IRealtimeHub foreach (var message in messages) { // Invoke function - if (message.MessageType == MessageTypeName.FunctionCall) + if (message.MessageType == MessageTypeName.FunctionCall && + !string.IsNullOrEmpty(message.FunctionName)) { await routing.InvokeFunction(message.FunctionName, message); - message.Role = AgentRole.Function; - - if (message.FunctionName == "route_to_agent") - { - var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - message.Content = $"Connected to agent of {inst.AgentName}"; - conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); - - await completer.UpdateSession(conn); - await completer.InsertConversationItem(message); - await completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures."); - } - else if (message.FunctionName == "util-routing-fallback_to_router") - { - var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - message.Content = $"Returned to Router due to {inst.Reason}"; - conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); - - await completer.UpdateSession(conn); - await completer.InsertConversationItem(message); - await completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}"); - } - else - { - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); - } } else { @@ -210,9 +186,9 @@ public class RealtimeHub : IRealtimeHub onUserInterrupted: async () => { // Reset states - conn.ResetResponseState(); + _conn.ResetResponseState(); - var data = conn.OnModelUserInterrupted(); + var data = _conn.OnModelUserInterrupted(); await SendEventToUser(userWebSocket, data); }); } @@ -232,17 +208,17 @@ public class RealtimeHub : IRealtimeHub } } - private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn) + private async Task HandleUserDtmfReceived() { var routing = _services.GetRequiredService(); var hookProvider = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conn.CurrentAgentId); + var agent = await agentService.LoadAgent(_conn.CurrentAgentId); var dialogs = routing.Context.GetDialogs(); var convService = _services.GetRequiredService(); - var conversation = await convService.GetConversation(conn.ConversationId); + var conversation = await convService.GetConversation(_conn.ConversationId); - var message = new RoleDialogModel(AgentRole.User, conn.Data) + var message = new RoleDialogModel(AgentRole.User, _conn.Data) { CurrentAgentId = routing.Context.GetCurrentAgentId() }; @@ -256,11 +232,11 @@ public class RealtimeHub : IRealtimeHub await hook.OnMessageReceived(message); } - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the user input"); + await _completer.InsertConversationItem(message); + await _completer.TriggerModelInference("Reply based on the user input"); } - private async Task HandleUserDisconnected(RealtimeHubConnection conn) + private async Task HandleUserDisconnected() { // Save dialog history var routing = _services.GetRequiredService(); @@ -268,7 +244,7 @@ public class RealtimeHub : IRealtimeHub var dialogs = routing.Context.GetDialogs(); foreach (var item in dialogs) { - storage.Append(conn.ConversationId, item); + storage.Append(_conn.ConversationId, item); } } @@ -278,4 +254,20 @@ public class RealtimeHub : IRealtimeHub var buffer = Encoding.UTF8.GetBytes(data); await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } + + public RealtimeHubConnection SetHubConnection(string conversationId) + { + _conn = new RealtimeHubConnection + { + ConversationId = conversationId + }; + + return _conn; + } + + public IRealTimeCompletion SetCompleter(string provider) + { + _completer = _services.GetServices().First(x => x.Provider == provider); + return _completer; + } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 6e314a99..323d0b62 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -550,7 +550,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion })); prompt += $"{verbose}\r\n"; - prompt += "\r\n[CONVERSATION]"; verbose = string.Join("\r\n", messages .Where(x => x as SystemChatMessage == null) .Select(x => @@ -581,7 +580,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return string.Empty; })); - prompt += $"\r\n{verbose}\r\n"; + + if (!string.IsNullOrEmpty(verbose)) + { + prompt += $"\r\n[CONVERSATION]\r\n{verbose}\r\n"; + } } if (!options.Tools.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 0e3a68c0..35118ab3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -51,12 +51,9 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); - - var conn = new RealtimeHubConnection - { - ConversationId = conversationId - }; - + var conn = hub.SetHubConnection(conversationId); + var completer = hub.SetCompleter("openai"); + // load conversation and state var convService = services.GetRequiredService(); convService.SetConversationId(conversationId, []); @@ -131,8 +128,6 @@ public class TwilioStreamMiddleware @event = "clear", streamSid = response.StreamSid }; - - return conn; }); } } From 28d4e3291a961133aa701f876b036c1308124a90 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 6 Mar 2025 15:10:30 -0600 Subject: [PATCH 44/84] Upgrade Anthropic.SDK --- Directory.Packages.props | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index f553f838..4683fc9d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,7 +13,7 @@ - + @@ -29,7 +29,7 @@ - + @@ -43,7 +43,7 @@ - + @@ -67,7 +67,7 @@ - + From 1275053d660c61c27f8be54af724d6724da4e2b0 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 6 Mar 2025 16:46:38 -0600 Subject: [PATCH 45/84] migrate conv latest states --- .../Conversations/IConversationService.cs | 2 + .../Repositories/IBotSharpRepository.cs | 4 + .../Services/ConversationService.Migration.cs | 98 +++++++++++++++++++ .../Services/LoggerService.Instruction.cs | 2 +- .../FileRepository.Conversation.cs | 55 +++++++++++ .../Controllers/ConversationController.cs | 10 ++ .../MigrateLatestStateRequest.cs | 7 ++ .../MongoRepository.Conversation.cs | 38 +++++++ 8 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 613940bf..6e558b1b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -70,4 +70,6 @@ public interface IConversationService /// if pre-loading, then keys are not filter by the search query /// Task> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false); + + Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 6f8de07d..d523dd09 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -151,6 +151,10 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) => throw new NotImplementedException(); + List GetConversationsToMigrate(int batchSize = 100) + => throw new NotImplementedException(); + bool MigrateConvsersationLatestStates(string conversationId) + => throw new NotImplementedException(); #endregion #region LLM Completion Log diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs new file mode 100644 index 00000000..23012706 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs @@ -0,0 +1,98 @@ +using NetTopologySuite.Algorithm; +using System.Diagnostics; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10) + { + var db = _services.GetRequiredService(); + var isSuccess = true; + var errorCount = 0; + var batchNum = 0; + var info = string.Empty; + var error = string.Empty; + +#if DEBUG + Console.WriteLine($"\r\n#Start migrating Conversation Latest States...\r\n"); +#else + _logger.LogInformation($"#Start migrating Conversation Latest States..."); +#endif + var sw = Stopwatch.StartNew(); + + var convIds = db.GetConversationsToMigrate(batchSize); + + while (!convIds.IsNullOrEmpty()) + { + batchNum++; + var innerSw = Stopwatch.StartNew(); +#if DEBUG + Console.WriteLine($"\r\n#Start migrating Conversation Latest States (batch number: {batchNum})\r\n"); +#else + _logger.LogInformation($"#Start migrating Conversation Latest States (batch number: {batchNum})"); +#endif + + for (int i = 0; i < convIds.Count; i++) + { + var convId = convIds.ElementAt(i); + try + { + var done = db.MigrateConvsersationLatestStates(convId); + info = $"Conversation {convId} latest states have been migrated ({i + 1}/{convIds.Count})!"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + } + catch (Exception ex) + { + errorCount++; + error = $"Conversation {convId} latest states fail to be migrated! ({i + 1}/{convIds.Count})\r\n{ex.Message}\r\n{ex.InnerException}"; +#if DEBUG + Console.WriteLine($"\r\n{error}\r\n"); +#else + _logger.LogError($"{error}"); +#endif + } + } + + if (errorCount >= errorLimit) + { + error = $"\r\nErrors exceed limit => stop the migration!\r\n"; +#if DEBUG + Console.WriteLine($"{error}"); +#else + _logger.LogError($"{error}"); +#endif + innerSw.Stop(); + isSuccess = false; + break; + } + + innerSw.Stop(); + info = $"#Done migrating Conversation Latest States (batch number: {batchNum}) " + + $"(Total time: {innerSw.Elapsed.Hours} hrs, {innerSw.Elapsed.Minutes} mins, {innerSw.Elapsed.Seconds} seconds)"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + + await Task.Delay(100); + convIds = db.GetConversationsToMigrate(batchSize); + } + + sw.Stop(); + info = $"#Done with migrating Conversation Latest States! " + + $"(Total time: {sw.Elapsed.Days} days, {sw.Elapsed.Hours} hrs, {sw.Elapsed.Minutes} mins, {sw.Elapsed.Seconds} seconds)"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + + return isSuccess; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs index 18a0564b..84efae99 100644 --- a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs @@ -41,7 +41,7 @@ public partial class LoggerService var items = logs.Items.Select(x => { - x.AgentId = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; if (!isAdmin) { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 08a534d4..8880a4e0 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -687,6 +687,56 @@ public partial class FileRepository } + + public List GetConversationsToMigrate(int batchSize = 100) + { + var baseDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(baseDir)) return []; + + var convIds = new List(); + var dirs = Directory.GetDirectories(baseDir); + + foreach (var dir in dirs) + { + var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) continue; + + var convId = dir.Split(Path.DirectorySeparatorChar).Last(); + if (string.IsNullOrEmpty(convId)) continue; + + convIds.Add(convId); + if (convIds.Count >= batchSize) + { + break; + } + } + + return convIds; + } + + + public bool MigrateConvsersationLatestStates(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) + { + return false; + } + + var stateFile = Path.Combine(convDir, STATE_FILE); + var states = CollectConversationStates(stateFile); + var latestStates = BuildLatestStates(states); + + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateFile, stateStr); + + return true; + } + + #region Private methods private string? FindConversationDirectory(string conversationId) { @@ -883,6 +933,11 @@ public partial class FileRepository private Dictionary BuildLatestStates(List states) { var endNodes = new Dictionary(); + if (states.IsNullOrEmpty()) + { + return endNodes; + } + foreach (var pair in states) { var value = pair.Values?.LastOrDefault(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 75dc1113..10317815 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -563,6 +563,16 @@ public class ConversationController : ControllerBase } #endregion + #region Migrate Latest States + [HttpPost("/conversation/latest-state/migrate")] + public async Task MigrateConversationLatestStates([FromBody] MigrateLatestStateRequest request) + { + var convService = _services.GetRequiredService(); + var res = await convService.MigrateLatestStates(request.BatchSize, request.ErrorLimit); + return res; + } + #endregion + #region Private methods private void SetStates(IConversationService conv, NewMessageModel input) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs new file mode 100644 index 00000000..99085b0a --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class MigrateLatestStateRequest +{ + public int BatchSize { get; set; } = 1000; + public int ErrorLimit { get; set; } = 10; +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 1c0eeb2f..b9931bd4 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -661,6 +662,37 @@ public partial class MongoRepository return keys; } + + + public List GetConversationsToMigrate(int batchSize = 100) + { + var convFilter = Builders.Filter.Exists(x => x.LatestStates, false); + var sortDef = Builders.Sort.Ascending(x => x.CreatedTime); + var convIds = _dc.Conversations.Find(convFilter).Sort(sortDef) + .Limit(batchSize).ToEnumerable() + .Select(x => x.Id).ToList(); + return convIds ?? []; + } + + public bool MigrateConvsersationLatestStates(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var stateFilter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault(); + if (foundStates?.States == null) return false; + + var states = foundStates.States.ToList(); + var latestStates = BuildLatestStates(states); + + var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); + var convUpdate = Builders.Update.Set(x => x.LatestStates, latestStates); + _dc.Conversations.UpdateOne(convFilter, convUpdate); + + return true; + } + + #region Private methods private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); @@ -682,6 +714,11 @@ public partial class MongoRepository private Dictionary BuildLatestStates(List states) { var endNodes = new Dictionary(); + if (states.IsNullOrEmpty()) + { + return endNodes; + } + foreach (var pair in states) { var value = pair.Values?.LastOrDefault(); @@ -703,4 +740,5 @@ public partial class MongoRepository return endNodes; } + #endregion } From 6c735a0aad3dbd31930f6a1d3487bfc14ebf309c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 6 Mar 2025 17:44:49 -0600 Subject: [PATCH 46/84] refine document --- .../Plugins/Models/PluginMenuDef.cs | 3 +++ ...aDocument.cs => InstructionLogDocument.cs} | 8 +++---- .../MongoDbContext.cs | 21 +++++++++++++++---- .../Repository/MongoRepository.Log.cs | 12 +++++------ 4 files changed, 30 insertions(+), 14 deletions(-) rename src/Plugins/BotSharp.Plugin.MongoStorage/Collections/{InstructionLogBetaDocument.cs => InstructionLogDocument.cs} (88%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs index c387d1ca..8742312d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs @@ -13,6 +13,9 @@ public class PluginMenuDef(string label, string? link = null, string? icon = nul [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Link { get; set; } = link; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? IFrameUrl { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? IsHeader { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogDocument.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs rename to src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogDocument.cs index 4f70ad60..91bcdc68 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogDocument.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Loggers.Models; namespace BotSharp.Plugin.MongoStorage.Collections; -public class InstructionLogBetaDocument : MongoBase +public class InstructionLogDocument : MongoBase { public string? AgentId { get; set; } public string Provider { get; set; } = default!; @@ -15,9 +15,9 @@ public class InstructionLogBetaDocument : MongoBase public Dictionary States { get; set; } = new(); public DateTime CreatedTime { get; set; } - public static InstructionLogBetaDocument ToMongoModel(InstructionLogModel log) + public static InstructionLogDocument ToMongoModel(InstructionLogModel log) { - return new InstructionLogBetaDocument + return new InstructionLogDocument { AgentId = log.AgentId, Provider = log.Provider, @@ -31,7 +31,7 @@ public class InstructionLogBetaDocument : MongoBase }; } - public static InstructionLogModel ToDomainModel(InstructionLogBetaDocument log) + public static InstructionLogModel ToDomainModel(InstructionLogDocument log) { return new InstructionLogModel { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 277f9e25..73853230 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -115,7 +115,7 @@ public class MongoDbContext { var collection = GetCollectionOrCreate("ConversationContentLogs"); var indexes = collection.Indexes.List().ToList(); - var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); if (createTimeIndex == null) { var indexDef = Builders.IndexKeys.Ascending(x => x.CreatedTime); @@ -128,7 +128,7 @@ public class MongoDbContext { var collection = GetCollectionOrCreate("ConversationStateLogs"); var indexes = collection.Indexes.List().ToList(); - var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); if (createTimeIndex == null) { var indexDef = Builders.IndexKeys.Ascending(x => x.CreatedTime); @@ -136,6 +136,19 @@ public class MongoDbContext } return collection; } + + private IMongoCollection CreateInstructionLogIndex() + { + var collection = GetCollectionOrCreate("InstructionLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } #endregion #endregion @@ -193,6 +206,6 @@ public class MongoDbContext public IMongoCollection GlobalStatistics => GetCollectionOrCreate("GlobalStatistics"); - public IMongoCollection InstructionLogs - => GetCollectionOrCreate("InstructionLogs"); + public IMongoCollection InstructionLogs + => CreateInstructionLogIndex(); } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 20cca2a7..40da0321 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -119,10 +119,10 @@ public partial class MongoRepository { if (logs.IsNullOrEmpty()) return false; - var docs = new List(); + var docs = new List(); foreach (var log in logs) { - var doc = InstructionLogBetaDocument.ToMongoModel(log); + var doc = InstructionLogDocument.ToMongoModel(log); foreach (var pair in log.States) { try @@ -152,8 +152,8 @@ public partial class MongoRepository filter = InstructLogFilter.Empty(); } - var builder = Builders.Filter; - var filters = new List>() { builder.Empty }; + var builder = Builders.Filter; + var filters = new List>() { builder.Empty }; // Filter logs if (!filter.AgentIds.IsNullOrEmpty()) @@ -174,13 +174,13 @@ public partial class MongoRepository } var filterDef = builder.And(filters); - var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); var count = _dc.InstructionLogs.CountDocuments(filterDef); var logs = docs.Select(x => { - var log = InstructionLogBetaDocument.ToDomainModel(x); + var log = InstructionLogDocument.ToDomainModel(x); log.States = x.States.ToDictionary(p => p.Key, p => { var jsonStr = p.Value.ToJson(); From 96ec0b59695070de1816c028cbba708cc11bae6d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 7 Mar 2025 11:01:54 -0600 Subject: [PATCH 47/84] rename --- .../BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs index 8742312d..77205aa9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs @@ -14,7 +14,7 @@ public class PluginMenuDef(string label, string? link = null, string? icon = nul public string? Link { get; set; } = link; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? IFrameUrl { get; set; } + public string? EmbedUrl { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? IsHeader { get; set; } From e7150729d9dfac0393dcc145f58eaa32f4b869b8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 7 Mar 2025 17:13:08 -0600 Subject: [PATCH 48/84] refine menu --- .../Plugins/Models/PluginMenuDef.cs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs index 77205aa9..f347fb84 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs @@ -14,7 +14,7 @@ public class PluginMenuDef(string label, string? link = null, string? icon = nul public string? Link { get; set; } = link; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? EmbedUrl { get; set; } + public EmbeddingData? EmbeddingInfo { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? IsHeader { get; set; } @@ -33,3 +33,35 @@ public class PluginMenuDef(string label, string? link = null, string? icon = nul return $"{Label} {Link} {Weight}"; } } + +public class EmbeddingData +{ + /// + /// Embedding source, e.g., tableau + /// + public string Source { get; set; } = default!; + + /// + /// Embedding url + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Url { get; set; } + + /// + /// Html tag + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? HtmlTag { get; set; } + + /// + /// Javascript script src + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScriptSrc { get; set; } + + /// + /// Javascript script type, e.g., module + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScriptType { get; set; } +} From 17bb86b311880806f74205ba3c69d8af8a8c7d23 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 7 Mar 2025 18:40:28 -0600 Subject: [PATCH 49/84] Return instruction for realtime response.create. --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Routing/Models/RoutingArgs.cs | 4 ---- .../Hooks/RealtimeConversationHook.cs | 14 +++++++------- .../Realtime/RealTimeCompletionProvider.cs | 4 +++- .../util-twilio-hangup_phone_call.fn.liquid | 4 +++- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 2d0150ec..075d7291 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); + Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index 5b1c6a2a..bc9a6fa8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -12,10 +12,6 @@ public class RoutingArgs [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? NextActionReason { get; set; } - [JsonPropertyName("reason")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Reason { get; set; } - [JsonPropertyName("conversation_end")] public bool ConversationEnd { get; set; } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index e7478e29..3d45ecfd 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -36,12 +36,12 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook if (message.FunctionName == "route_to_agent") { var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") ?? new(); - message.Content = $"Connected to agent of {inst.AgentName}"; + message.Content = $"I'm your AI assistant '{inst.AgentName}' to help with: '{inst.NextActionReason}'"; hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId(); - await hub.Completer.UpdateSession(hub.HubConn); + var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); - await hub.Completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures."); + await hub.Completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user task: {inst.NextActionReason}"); } else if (message.FunctionName == "util-routing-fallback_to_router") { @@ -49,16 +49,16 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook message.Content = $"Returned to Router due to {inst.Reason}"; hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId(); - await hub.Completer.UpdateSession(hub.HubConn); + var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); - await hub.Completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}"); + await hub.Completer.TriggerModelInference(instruction); } else { // Update session for changed states - await hub.Completer.UpdateSession(hub.HubConn); + var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); - await hub.Completer.TriggerModelInference("Reply based on the function's output."); + await hub.Completer.TriggerModelInference(instruction); } } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 323d0b62..1d44bc8d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -285,7 +285,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) + public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -347,6 +347,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); await SendEventToModel(sessionUpdate); + + return instruction; } public async Task InsertConversationItem(RoleDialogModel message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid index cc8360be..4a6b5034 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -1 +1,3 @@ -** Please call util-twilio-hangup_phone_call if user wants to end the phone call. \ No newline at end of file +{% if channel == 'phone' %} +** Please call util-twilio-hangup_phone_call if user wants to end the phone call. +{% endif %} \ No newline at end of file From eb81ad212abe6ff98da8acde8829d4fd180b8146 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Sun, 9 Mar 2025 22:21:34 -0500 Subject: [PATCH 50/84] add instruction log setting and state search --- .../Instructs/Models/InstructLogFilter.cs | 1 + .../Instructs/Settings/InstructionSettings.cs | 6 ++ .../BotSharp.Abstraction/Models/KeyValue.cs | 6 +- .../Instructs/InsturctionPlugin.cs | 8 ++- .../FileRepository/FileRepository.Log.cs | 60 +++++++++++++++++++ .../Hooks/InstructionLogHook.cs | 4 +- .../MongoRepository.Conversation.cs | 2 +- .../Repository/MongoRepository.Log.cs | 58 +++++++++++++++--- src/WebStarter/appsettings.json | 4 ++ 9 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs index 6e192619..43d655ce 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs @@ -7,6 +7,7 @@ public class InstructLogFilter : Pagination public List? Models { get; set; } public List? TemplateNames { get; set; } public List? UserIds { get; set; } + public List? States { get; set; } public static InstructLogFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs new file mode 100644 index 00000000..d1f51348 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Instructs.Settings; + +public class InstructionSettings +{ + public bool EnableLog { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs index b1dfa734..3cb9c73f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -3,10 +3,10 @@ namespace BotSharp.Abstraction.Models; public class KeyValue { [JsonPropertyName("key")] - public string Key { get; set; } + public string Key { get; set; } = string.Empty; [JsonPropertyName("value")] - public string Value { get; set; } + public string? Value { get; set; } public override string ToString() { @@ -17,7 +17,7 @@ public class KeyValue public class KeyValue { [JsonPropertyName("key")] - public string Key { get; set; } + public string Key { get; set; } = string.Empty; [JsonPropertyName("value")] public T? Value { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs index 47f6b245..c44cb874 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Instructs.Settings; using BotSharp.Abstraction.Plugins.Models; +using BotSharp.Abstraction.Settings; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Instructs; @@ -11,7 +13,11 @@ public class InsturctionPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("Instruction"); + }); } public bool AttachMenu(List menu) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 398fb03a..0b9b27a7 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -187,6 +187,66 @@ namespace BotSharp.Core.Repository matched = matched && filter.UserIds.Contains(log.UserId); } + // Check states + if (matched && filter != null && !filter.States.IsNullOrEmpty()) + { + var logStates = log.InnerStates; + if (logStates.IsNullOrEmpty()) + { + matched = false; + } + else + { + foreach (var pair in filter.States) + { + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; + + var components = pair.Key.Split(".").ToList(); + var primaryKey = components[0]; + if (logStates.TryGetValue(primaryKey, out var doc)) + { + var elem = doc.RootElement.GetProperty("data"); + if (components.Count < 2) + { + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + if (elem.ValueKind == JsonValueKind.Null) + { + matched = false; + } + else if (elem.ValueKind == JsonValueKind.Array) + { + matched = elem.EnumerateArray().Where(x => x.ValueKind != JsonValueKind.Null) + .Select(x => x.ToString()) + .Any(x => x == pair.Value); + } + else if (elem.ValueKind == JsonValueKind.String) + { + matched = elem.GetString() == pair.Value; + } + else + { + matched = elem.GetRawText() == pair.Value; + } + } + } + else + { + var paths = components.Where((_, idx) => idx > 0); + var found = FindState(elem, paths, pair.Value); + matched = found != null; + } + } + else + { + matched = false; + } + + if (!matched) break; + } + } + } + if (!matched) continue; log.Id = Path.GetFileNameWithoutExtension(file); diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs index 265b631a..0abea5eb 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Instructs.Settings; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Users; @@ -24,7 +25,8 @@ public class InstructionLogHook : InstructHookBase public override async Task OnResponseGenerated(InstructResponseModel response) { - if (response == null) return; + var settings = _services.GetRequiredService(); + if (!settings.EnableLog || response == null) return; var db = _services.GetRequiredService(); var state = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index b9931bd4..f39ad2dc 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -383,7 +383,7 @@ public partial class MongoRepository } // Filter states - if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty()) + if (filter != null && !filter.States.IsNullOrEmpty()) { foreach (var pair in filter.States) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 40da0321..d94b1d6a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -152,28 +152,72 @@ public partial class MongoRepository filter = InstructLogFilter.Empty(); } - var builder = Builders.Filter; - var filters = new List>() { builder.Empty }; + var logBuilder = Builders.Filter; + var logFilters = new List>() { logBuilder.Empty }; // Filter logs if (!filter.AgentIds.IsNullOrEmpty()) { - filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + logFilters.Add(logBuilder.In(x => x.AgentId, filter.AgentIds)); } if (!filter.Providers.IsNullOrEmpty()) { - filters.Add(builder.In(x => x.Provider, filter.Providers)); + logFilters.Add(logBuilder.In(x => x.Provider, filter.Providers)); } if (!filter.Models.IsNullOrEmpty()) { - filters.Add(builder.In(x => x.Model, filter.Models)); + logFilters.Add(logBuilder.In(x => x.Model, filter.Models)); } if (!filter.TemplateNames.IsNullOrEmpty()) { - filters.Add(builder.In(x => x.TemplateName, filter.TemplateNames)); + logFilters.Add(logBuilder.In(x => x.TemplateName, filter.TemplateNames)); } - var filterDef = builder.And(filters); + // Filter states + if (filter != null && !filter.States.IsNullOrEmpty()) + { + foreach (var pair in filter.States) + { + if (string.IsNullOrWhiteSpace(pair.Key)) continue; + + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "States"); + var formattedKey = string.Join(".", keys); + + if (string.IsNullOrWhiteSpace(pair.Value)) + { + logFilters.Add(logBuilder.Exists(formattedKey)); + } + else if (bool.TryParse(pair.Value, out var boolValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, boolValue)); + } + else if (int.TryParse(pair.Value, out var intValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, intValue)); + } + else if (decimal.TryParse(pair.Value, out var decimalValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, decimalValue)); + } + else if (float.TryParse(pair.Value, out var floatValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, floatValue)); + } + else if (double.TryParse(pair.Value, out var doubleValue)) + { + logFilters.Add(logBuilder.Eq(formattedKey, doubleValue)); + } + else + { + logFilters.Add(logBuilder.Eq(formattedKey, pair.Value)); + } + } + } + + var filterDef = logBuilder.And(logFilters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); var count = _dc.InstructionLogs.CountDocuments(filterDef); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 04216a1b..9d8afd80 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -222,6 +222,10 @@ "Enabled": false }, + "Instruction": { + "EnableLog": true + }, + "ChatHub": { "EventDispatchBy": "group" }, From ae7680da3d7bedde6b9b3332cec7be84bb34587f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 10 Mar 2025 10:10:22 -0500 Subject: [PATCH 51/84] Hang-up if waiting AI Response Timeout --- .../Conversations/IConversationService.cs | 7 +- .../BotSharp.Plugin.Twilio.csproj | 3 +- .../Controllers/TwilioVoiceController.cs | 22 +- .../Models/ConversationalVoiceRequest.cs | 8 +- .../Services/TwilioMessageQueueService.cs | 311 +++++++++--------- 5 files changed, 185 insertions(+), 166 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 6e558b1b..4bb795ad 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -32,13 +32,12 @@ public interface IConversationService /// Send message to LLM /// /// - /// + /// + /// /// Received the response from AI Agent - /// This delegate is useful when you want to report progress on UI - /// This delegate is useful when you want to report progress on UI /// Task SendMessage(string agentId, - RoleDialogModel lastDalog, + RoleDialogModel lastDialog, PostbackMessageModel? replyMessage, Func onResponseReceived); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 3f820090..72714388 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -6,6 +6,7 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages + enable diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 7b041ca6..807d7af8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -142,7 +142,7 @@ public class TwilioVoiceController : TwilioController await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); await HookEmitter.Emit(_services, async hook => { @@ -217,8 +217,22 @@ public class TwilioVoiceController : TwilioController var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum); VoiceResponse response; + + if (request.AIResponseWaitTime > 5) + { + // Wait AI Response Timeout + await HookEmitter.Emit(_services, async hook => + { + request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!"; + await hook.OnAgentHangUp(request); + }, new HookEmitOption + { + OnlyOnce = true + }); - if (reply == null) + response = twilio.HangUp($"twilio/error.mp3"); + } + else if (reply == null) { var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); if (indication != null) @@ -262,7 +276,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { SpeechPaths = speechPaths, - CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; @@ -301,7 +315,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { SpeechPaths = instructions, - CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index e2b6ee13..33ee7633 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -5,17 +5,19 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceRequest : VoiceRequest { [FromQuery(Name = "agent-id")] - public string AgentId { get; set; } + public string AgentId { get; set; } = string.Empty; [FromRoute] - public string ConversationId { get; set; } + public string ConversationId { get; set; } = string.Empty; [FromRoute] public int SeqNum { get; set; } public int Attempts { get; set; } = 1; + public int AIResponseWaitTime { get; set; } = 0; + public string? AIResponseErrorMessage { get; set; } = string.Empty; - public string Intent { get; set; } + public string Intent { get; set; } = string.Empty; public List States { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 2c942aa5..fb992c4a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -9,175 +9,178 @@ using System.Security.Claims; using System.Threading; using Task = System.Threading.Tasks.Task; -namespace BotSharp.Plugin.Twilio.Services +namespace BotSharp.Plugin.Twilio.Services; + +public class TwilioMessageQueueService : BackgroundService { - public class TwilioMessageQueueService : BackgroundService + private readonly TwilioMessageQueue _queue; + private readonly IServiceProvider _serviceProvider; + private readonly SemaphoreSlim _throttler; + private readonly ILogger _logger; + + public TwilioMessageQueueService( + TwilioMessageQueue queue, + IServiceProvider serviceProvider, + ILogger logger) { - private readonly TwilioMessageQueue _queue; - private readonly IServiceProvider _serviceProvider; - private readonly SemaphoreSlim _throttler; + _queue = queue; + _serviceProvider = serviceProvider; + _throttler = new SemaphoreSlim(10, 10); + _logger = logger; + } - public TwilioMessageQueueService( - TwilioMessageQueue queue, - IServiceProvider serviceProvider) + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken)) { - _queue = queue; - _serviceProvider = serviceProvider; - _throttler = new SemaphoreSlim(4, 4); - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken)) + await _throttler.WaitAsync(stoppingToken); + _ = Task.Run(async () => { - await _throttler.WaitAsync(stoppingToken); - _ = Task.Run(async () => + try { - try - { - Console.WriteLine($"Start processing {message}."); - await ProcessUserMessageAsync(message); - } - catch (Exception ex) - { - Console.WriteLine($"Processing {message} failed due to {ex.Message}."); - } - finally - { - _throttler.Release(); - } - }); - } - } - - public override async Task StopAsync(CancellationToken cancellationToken) - { - _queue.Stop(); - await base.StopAsync(cancellationToken); - } - - private async Task ProcessUserMessageAsync(CallerMessage message) - { - using var scope = _serviceProvider.CreateScope(); - var sp = scope.ServiceProvider; - - // Clean static HttpContext - var httpContext = sp.GetRequiredService(); - httpContext.HttpContext = new DefaultHttpContext(); - httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity()); - foreach (var header in message.RequestHeaders ?? []) - { - httpContext.HttpContext.Request.Headers[header.Key] = header.Value; - } - httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST"; - - AssistantMessage reply = null; - var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); - var conv = sp.GetRequiredService(); - var routing = sp.GetRequiredService(); - var config = sp.GetRequiredService(); - var sessionManager = sp.GetRequiredService(); - var progressService = sp.GetRequiredService(); - InitProgressService(message, sessionManager, progressService); - InitConversation(message, inputMsg, conv, routing); - - // Need to consider Inbound and Outbound call - var conversation = await conv.GetConversation(message.ConversationId); - var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId; - - var result = await conv.SendMessage(agentId, - inputMsg, - replyMessage: BuildPostbackMessageModel(conv, message), - async msg => - { - reply = new AssistantMessage() - { - ConversationEnd = msg.Instruction?.ConversationEnd ?? false, - HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), - Content = msg.Content, - MessageId = msg.MessageId - }; + _logger.LogInformation($"Start processing {message}."); + await ProcessUserMessageAsync(message); } - ); - reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp); - reply.Hints = GetHints(reply); - reply.Content = null; - await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); - } - - private PostbackMessageModel BuildPostbackMessageModel(IConversationService conv, CallerMessage message) - { - var messages = conv.GetDialogHistory(1); - if (!messages.Any()) return null; - var lastMessage = messages[0]; - if (string.IsNullOrEmpty(lastMessage.PostbackFunctionName)) return null; - return new PostbackMessageModel - { - FunctionName = lastMessage.PostbackFunctionName, - ParentId = lastMessage.MessageId, - Payload = message.Digits - }; - } - - private static void InitConversation(CallerMessage message, RoleDialogModel inputMsg, IConversationService conv, IRoutingService routing) - { - routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); - var states = new List - { - new("channel", ConversationChannel.Phone), - new("channel_id", message.From), - new("calling_phone", message.From) - }; - states.AddRange(message.States.Select(kvp => new MessageState(kvp.Key, kvp.Value))); - conv.SetConversationId(message.ConversationId, states); - } - - private static async Task GetReplySpeechFileName(string conversationId, AssistantMessage reply, IServiceProvider sp) - { - var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); - var fileStorage = sp.GetRequiredService(); - var data = await completion.GenerateAudioFromTextAsync(reply.Content); - var fileName = $"reply_{reply.MessageId}.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - return fileName; - } - - private static string GetHints(AssistantMessage reply) - { - var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries); - int capcity = 100; - var hints = new List(capcity); - for (int i = phrases.Length - 1; i >= 0; i--) - { - var words = phrases[i].Split(' ', StringSplitOptions.RemoveEmptyEntries); - for (int j = words.Length - 1; j >= 0; j--) + catch (Exception ex) { - hints.Add(words[j]); - if (hints.Count >= capcity) - { - break; - } + _logger.LogError($"Processing {message} failed due to {ex.Message}."); } + finally + { + _throttler.Release(); + } + }); + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Stop(); + await base.StopAsync(cancellationToken); + } + + private async Task ProcessUserMessageAsync(CallerMessage message) + { + using var scope = _serviceProvider.CreateScope(); + var sp = scope.ServiceProvider; + + // Clean static HttpContext + var httpContext = sp.GetRequiredService(); + httpContext.HttpContext = new DefaultHttpContext(); + httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity()); + foreach (var header in message.RequestHeaders ?? []) + { + httpContext.HttpContext.Request.Headers[header.Key] = header.Value; + } + httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST"; + + AssistantMessage reply = null; + + var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); + var conv = sp.GetRequiredService(); + var routing = sp.GetRequiredService(); + var config = sp.GetRequiredService(); + var sessionManager = sp.GetRequiredService(); + var progressService = sp.GetRequiredService(); + InitProgressService(message, sessionManager, progressService); + InitConversation(message, inputMsg, conv, routing); + + // Need to consider Inbound and Outbound call + var conversation = await conv.GetConversation(message.ConversationId); + var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId; + + var result = await conv.SendMessage(agentId, + inputMsg, + replyMessage: BuildPostbackMessageModel(conv, message), + async msg => + { + reply = new AssistantMessage() + { + ConversationEnd = msg.Instruction?.ConversationEnd ?? false, + HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), + Content = msg.Content, + MessageId = msg.MessageId + }; + } + ); + reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp); + reply.Hints = GetHints(reply); + reply.Content = null; + await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); + } + + private PostbackMessageModel BuildPostbackMessageModel(IConversationService conv, CallerMessage message) + { + var messages = conv.GetDialogHistory(1); + if (!messages.Any()) return null; + var lastMessage = messages[0]; + if (string.IsNullOrEmpty(lastMessage.PostbackFunctionName)) return null; + return new PostbackMessageModel + { + FunctionName = lastMessage.PostbackFunctionName, + ParentId = lastMessage.MessageId, + Payload = message.Digits + }; + } + + private static void InitConversation(CallerMessage message, RoleDialogModel inputMsg, IConversationService conv, IRoutingService routing) + { + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); + var states = new List + { + new("channel", ConversationChannel.Phone), + new("channel_id", message.From), + new("calling_phone", message.From) + }; + states.AddRange(message.States.Select(kvp => new MessageState(kvp.Key, kvp.Value))); + conv.SetConversationId(message.ConversationId, states); + } + + private static async Task GetReplySpeechFileName(string conversationId, AssistantMessage reply, IServiceProvider sp) + { + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); + var fileName = $"reply_{reply.MessageId}.mp3"; + fileStorage.SaveSpeechFile(conversationId, fileName, data); + return fileName; + } + + private static string GetHints(AssistantMessage reply) + { + var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries); + int capcity = 100; + var hints = new List(capcity); + for (int i = phrases.Length - 1; i >= 0; i--) + { + var words = phrases[i].Split(' ', StringSplitOptions.RemoveEmptyEntries); + for (int j = words.Length - 1; j >= 0; j--) + { + hints.Add(words[j]); if (hints.Count >= capcity) { break; } } - // add frequency short words - hints.AddRange(["yes", "no", "correct", "right"]); - return string.Join(", ", hints.Select(x => x.ToLower()).Distinct().Reverse()); - } - - private static void InitProgressService(CallerMessage message, ITwilioSessionManager sessionManager, IConversationProgressService progressService) - { - progressService.OnFunctionExecuting = async msg => + if (hints.Count >= capcity) { - if (!string.IsNullOrEmpty(msg.Indication)) - { - await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); - } - }; - progressService.OnFunctionExecuted = async msg => { }; + break; + } } + // add frequency short words + hints.AddRange(["yes", "no", "correct", "right"]); + return string.Join(", ", hints.Select(x => x.ToLower()).Distinct().Reverse()); + } + + private static void InitProgressService(CallerMessage message, ITwilioSessionManager sessionManager, IConversationProgressService progressService) + { + progressService.OnFunctionExecuting = async msg => + { + if (!string.IsNullOrEmpty(msg.Indication)) + { + await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); + } + }; + progressService.OnFunctionExecuted = async msg => { }; } } From 9f7181617703c828d3c0584f53ef622ffc1f1c93 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 10 Mar 2025 12:14:44 -0500 Subject: [PATCH 52/84] refine state key search --- .../Agents/IAgentService.cs | 2 +- .../Conversations/Enums/MessageTypeName.cs | 1 + .../Conversations/IConversationService.cs | 9 +---- .../Loggers/Services/ILoggerService.cs | 3 +- .../Filters/ConversationFilter.cs | 4 +- .../Filters/ConversationStateKeysFilter.cs | 21 ++++++++++ .../Filters}/InstructLogFilter.cs | 4 +- .../Filters/InstructLogKeysFilter.cs | 21 ++++++++++ .../Repositories/IBotSharpRepository.cs | 6 ++- .../Users/IUserService.cs | 2 +- .../Agents/Services/AgentService.GetAgents.cs | 7 +++- .../Services/AgentService.RefreshAgents.cs | 2 +- .../Services/ConversationService.cs | 19 ++++++--- .../Services/LoggerService.Instruction.cs | 30 +++++++++++--- .../FileRepository.Conversation.cs | 20 ++++++---- .../FileRepository/FileRepository.Log.cs | 39 +++++++++++++++++- .../Users/Services/UserService.cs | 11 ++--- .../Controllers/ConversationController.cs | 40 ++++++++++--------- .../Controllers/LoggerController.cs | 13 +++++- .../Controllers/PluginController.cs | 3 +- .../Controllers/RoleController.cs | 4 +- .../Controllers/UserController.cs | 3 +- .../MongoRepository.Conversation.cs | 32 ++++++++++----- .../Repository/MongoRepository.Log.cs | 29 +++++++++++++- 24 files changed, 246 insertions(+), 79 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs rename src/Infrastructure/BotSharp.Abstraction/{Instructs/Models => Repositories/Filters}/InstructLogFilter.cs (81%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index cf49da42..7c2466d9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -12,7 +12,7 @@ public interface IAgentService Task CreateAgent(Agent agent); Task RefreshAgents(); Task> GetAgents(AgentFilter filter); - Task> GetAgentOptions(); + Task> GetAgentOptions(List? agentIds = null); /// /// Load agent configurations and trigger hooks diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs index c13f26a8..e59622bb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs @@ -6,4 +6,5 @@ public static class MessageTypeName public const string Notification = "notification"; public const string FunctionCall = "function"; public const string Audio = "audio"; + public const string Error = "error"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 6e558b1b..d9320257 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -62,14 +62,7 @@ public interface IConversationService void SaveStates(); - /// - /// Get conversation keys for searching - /// - /// search query - /// conversation limit - /// if pre-loading, then keys are not filter by the search query - /// - Task> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false); + Task> GetConversationStateSearhKeys(ConversationStateKeysFilter filter); Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs index 23a4809f..d704b154 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -1,5 +1,5 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories.Filters; namespace BotSharp.Abstraction.Loggers.Services; @@ -12,5 +12,6 @@ public interface ILoggerService #region Instruction Task> GetInstructionLogs(InstructLogFilter filter); + Task> GetInstructionLogSearchKeys(InstructLogKeysFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs index 23543049..d05c49c6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs @@ -23,9 +23,9 @@ public class ConversationFilter /// /// Check whether each key in the list is in the conversation states and its value equals to target value if not empty /// - public IEnumerable? States { get; set; } = []; + public List? States { get; set; } - public IEnumerable? Tags { get; set; } = []; + public List? Tags { get; set; } public static ConversationFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs new file mode 100644 index 00000000..bff4b43c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs @@ -0,0 +1,21 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class ConversationStateKeysFilter +{ + public string? Query { get; set; } + public int KeyLimit { get; set; } = 10; + public int ConvLimit { get; set; } = 100; + public bool PreLoad { get; set; } + public List? AgentIds { get; set; } + public List? UserIds { get; set; } + + public ConversationStateKeysFilter() + { + + } + + public static ConversationStateKeysFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs similarity index 81% rename from src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs rename to src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs index 43d655ce..c1040f94 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Abstraction.Instructs.Models; +namespace BotSharp.Abstraction.Repositories.Filters; public class InstructLogFilter : Pagination { @@ -11,6 +11,6 @@ public class InstructLogFilter : Pagination public static InstructLogFilter Empty() { - return new InstructLogFilter(); + return new(); } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs new file mode 100644 index 00000000..c3eade89 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs @@ -0,0 +1,21 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class InstructLogKeysFilter +{ + public string? Query { get; set; } + public int KeyLimit { get; set; } = 10; + public int LogLimit { get; set; } = 100; + public bool PreLoad { get; set; } + public List? AgentIds { get; set; } + public List? UserIds { get; set; } + + public InstructLogKeysFilter() + { + + } + + public static InstructLogKeysFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index d523dd09..5b92d322 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; @@ -149,7 +148,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); - List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) + List GetConversationStateSearchKeys(ConversationStateKeysFilter filter) => throw new NotImplementedException(); List GetConversationsToMigrate(int batchSize = 100) => throw new NotImplementedException(); @@ -182,6 +181,9 @@ public interface IBotSharpRepository : IHaveServiceProvider PagedItems GetInstructionLogs(InstructLogFilter filter) => throw new NotImplementedException(); + + List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + => throw new NotImplementedException(); #endregion #region Statistics diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 15b04f28..1499ea17 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -10,7 +10,7 @@ public interface IUserService Task> GetUsers(UserFilter filter); Task> SearchLoginUsers(User filter); Task GetUserDetails(string userId, bool includeAgent = false); - Task IsAdminUser(string userId); + Task<(bool, User?)> IsAdminUser(string userId); Task GetUserAuthorizations(IEnumerable? agentIds = null); Task UpdateUser(User user, bool isUpdateUserAgents = false); Task CreateUser(User user); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index ec95a51e..092273fa 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -30,9 +30,12 @@ public partial class AgentService #if !DEBUG [SharpCache(10, perInstanceCache: true)] #endif - public async Task> GetAgentOptions() + public async Task> GetAgentOptions(List? agentIds) { - var agents = _db.GetAgents(AgentFilter.Empty()); + var agents = _db.GetAgents(new AgentFilter + { + AgentIds = !agentIds.IsNullOrEmpty() ? agentIds : null + }); return agents?.Select(x => new IdName(x.Id, x.Name))?.OrderBy(x => x.Name)?.ToList() ?? []; } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index eb0497ae..c9b3c90c 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -18,7 +18,7 @@ public partial class AgentService var userIdentity = _services.GetRequiredService(); var userService = _services.GetRequiredService(); - var isValid = await userService.IsAdminUser(userIdentity.Id); + var (isValid, _) = await userService.IsAdminUser(userIdentity.Id); if (!isValid) { return "Unauthorized user."; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 0cec37d8..a1a0868a 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -222,17 +222,26 @@ public partial class ConversationService : IConversationService _state.Save(); } - public async Task> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false) + public async Task> GetConversationStateSearhKeys(ConversationStateKeysFilter filter) { + if (filter == null) + { + filter = ConversationStateKeysFilter.Empty(); + } + var keys = new List(); - if (!preload && string.IsNullOrWhiteSpace(query)) + if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query)) { return keys; } + var userService = _services.GetRequiredService(); var db = _services.GetRequiredService(); - keys = db.GetConversationStateSearchKeys(convUpperLimit: convLimit); - keys = preload ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); - return keys.OrderBy(x => x).Take(keyLimit).ToList(); + + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); + filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : []; + keys = db.GetConversationStateSearchKeys(filter); + keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList(); } } diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs index 84efae99..42ad45ab 100644 --- a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Users.Enums; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Core.Loggers.Services; @@ -15,11 +13,10 @@ public partial class LoggerService } var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); - var isAdmin = UserConstant.AdminRoles.Contains(user?.Role); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); if (!isAdmin && user?.Id == null) return new(); - filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : []; + filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null; var agents = new List(); var users = new List(); @@ -61,4 +58,27 @@ public partial class LoggerService Count = logs.Count }; } + + public async Task> GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + { + if (filter == null) + { + filter = InstructLogKeysFilter.Empty(); + } + + var keys = new List(); + if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query)) + { + return keys; + } + + var userService = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); + filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null; + keys = db.GetInstructionLogSearchKeys(filter); + keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 8880a4e0..3d70264c 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users.Models; using System.IO; namespace BotSharp.Core.Repository; @@ -647,7 +648,7 @@ public partial class FileRepository #if !DEBUG [SharpCache(10)] #endif - public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) + public List GetConversationStateSearchKeys(ConversationStateKeysFilter filter) { var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); if (!Directory.Exists(dir)) return []; @@ -658,26 +659,29 @@ public partial class FileRepository foreach (var d in Directory.GetDirectories(dir)) { var convFile = Path.Combine(d, CONVERSATION_FILE); - var stateFile = Path.Combine(d, STATE_FILE); - if (!File.Exists(convFile) || !File.Exists(stateFile)) + var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE); + if (!File.Exists(convFile) || !File.Exists(latestStateFile)) { continue; } var convJson = File.ReadAllText(convFile); - var stateJson = File.ReadAllText(stateFile); + var stateJson = File.ReadAllText(latestStateFile); var conv = JsonSerializer.Deserialize(convJson, _options); - var states = JsonSerializer.Deserialize>(stateJson, _options); - if (conv == null || conv.DialogCount < messageLowerLimit) + var states = JsonSerializer.Deserialize>(stateJson, _options); + if (conv == null + || states.IsNullOrEmpty() + || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(conv.AgentId)) + || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId))) { continue; } - var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; + var stateKeys = states?.Select(x => x.Key)?.ToList() ?? []; keys.AddRange(stateKeys); count++; - if (count >= convUpperLimit) + if (count >= filter.ConvLimit) { break; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 0b9b27a7..78027987 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using System.IO; @@ -271,6 +270,44 @@ namespace BotSharp.Core.Repository Count = logs.Count() }; } + + public List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + { + var keys = new List(); + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + return keys; + } + + var count = 0; + var files = Directory.GetFiles(baseDir); + foreach (var file in files) + { + var json = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(json, _options); + if (log == null) continue; + + if (log == null + || log.InnerStates.IsNullOrEmpty() + || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(log.UserId)) + || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId))) + { + continue; + } + + var stateKeys = log.InnerStates.Select(x => x.Key)?.ToList() ?? []; + keys.AddRange(stateKeys); + count++; + + if (count >= filter.LogLimit) + { + break; + } + } + + return keys.Distinct().ToList(); + } #endregion #region Private methods diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 60192e5f..e4ebe311 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -418,23 +418,24 @@ public class UserService : IUserService return users; } - public async Task IsAdminUser(string userId) + [SharpCache(10, perInstanceCache: true)] + public async Task<(bool, User?)> IsAdminUser(string userId) { var db = _services.GetRequiredService(); var user = db.GetUserById(userId); - return user != null && UserConstant.AdminRoles.Contains(user.Role); + var isAdmin = user != null && UserConstant.AdminRoles.Contains(user.Role); + return (isAdmin, user); } public async Task GetUserAuthorizations(IEnumerable? agentIds = null) { var db = _services.GetRequiredService(); - var user = db.GetUserById(_user.Id); + var (isAdmin, user) = await IsAdminUser(_user.Id); var auth = new UserAuthorization(); if (user == null) return auth; - auth.IsAdmin = UserConstant.AdminRoles.Contains(user.Role); - + auth.IsAdmin = isAdmin; var role = db.GetRoles(new RoleFilter { Names = [user.Role] }).FirstOrDefault(); var permissions = user.Permissions?.Any() == true ? user.Permissions : role?.Permissions ?? []; auth.Permissions = permissions; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 10317815..d8b71339 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,10 +1,8 @@ -using Azure; using BotSharp.Abstraction.Files.Constants; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Users.Enums; using BotSharp.Core.Infrastructures; namespace BotSharp.OpenAPI.Controllers; @@ -45,27 +43,33 @@ public class ConversationController : ControllerBase return ConversationViewModel.FromSession(conv); } - [HttpPost("/conversations")] - public async Task> GetConversations([FromBody] ConversationFilter filter) + [HttpGet("/conversations")] + public async Task> GetConversations([FromQuery] ConversationFilter filter) { var convService = _services.GetRequiredService(); var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); if (user == null) { return new PagedItems(); } - filter.UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : filter.UserId; + filter.UserId = !isAdmin ? user.Id : filter.UserId; var conversations = await convService.GetConversations(filter); var agentService = _services.GetRequiredService(); var list = conversations.Items.Select(x => ConversationViewModel.FromSession(x)).ToList(); + var agentIds = list.Select(x => x.AgentId).ToList(); + var agents = await agentService.GetAgentOptions(agentIds); + + var userIds = list.Select(x => x.User.Id).ToList(); + var users = await userService.GetUsers(new UserFilter { UserIds = userIds, Size = filter.Pager.Size }); + foreach (var item in list) { - user = await userService.GetUser(item.User.Id); + user = users.Items.FirstOrDefault(x => x.Id == item.User.Id); item.User = UserViewModel.FromUser(user); - var agent = await agentService.GetAgent(item.AgentId); + var agent = agents.FirstOrDefault(x => x.Id == item.AgentId); item.AgentName = agent?.Name ?? "Unkown"; } @@ -138,7 +142,7 @@ public class ConversationController : ControllerBase { var service = _services.GetRequiredService(); var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); if (user == null) { return null; @@ -147,7 +151,7 @@ public class ConversationController : ControllerBase var filter = new ConversationFilter { Id = conversationId, - UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : null + UserId = !isAdmin ? user.Id : null }; var conversations = await service.GetConversations(filter); if (conversations.Items.IsNullOrEmpty()) @@ -206,11 +210,11 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); var filter = new ConversationFilter { Id = conversationId, - UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : null + UserId = !isAdmin ? user?.Id : null }; var conversations = await conv.GetConversations(filter); @@ -229,11 +233,11 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var conversationService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); var filter = new ConversationFilter { Id = conversationId, - UserId = user.Role != UserRole.Admin ? user.Id : null + UserId = !isAdmin ? user?.Id : null }; var conversations = await conversationService.GetConversations(filter); @@ -283,11 +287,11 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var conversationService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); var filter = new ConversationFilter { Id = conversationId, - UserId = !UserConstant.AdminRoles.Contains(user?.Role) ? user.Id : null + UserId = !isAdmin ? user?.Id : null }; var conversations = await conversationService.GetConversations(filter); @@ -555,10 +559,10 @@ public class ConversationController : ControllerBase #region Search state keys [HttpGet("/conversation/state/keys")] - public async Task> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] int convLimit = 100, [FromQuery] bool preload = false) + public async Task> GetConversationStateKeys([FromQuery] ConversationStateKeysFilter request) { var convService = _services.GetRequiredService(); - var keys = await convService.GetConversationStateSearhKeys(query, keyLimit: keyLimit, convLimit: convLimit, preload: preload); + var keys = await convService.GetConversationStateSearhKeys(request); return keys; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index d67ba8cd..892c3195 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Loggers.Services; using BotSharp.OpenAPI.ViewModels.Instructs; @@ -39,6 +38,7 @@ public class LoggerController : ControllerBase } } + #region Conversation log [HttpGet("/logger/conversation/{conversationId}/content-log")] public async Task> GetConversationContentLogs([FromRoute] string conversationId) { @@ -52,7 +52,9 @@ public class LoggerController : ControllerBase var logging = _services.GetRequiredService(); return await logging.GetConversationStateLogs(conversationId); } + #endregion + #region Instruction log [HttpGet("/logger/instruction/log")] public async Task> GetInstructionLogs([FromQuery] InstructLogFilter request) { @@ -65,4 +67,13 @@ public class LoggerController : ControllerBase Count = logs.Count }; } + + [HttpGet("/logger/instruction/log/keys")] + public async Task> GetInstructionLogKeys([FromQuery] InstructLogKeysFilter request) + { + var logging = _services.GetRequiredService(); + var keys = await logging.GetInstructionLogSearchKeys(request); + return keys; + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs index 55a5f1a8..4f6d3f2b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs @@ -89,6 +89,7 @@ public class PluginController(IServiceProvider services, IUserIdentity user, Plu private async Task IsValidUser() { var userService = services.GetRequiredService(); - return await userService.IsAdminUser(_user.Id); + var (isAdmin, _) = await userService.IsAdminUser(_user.Id); + return isAdmin; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs index d4ca64fc..ee3ac863 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Roles; -using BotSharp.Abstraction.Users.Enums; namespace BotSharp.OpenAPI.Controllers; @@ -83,6 +82,7 @@ public class RoleController : ControllerBase private async Task IsValidUser() { var userService = _services.GetRequiredService(); - return await userService.IsAdminUser(_user.Id); + var (isAdmin, _) = await userService.IsAdminUser(_user.Id); + return isAdmin; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a853e1eb..7cb1aa2b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -254,7 +254,8 @@ public class UserController : ControllerBase private async Task IsValidUser() { var userService = _services.GetRequiredService(); - return await userService.IsAdminUser(_user.Id); + var (isAdmin, _) = await userService.IsAdminUser(_user.Id); + return isAdmin; } private FileContentResult BuildFileResult(string file) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index f39ad2dc..908e66b5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -644,21 +644,31 @@ public partial class MongoRepository #if !DEBUG [SharpCache(10)] #endif - public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) + public List GetConversationStateSearchKeys(ConversationStateKeysFilter filter) { - var stateBuilder = Builders.Filter; - var sortDef = Builders.Sort.Descending(x => x.UpdatedTime); - var stateFilters = new List>() + var builder = Builders.Filter; + var sortDef = Builders.Sort.Descending(x => x.UpdatedTime); + var filters = new List>() { - stateBuilder.Exists(x => x.States), - stateBuilder.Ne(x => x.States, []) + builder.Exists(x => x.LatestStates), + builder.Ne(x => x.LatestStates, []) }; - var states = _dc.ConversationStates.Find(stateBuilder.And(stateFilters)) - .Sort(sortDef) - .Limit(convUpperLimit) - .ToList(); - var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + if (!filter.AgentIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + } + + if (!filter.UserIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.UserId, filter.UserIds)); + } + + var convDocs = _dc.Conversations.Find(builder.And(filters)) + .Sort(sortDef) + .Limit(filter.ConvLimit) + .ToList(); + var keys = convDocs.SelectMany(x => x.LatestStates.Select(x => x.Key)).Distinct().ToList(); return keys; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index d94b1d6a..457faba2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; using System.Text.Json; @@ -241,5 +240,33 @@ public partial class MongoRepository Count = (int)count }; } + + public List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) + { + var builder = Builders.Filter; + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var filters = new List>() + { + builder.Exists(x => x.States), + builder.Ne(x => x.States, []) + }; + + if (!filter.AgentIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + } + + if (!filter.UserIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.UserId, filter.UserIds)); + } + + var convDocs = _dc.InstructionLogs.Find(builder.And(filters)) + .Sort(sortDef) + .Limit(filter.LogLimit) + .ToList(); + var keys = convDocs.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + return keys; + } #endregion } From 7571512e0d67ee9e8680a92af83b0e0192576998 Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Mon, 10 Mar 2025 15:52:43 -0500 Subject: [PATCH 53/84] Update headless setting for Playwright --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 6 +++++- .../BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 4c65aa32..9f830f2e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -33,9 +33,13 @@ public class PlaywrightInstance : IDisposable public async Task GetContext(string ctxId) { + var headless = true; +#if DEBUG + headless = false; +#endif if (!_contexts.ContainsKey(ctxId)) { - await InitContext(ctxId, new BrowserActionArgs()); + await InitContext(ctxId, new BrowserActionArgs() { Headless = headless }); } return _contexts[ctxId]; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 6a5a1ac1..ed1c7398 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -29,9 +29,13 @@ public class OpenBrowserFn : IFunctionCallback ContextId = convService.ConversationId, MessageId = message.MessageId }; + var headless = true; +#if DEBUG + headless = false; +#endif var result = await _browser.LaunchBrowser(msgInfo, new BrowserActionArgs { - Headless = false + Headless = headless }); result = await _browser.GoToPage(msgInfo, new PageActionArgs { From b8083b2f3893bf86ae01e588a2b6e6edd8a822f3 Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Mon, 10 Mar 2025 16:54:51 -0500 Subject: [PATCH 54/84] Fetch WebDrive headless setting from config --- .../Browsing/Settings/WebBrowsingSettings.cs | 1 + .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 8 +++----- .../Functions/OpenBrowserFn.cs | 13 +++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 25149fe0..e1ae9a79 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -3,4 +3,5 @@ namespace BotSharp.Abstraction.Browsing.Settings; public class WebBrowsingSettings { public string Driver { get; set; } = "Playwright"; + public bool Headless { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 9f830f2e..755d4faf 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,4 +1,5 @@ using Azure; +using BotSharp.Abstraction.Browsing.Settings; using System.IO; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -33,13 +34,10 @@ public class PlaywrightInstance : IDisposable public async Task GetContext(string ctxId) { - var headless = true; -#if DEBUG - headless = false; -#endif + var _webDriver = _services.GetRequiredService(); if (!_contexts.ContainsKey(ctxId)) { - await InitContext(ctxId, new BrowserActionArgs() { Headless = headless }); + await InitContext(ctxId, new BrowserActionArgs() { Headless = _webDriver.Headless }); } return _contexts[ctxId]; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index ed1c7398..92935233 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Functions; public class OpenBrowserFn : IFunctionCallback @@ -6,12 +8,15 @@ public class OpenBrowserFn : IFunctionCallback private readonly IServiceProvider _services; private readonly IWebBrowser _browser; + private readonly WebBrowsingSettings _webBrowsingSettings; public OpenBrowserFn(IServiceProvider services, - IWebBrowser browser) + IWebBrowser browser, + WebBrowsingSettings webBrowsingSettings) { _services = services; _browser = browser; + _webBrowsingSettings = webBrowsingSettings; } public async Task Execute(RoleDialogModel message) @@ -29,13 +34,9 @@ public class OpenBrowserFn : IFunctionCallback ContextId = convService.ConversationId, MessageId = message.MessageId }; - var headless = true; -#if DEBUG - headless = false; -#endif var result = await _browser.LaunchBrowser(msgInfo, new BrowserActionArgs { - Headless = headless + Headless = _webBrowsingSettings.Headless }); result = await _browser.GoToPage(msgInfo, new PageActionArgs { From 25a9c77cddcd672ebca9640285d187544b75ac43 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 10 Mar 2025 17:03:23 -0500 Subject: [PATCH 55/84] refine agent task --- .../Tasks/Models/AgentTask.cs | 17 +++++++++++++---- .../Services/CrontabService.cs | 2 +- .../Agents/Services/AgentService.CreateAgent.cs | 4 ++-- .../FileRepository/FileRepository.AgentTask.cs | 8 ++++---- .../Tasks/Services/AgentTaskService.cs | 2 +- .../ViewModels/Agents/AgentTaskViewModel.cs | 12 ++++++------ .../Collections/AgentTaskDocument.cs | 8 ++++---- 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs index e4852307..f5e379c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs @@ -2,10 +2,13 @@ namespace BotSharp.Abstraction.Tasks.Models; public class AgentTask { + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string Id { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; public string? Description { get; set; } public bool Enabled { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string Content { get; set; } = string.Empty; [JsonIgnore(Condition = JsonIgnoreCondition.Always)] @@ -19,9 +22,15 @@ public class AgentTask /// public string Status { get; set; } = TaskStatus.New; - public DateTime? LastExecutedDateTime { get; set; } - public DateTime? NextExecutionDateTime { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTime? LastExecutionTime { get; set; } - public DateTime CreatedDateTime { get; set; } - public DateTime UpdatedDateTime { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTime? NextExecutionTime { get; set; } + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } + + [JsonPropertyName("updated_time")] + public DateTime UpdatedTime { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 5cbf9028..cf4e1846 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -97,7 +97,7 @@ public class CrontabService : ICrontabService, ITaskFeeder Status = TaskStatus.Scheduled, Enabled = !x.Disabled, Description = cron.Description, - LastExecutedDateTime = cron.LastExecutionTime + LastExecutionTime = cron.LastExecutionTime })); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index d4ff408b..5becbec6 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -185,8 +185,8 @@ public partial class AgentService Enabled = parsedTask.Enabled, Content = parsedTask.Content, AgentId = agentId, - CreatedDateTime = parsedTask.CreatedDateTime, - UpdatedDateTime = parsedTask.UpdatedDateTime + CreatedTime = parsedTask.CreatedTime, + UpdatedTime = parsedTask.UpdatedTime }; tasks.Add(task); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs index 6d47c4cc..016e0c36 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -129,8 +129,8 @@ public partial class FileRepository Name = task.Name, Description = task.Description, Enabled = task.Enabled, - CreatedDateTime = DateTime.UtcNow, - UpdatedDateTime = DateTime.UtcNow + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow }; var fileContent = BuildAgentTaskFileContent(metaData, task.Content); @@ -163,8 +163,8 @@ public partial class FileRepository Name = parsedTask.Name, Description = parsedTask.Description, Enabled = parsedTask.Enabled, - CreatedDateTime = parsedTask.CreatedDateTime, - UpdatedDateTime = DateTime.UtcNow + CreatedTime = parsedTask.CreatedTime, + UpdatedTime = DateTime.UtcNow }; var content = parsedTask.Content; diff --git a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs index 5ad0637e..ce5d72b8 100644 --- a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs +++ b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs @@ -31,7 +31,7 @@ public class AgentTaskService : IAgentTaskService return new PagedItems { - Items = items.OrderByDescending(x => x.UpdatedDateTime) + Items = items.OrderByDescending(x => x.UpdatedTime) .Skip(filter.Pager.Offset).Take(filter.Pager.Size), Count = items.Count() }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs index cf051544..d46d134e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs @@ -13,11 +13,11 @@ public class AgentTaskViewModel public string Status { get; set; } = null!; - [JsonPropertyName("created_datetime")] - public DateTime CreatedDateTime { get; set; } + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } - [JsonPropertyName("updated_datetime")] - public DateTime UpdatedDateTime { get; set; } + [JsonPropertyName("updated_time")] + public DateTime UpdatedTime { get; set; } [JsonPropertyName("agent_id")] public string AgentId { get; set; } = null!; @@ -37,8 +37,8 @@ public class AgentTaskViewModel AgentId = task.AgentId, AgentName = task.Agent?.Name, Status = task.Status, - CreatedDateTime = task.CreatedDateTime, - UpdatedDateTime = task.UpdatedDateTime + CreatedTime = task.CreatedTime, + UpdatedTime = task.UpdatedTime }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs index 7f09318a..8cade792 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs @@ -23,8 +23,8 @@ public class AgentTaskDocument : MongoBase Enabled = model.Enabled, AgentId = model.AgentId, Status = model.Status, - CreatedDateTime = model.CreatedTime, - UpdatedDateTime = model.UpdatedTime + CreatedTime = model.CreatedTime, + UpdatedTime = model.UpdatedTime }; } @@ -38,8 +38,8 @@ public class AgentTaskDocument : MongoBase Enabled = model.Enabled, AgentId = model.AgentId, Status = model.Status, - CreatedTime = model.CreatedDateTime, - UpdatedTime = model.UpdatedDateTime + CreatedTime = model.CreatedTime, + UpdatedTime = model.UpdatedTime }; } } From 0d8e9903cb419e8657f74c23be12d97baf0015df Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 10 Mar 2025 21:56:18 -0500 Subject: [PATCH 56/84] Increase ReceiveMessage buffer to 32K. --- .../Providers/Realtime/RealTimeCompletionProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 1d44bc8d..0d6f76b4 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -131,7 +131,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var buffer = new byte[1024 * 16]; + var buffer = new byte[1024 * 32]; WebSocketReceiveResult result; do From db3ed518751d8e3758057853f4e8eaad43ec0b30 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 11 Mar 2025 10:18:54 -0500 Subject: [PATCH 57/84] get template name from state --- .../BotSharp.Logger/Hooks/InstructionLogHook.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs index 0abea5eb..30e1d066 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -32,6 +32,8 @@ public class InstructionLogHook : InstructHookBase var state = _services.GetRequiredService(); var user = db.GetUserById(_user.Id); + var templateName = response.TemplateName ?? state.GetState("instruct_template_name") ?? null; + db.SaveInstructionLogs(new List { new InstructionLogModel @@ -39,7 +41,7 @@ public class InstructionLogHook : InstructHookBase AgentId = response.AgentId, Provider = response.Provider, Model = response.Model, - TemplateName = response.TemplateName, + TemplateName = !string.IsNullOrWhiteSpace(templateName) ? templateName : null, UserMessage = response.UserMessage, SystemInstruction = response.SystemInstruction, CompletionText = response.CompletionText, From 23baf7008868c9118a50be4e95f8256fafc1a0ed Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 11 Mar 2025 16:48:44 -0400 Subject: [PATCH 58/84] Update to M.E.AI 9.3.0-preview.1.25161.3 --- Directory.Packages.props | 2 +- .../MicrosoftExtensionsAIChatCompletionProvider.cs | 11 +++++------ .../MicrosoftExtensionsAITextCompletionProvider.cs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 4683fc9d..aa76cf61 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -54,7 +54,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 37d3877c..6ee77772 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -27,7 +27,6 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio private readonly IChatClient _client; private readonly ILogger _logger; private readonly IServiceProvider _services; - private List renderedInstructions = []; private string? _model; /// @@ -46,7 +45,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio /// public string Provider => "microsoft.extensions.ai"; - public string Model => _model; + public string Model => _model ?? ""; /// public void SetModelName(string model) => _model = model; @@ -56,7 +55,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio { // Before chat completion hook var hooks = _services.GetServices().ToArray(); - renderedInstructions = []; + List renderedInstructions = []; await Task.WhenAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations))); // Configure options @@ -145,13 +144,13 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio var completion = await _client.GetResponseAsync(messages); - RoleDialogModel result = new(AgentRole.Assistant, string.Concat(completion.Message.Contents.OfType())) + RoleDialogModel result = new(AgentRole.Assistant, completion.Text) { CurrentAgentId = agent.Id, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + //RenderedInstruction = renderedInstructions, }; - if (completion.Message.Contents.OfType().FirstOrDefault() is { } fcc) + if (completion.Messages.SelectMany(m => m.Contents).OfType().FirstOrDefault() is { } fcc) { result.Role = AgentRole.Function; result.MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs index ed0e94d9..ef3b35c7 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs @@ -51,7 +51,7 @@ public sealed class MicrosoftExtensionsAITextCompletionProvider : ITextCompletio _tokenStatistics.StartTimer(); var completion = await _chatClient.GetResponseAsync(text); - var result = string.Concat(completion.Message.Contents.OfType()); + var result = completion.Text; _tokenStatistics.StopTimer(); // After chat completion hook From a537a647cdf0d4b15f94d3b7b27b4b9f6d1bcaaa Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 11 Mar 2025 17:20:44 -0500 Subject: [PATCH 59/84] politely ending phone call --- .../Functions/HangupPhoneCallFn.cs | 24 +++++++++++++------ .../LlmContexts/HangupPhoneCallArgs.cs | 9 +++++++ .../LlmContexts/LlmContextIn.cs | 4 ++-- .../util-twilio-hangup_phone_call.json | 6 ++++- 4 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index f54c1b42..bc1bad36 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,7 @@ +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Microsoft.VisualBasic; using Twilio.Rest.Api.V2010.Account; +using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -20,6 +23,7 @@ public class HangupPhoneCallFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var args = JsonSerializer.Deserialize(message.FunctionArgs); var states = _services.GetRequiredService(); var callSid = states.GetState("twilio_call_sid"); @@ -30,14 +34,20 @@ public class HangupPhoneCallFn : IFunctionCallback return false; } - // Have to find the SID by the phone number - var call = CallResource.Update( - status: CallResource.UpdateStatusEnum.Completed, - pathSid: callSid - ); + message.Content = args.GoodbyeMessage; - message.Content = "The call has ended."; - message.StopCompletion = true; + _ = Task.Run(async () => + { + await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400); + // Have to find the SID by the phone number + var call = CallResource.Update( + status: CallResource.UpdateStatusEnum.Completed, + pathSid: callSid + ); + + message.Content = "The call has ended."; + message.StopCompletion = true; + }); return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs new file mode 100644 index 00000000..ea2075d9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class HangupPhoneCallArgs +{ + [JsonPropertyName("goodbye_message")] + public string? GoodbyeMessage { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index cde85aa9..3f7ea8b6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -5,8 +5,8 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class LlmContextIn { [JsonPropertyName("phone_number")] - public string PhoneNumber { get; set; } + public string PhoneNumber { get; set; } = null!; [JsonPropertyName("initial_message")] - public string InitialMessage { get; set; } + public string? InitialMessage { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 76b0e841..773fac23 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -5,7 +5,11 @@ "parameters": { "type": "object", "properties": { + "goodbye_message": { + "type": "string", + "description": "A polite closing statement for ending a conversation." + } }, - "required": [] + "required": [ "goodbye_message" ] } } \ No newline at end of file From 9fe9b123f13039b9a15e6f96fc0e8c061bcba20d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 11 Mar 2025 20:57:49 -0500 Subject: [PATCH 60/84] HandleUserDtmfReceived --- .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 68636bd1..e8548c4d 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -233,7 +233,8 @@ public class RealtimeHub : IRealtimeHub } await _completer.InsertConversationItem(message); - await _completer.TriggerModelInference("Reply based on the user input"); + var instruction = await _completer.UpdateSession(_conn); + await _completer.TriggerModelInference($"{instruction}\r\n\r\nReply based on the user input: {message.Content}"); } private async Task HandleUserDisconnected() From 7d4d233f7e5c6248802b1891893dea8e0f419f0c Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Tue, 11 Mar 2025 22:01:16 -0500 Subject: [PATCH 61/84] Update timeout for web driver logic to fetch from appSettings --- .../Browsing/Settings/WebBrowsingSettings.cs | 1 + .../BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs | 7 +++++-- .../BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs | 5 +++-- .../UtilFunctions/UtilWebGoToPageFn.cs | 4 ++++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index e1ae9a79..4aebc1b3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -4,4 +4,5 @@ public class WebBrowsingSettings { public string Driver { get; set; } = "Playwright"; public bool Headless { get; set; } + public float? DefaultTimeOut { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index e1041ae6..9e93b2a7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Functions; public class GoToPageFn : IFunctionCallback @@ -25,7 +27,7 @@ public class GoToPageFn : IFunctionCallback var url = webDriverService.ReplaceToken(args.Url); url = url.Replace("https://https://", "https://"); - + var _webDriver = _services.GetRequiredService(); var result = await _browser.GoToPage(new MessageInfo { AgentId = message.CurrentAgentId, @@ -33,7 +35,8 @@ public class GoToPageFn : IFunctionCallback MessageId = message.MessageId }, new PageActionArgs { - Url = url + Url = url, + Timeout = _webDriver.DefaultTimeOut }); message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.Message}"; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 92935233..beb412b5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -26,7 +26,7 @@ public class OpenBrowserFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var url = webDriverService.ReplaceToken(args.Url); - + var _webDriver = _services.GetRequiredService(); url = url.Replace("https://https://", "https://"); var msgInfo = new MessageInfo { @@ -40,7 +40,8 @@ public class OpenBrowserFn : IFunctionCallback }); result = await _browser.GoToPage(msgInfo, new PageActionArgs { - Url = url + Url = url, + Timeout = _webDriver.DefaultTimeOut }); if (result.IsSuccess) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index 675e1ec8..f62a6ae9 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.UtilFunctions; public class UtilWebGoToPageFn : IFunctionCallback @@ -21,6 +23,8 @@ public class UtilWebGoToPageFn : IFunctionCallback { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var _webDriver = _services.GetRequiredService(); + args.Timeout = _webDriver.DefaultTimeOut; args.WaitForNetworkIdle = false; args.WaitTime = 5; args.OpenNewTab = true; From e570672a365d4fee9273fa3be1267067b0d4c76c Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Wed, 12 Mar 2025 08:49:31 -0500 Subject: [PATCH 62/84] Update default timeout to 30sec --- .../Browsing/Settings/WebBrowsingSettings.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 4aebc1b3..3245a802 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -4,5 +4,6 @@ public class WebBrowsingSettings { public string Driver { get; set; } = "Playwright"; public bool Headless { get; set; } - public float? DefaultTimeOut { get; set; } + // Default timeout in milliseconds + public float DefaultTimeOut { get; set; } = 30000; } From 47d728b68a9c1ae3de179fa54cef64b91d3c20b3 Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Wed, 12 Mar 2025 08:53:38 -0500 Subject: [PATCH 63/84] Update default timeout to lowercasing --- .../Browsing/Settings/WebBrowsingSettings.cs | 2 +- src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs | 2 +- .../BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs | 2 +- .../UtilFunctions/UtilWebGoToPageFn.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 3245a802..9565e217 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -5,5 +5,5 @@ public class WebBrowsingSettings public string Driver { get; set; } = "Playwright"; public bool Headless { get; set; } // Default timeout in milliseconds - public float DefaultTimeOut { get; set; } = 30000; + public float DefaultTimeout { get; set; } = 30000; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 9e93b2a7..fe69f9d1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -36,7 +36,7 @@ public class GoToPageFn : IFunctionCallback }, new PageActionArgs { Url = url, - Timeout = _webDriver.DefaultTimeOut + Timeout = _webDriver.DefaultTimeout }); message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.Message}"; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index beb412b5..0beb80fc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -41,7 +41,7 @@ public class OpenBrowserFn : IFunctionCallback result = await _browser.GoToPage(msgInfo, new PageActionArgs { Url = url, - Timeout = _webDriver.DefaultTimeOut + Timeout = _webDriver.DefaultTimeout }); if (result.IsSuccess) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index f62a6ae9..a29af212 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -24,7 +24,7 @@ public class UtilWebGoToPageFn : IFunctionCallback PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); var _webDriver = _services.GetRequiredService(); - args.Timeout = _webDriver.DefaultTimeOut; + args.Timeout = _webDriver.DefaultTimeout; args.WaitForNetworkIdle = false; args.WaitTime = 5; args.OpenNewTab = true; From 025b0a1d5cf3d187a71b6e5679dafb0c20d38e0c Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Wed, 12 Mar 2025 22:57:12 +0800 Subject: [PATCH 64/84] optimize input audio transcription --- .../Models/Realtime/RealtimeSessionBody.cs | 6 ++++++ .../Providers/Realtime/RealTimeCompletionProvider.cs | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index 0d0947e5..ff2bfc3f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -72,4 +72,10 @@ public class InputAudioTranscription { [JsonPropertyName("model")] public string Model { get; set; } = null!; + + [JsonPropertyName("language")] + public string Language { get; set; } = "en"; + + [JsonPropertyName("prompt")] + public string Prompt { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 0d6f76b4..af6f0ad3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -138,7 +138,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { result = await _webSocket.ReceiveAsync( new ArraySegment(buffer), CancellationToken.None); - + // Convert received data to text/audio (Twilio sends Base64-encoded audio) string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); if (string.IsNullOrEmpty(receivedText)) @@ -319,6 +319,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion InputAudioTranscription = new InputAudioTranscription { Model = "whisper-1", + Language = "en", }, Voice = "alloy", Instructions = instruction, @@ -329,7 +330,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MaxResponseOutputTokens = 512, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.8f, + Threshold = 0.9f, PrefixPadding = 300, SilenceDuration = 800 } From 85cf382adcde5dee6f289d19ea98822e0524d661 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 12 Mar 2025 10:13:57 -0500 Subject: [PATCH 65/84] refine chat log --- .../Loggers/Services/ILoggerService.cs | 4 +- .../Filters/ConversationLogFilter.cs | 17 ++++ .../Repositories/IBotSharpRepository.cs | 4 +- .../Utilities/DateTimePagination.cs | 6 ++ .../Services/LoggerService.Conversation.cs | 18 +++- .../FileRepository/FileRepository.Log.cs | 45 ++++++--- .../Controllers/ConversationController.cs | 4 +- .../Controllers/LoggerController.cs | 17 ++-- .../Hooks/ChatHubCrontabHook.cs | 18 +--- .../Repository/MongoRepository.Log.cs | 95 ++++++++++++------- 10 files changed, 145 insertions(+), 83 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs index d704b154..564fcf37 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -6,8 +6,8 @@ namespace BotSharp.Abstraction.Loggers.Services; public interface ILoggerService { #region Conversation - Task> GetConversationContentLogs(string conversationId); - Task> GetConversationStateLogs(string conversationId); + Task> GetConversationContentLogs(string conversationId, ConversationLogFilter filter); + Task> GetConversationStateLogs(string conversationId, ConversationLogFilter filter); #endregion #region Instruction diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs new file mode 100644 index 00000000..59cf9d27 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class ConversationLogFilter +{ + public int Size { get; set; } = 20; + public DateTime StartTime { get; set; } = DateTime.UtcNow; + + public ConversationLogFilter() + { + + } + + public static ConversationLogFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 5b92d322..9bccb82e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -164,14 +164,14 @@ public interface IBotSharpRepository : IHaveServiceProvider #region Conversation Content Log void SaveConversationContentLog(ContentLogOutputModel log) => throw new NotImplementedException(); - List GetConversationContentLogs(string conversationId) + DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) => throw new NotImplementedException(); #endregion #region Conversation State Log void SaveConversationStateLog(ConversationStateLogModel log) => throw new NotImplementedException(); - List GetConversationStateLogs(string conversationId) + DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs new file mode 100644 index 00000000..fc56e16e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Utilities; + +public class DateTimePagination : PagedItems +{ + public DateTime? NextTime { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs index 5ce057c3..c424d237 100644 --- a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs @@ -4,18 +4,28 @@ namespace BotSharp.Core.Loggers.Services; public partial class LoggerService { - public async Task> GetConversationContentLogs(string conversationId) + public async Task> GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { + if (filter == null) + { + filter = ConversationLogFilter.Empty(); + } + var db = _services.GetRequiredService(); - var logs = db.GetConversationContentLogs(conversationId); + var logs = db.GetConversationContentLogs(conversationId, filter); return await Task.FromResult(logs); } - public async Task> GetConversationStateLogs(string conversationId) + public async Task> GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { + if (filter == null) + { + filter = ConversationLogFilter.Empty(); + } + var db = _services.GetRequiredService(); - var logs = db.GetConversationStateLogs(conversationId); + var logs = db.GetConversationStateLogs(conversationId, filter); return await Task.FromResult(logs); } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 78027987..3913a404 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Loggers.Models; +using Microsoft.IdentityModel.Logging; using System.IO; namespace BotSharp.Core.Repository @@ -54,26 +55,34 @@ namespace BotSharp.Core.Repository File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } - public List GetConversationContentLogs(string conversationId) + public DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; + if (string.IsNullOrEmpty(conversationId)) return new(); var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return logs; + if (string.IsNullOrEmpty(convDir)) return new(); var logDir = Path.Combine(convDir, "content_log"); - if (!Directory.Exists(logDir)) return logs; + if (!Directory.Exists(logDir)) return new(); + var logs = new List(); foreach (var file in Directory.GetFiles(logDir)) { var text = File.ReadAllText(file); var log = JsonSerializer.Deserialize(text); - if (log == null) continue; + if (log == null || log.CreatedTime >= filter.StartTime) continue; logs.Add(log); } - return logs.OrderBy(x => x.CreatedTime).ToList(); + + logs = logs.OrderByDescending(x => x.CreatedTime).Take(filter.Size).ToList(); + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion @@ -99,26 +108,34 @@ namespace BotSharp.Core.Repository File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } - public List GetConversationStateLogs(string conversationId) + public DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; + if (string.IsNullOrEmpty(conversationId)) return new(); var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return logs; + if (string.IsNullOrEmpty(convDir)) return new(); var logDir = Path.Combine(convDir, "state_log"); - if (!Directory.Exists(logDir)) return logs; + if (!Directory.Exists(logDir)) return new(); + var logs = new List(); foreach (var file in Directory.GetFiles(logDir)) { var text = File.ReadAllText(file); var log = JsonSerializer.Deserialize(text); - if (log == null) continue; + if (log == null || log.CreatedTime >= filter.StartTime) continue; logs.Add(log); } - return logs.OrderBy(x => x.CreatedTime).ToList(); + + logs = logs.OrderByDescending(x => x.CreatedTime).Take(filter.Size).ToList(); + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index d8b71339..6cb42ac6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -81,11 +81,11 @@ public class ConversationController : ControllerBase } [HttpGet("/conversation/{conversationId}/dialogs")] - public async Task> GetDialogs([FromRoute] string conversationId) + public async Task> GetDialogs([FromRoute] string conversationId, [FromQuery] int count = 100) { var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, [], isReadOnly: true); - var history = conv.GetDialogHistory(fromBreakpoint: false); + var history = conv.GetDialogHistory(lastCount: count, fromBreakpoint: false); var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index 892c3195..7032bcd3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -10,14 +10,11 @@ namespace BotSharp.OpenAPI.Controllers; public class LoggerController : ControllerBase { private readonly IServiceProvider _services; - private readonly IUserIdentity _user; public LoggerController( - IServiceProvider services, - IUserIdentity user) + IServiceProvider services) { _services = services; - _user = user; } [HttpGet("/logger/full-log")] @@ -40,17 +37,21 @@ public class LoggerController : ControllerBase #region Conversation log [HttpGet("/logger/conversation/{conversationId}/content-log")] - public async Task> GetConversationContentLogs([FromRoute] string conversationId) + public async Task> GetConversationContentLogs( + [FromRoute] string conversationId, + [FromQuery] ConversationLogFilter request) { var logging = _services.GetRequiredService(); - return await logging.GetConversationContentLogs(conversationId); + return await logging.GetConversationContentLogs(conversationId, request); } [HttpGet("/logger/conversation/{conversationId}/state-log")] - public async Task> GetConversationStateLogs([FromRoute] string conversationId) + public async Task> GetConversationStateLogs( + [FromRoute] string conversationId, + [FromQuery] ConversationLogFilter request) { var logging = _services.GetRequiredService(); - return await logging.GetConversationStateLogs(conversationId); + return await logging.GetConversationStateLogs(conversationId, request); } #endregion diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs index f4b4266f..cad0189d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs @@ -10,7 +10,6 @@ public class ChatHubCrontabHook : ICrontabHook private readonly IHubContext _chatHub; private readonly ILogger _logger; private readonly IUserIdentity _user; - private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; private readonly ChatHubSettings _settings; @@ -22,7 +21,6 @@ public class ChatHubCrontabHook : ICrontabHook IHubContext chatHub, ILogger logger, IUserIdentity user, - IConversationStorage storage, BotSharpOptions options, ChatHubSettings settings) { @@ -30,7 +28,6 @@ public class ChatHubCrontabHook : ICrontabHook _chatHub = chatHub; _logger = logger; _user = user; - _storage = storage; _options = options; _settings = settings; } @@ -58,19 +55,8 @@ public class ChatHubCrontabHook : ICrontabHook { try { - if (_settings.EventDispatchBy == EventDispatchType.Group) - { - await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); - } - else - { - await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); - } - } - catch (Exception ex) - { - _logger.LogWarning($"Failed to send event in {nameof(ChatHubCrontabHook)} (conversation id: {item.ConversationId})." + - $"\r\n{ex.Message}\r\n{ex.InnerException}"); + await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); } + catch { } } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 457faba2..8cd6e158 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -34,7 +35,8 @@ public partial class MongoRepository { if (log == null) return; - var found = _dc.Conversations.AsQueryable().FirstOrDefault(x => x.Id == log.ConversationId); + var filter = Builders.Filter.Eq(x => x.Id, log.ConversationId); + var found = _dc.Conversations.Find(filter).FirstOrDefault(); if (found == null) return; var logDoc = new ConversationContentLogDocument @@ -52,25 +54,36 @@ public partial class MongoRepository _dc.ContentLogs.InsertOne(logDoc); } - public List GetConversationContentLogs(string conversationId) + public DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { - var logs = _dc.ContentLogs - .AsQueryable() - .Where(x => x.ConversationId == conversationId) - .Select(x => new ContentLogOutputModel - { - ConversationId = x.ConversationId, - MessageId = x.MessageId, - Name = x.Name, - AgentId = x.AgentId, - Role = x.Role, - Source = x.Source, - Content = x.Content, - CreatedTime = x.CreatedTime - }) - .OrderBy(x => x.CreatedTime) - .ToList(); - return logs; + var builder = Builders.Filter; + var logFilters = new List> + { + builder.Eq(x => x.ConversationId, conversationId), + builder.Lt(x => x.CreatedTime, filter.StartTime) + }; + var logSortDef = Builders.Sort.Descending(x => x.CreatedTime); + + var docs = _dc.ContentLogs.Find(builder.And(logFilters)).Sort(logSortDef).Limit(filter.Size).ToList(); + var logs = docs.Select(x => new ContentLogOutputModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + Name = x.Name, + AgentId = x.AgentId, + Role = x.Role, + Source = x.Source, + Content = x.Content, + CreatedTime = x.CreatedTime + }).ToList(); + + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion @@ -79,7 +92,8 @@ public partial class MongoRepository { if (log == null) return; - var found = _dc.Conversations.AsQueryable().FirstOrDefault(x => x.Id == log.ConversationId); + var filter = Builders.Filter.Eq(x => x.Id, log.ConversationId); + var found = _dc.Conversations.Find(filter).FirstOrDefault(); if (found == null) return; var logDoc = new ConversationStateLogDocument @@ -94,22 +108,33 @@ public partial class MongoRepository _dc.StateLogs.InsertOne(logDoc); } - public List GetConversationStateLogs(string conversationId) + public DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { - var logs = _dc.StateLogs - .AsQueryable() - .Where(x => x.ConversationId == conversationId) - .Select(x => new ConversationStateLogModel - { - ConversationId = x.ConversationId, - AgentId = x.AgentId, - MessageId = x.MessageId, - States = x.States, - CreatedTime = x.CreatedTime - }) - .OrderBy(x => x.CreatedTime) - .ToList(); - return logs; + var builder = Builders.Filter; + var logFilters = new List> + { + builder.Eq(x => x.ConversationId, conversationId), + builder.Lt(x => x.CreatedTime, filter.StartTime) + }; + var logSortDef = Builders.Sort.Descending(x => x.CreatedTime); + + var docs = _dc.StateLogs.Find(builder.And(logFilters)).Sort(logSortDef).Limit(filter.Size).ToList(); + var logs = docs.Select(x => new ConversationStateLogModel + { + ConversationId = x.ConversationId, + AgentId = x.AgentId, + MessageId = x.MessageId, + States = x.States, + CreatedTime = x.CreatedTime + }).ToList(); + + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion From 9b6cf7a9ac2d500284fe839322d8ce68947a9d2a Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Wed, 12 Mar 2025 10:51:24 -0500 Subject: [PATCH 66/84] Update settings for IsEnableScreenshot for web driver --- .../Browsing/Settings/WebBrowsingSettings.cs | 1 + .../PlaywrightWebDriver.Screenshot.cs | 24 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 9565e217..7521a3e4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -6,4 +6,5 @@ public class WebBrowsingSettings public bool Headless { get; set; } // Default timeout in milliseconds public float DefaultTimeout { get; set; } = 30000; + public bool IsEnableScreenshot { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index 294c95db..b74af9c4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver @@ -5,19 +7,21 @@ public partial class PlaywrightWebDriver public async Task ScreenshotAsync(MessageInfo message, string path) { var result = new BrowserActionResult(); - - await _instance.Wait(message.ContextId, waitNetworkIdle: false); - var page = _instance.GetPage(message.ContextId); - - await Task.Delay(300); - var bytes = await page.ScreenshotAsync(new PageScreenshotOptions + var _webDriver = _services.GetRequiredService(); + if (_webDriver.IsEnableScreenshot) { - Path = path, - FullPage = true - }); + await _instance.Wait(message.ContextId, waitNetworkIdle: false); + var page = _instance.GetPage(message.ContextId); + await Task.Delay(300); + var bytes = await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = path, + FullPage = true + }); + result.Body = "data:image/png;base64," + Convert.ToBase64String(bytes); + } result.IsSuccess = true; - result.Body = "data:image/png;base64," + Convert.ToBase64String(bytes); return result; } } From 70eadbd5104b6b1b5a12994cb9fd3437822dac47 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 12 Mar 2025 14:03:38 -0500 Subject: [PATCH 67/84] IRealtimeHook --- .../Conversations/ConversationHookBase.cs | 3 +++ .../Conversations/IConversationHook.cs | 7 ++++++ .../Realtime/IRealtimeHook.cs | 6 +++++ .../BotSharp.Core.Realtime.csproj | 1 + .../Services/RealtimeHub.cs | 21 ++++++++++------ .../Models/Realtime/RealtimeSessionBody.cs | 3 ++- .../Realtime/RealTimeCompletionProvider.cs | 25 +++++++++++++------ .../Functions/HangupPhoneCallFn.cs | 2 +- 8 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index da87646e..0f115d5d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -78,4 +78,7 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; + + public Task OnUserDisconnected(Conversation conversation) + => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index c764f391..f99078ea 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -25,6 +25,13 @@ public interface IConversationHook /// Task OnUserAgentConnectedInitially(Conversation conversation); + /// + /// Triggered when user disconnects with agent. + /// + /// + /// + Task OnUserDisconnected(Conversation conversation); + /// /// Triggered once for every new conversation. /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs new file mode 100644 index 00000000..2ad8f1df --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Realtime; + +public interface IRealtimeHook +{ + string[] OnModelTranscriptPrompt(Agent agent); +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index b29ff39e..c004dc50 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index e8548c4d..0e6027da 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Utilities; +using BotSharp.Core.Infrastructures; +using Microsoft.AspNetCore.Cors.Infrastructure; namespace BotSharp.Core.Realtime.Services; @@ -86,11 +88,14 @@ public class RealtimeHub : IRealtimeHub var routing = _services.GetRequiredService(); routing.Context.Push(agent.Id); + var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); if (dialogs.Count == 0) { dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); + storage.Append(_conn.ConversationId, dialogs.First()); } + routing.Context.SetDialogs(dialogs); await _completer.Connect(_conn, @@ -155,6 +160,7 @@ public class RealtimeHub : IRealtimeHub { // append output audio transcript to conversation dialogs.Add(message); + storage.Append(_conn.ConversationId, message); foreach (var hook in hookProvider.HooksOrderByPriority) { @@ -174,6 +180,7 @@ public class RealtimeHub : IRealtimeHub { // append input audio transcript to conversation dialogs.Add(message); + storage.Append(_conn.ConversationId, message); foreach (var hook in hookProvider.HooksOrderByPriority) { @@ -224,6 +231,9 @@ public class RealtimeHub : IRealtimeHub }; dialogs.Add(message); + var storage = _services.GetRequiredService(); + storage.Append(_conn.ConversationId, message); + foreach (var hook in hookProvider.HooksOrderByPriority) { hook.SetAgent(agent) @@ -239,14 +249,9 @@ public class RealtimeHub : IRealtimeHub private async Task HandleUserDisconnected() { - // Save dialog history - var routing = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - var dialogs = routing.Context.GetDialogs(); - foreach (var item in dialogs) - { - storage.Append(_conn.ConversationId, item); - } + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(_conn.ConversationId); + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation)); } private async Task SendEventToUser(WebSocket webSocket, object message) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index ff2bfc3f..f6a6322c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -77,5 +77,6 @@ public class InputAudioTranscription public string Language { get; set; } = "en"; [JsonPropertyName("prompt")] - public string Prompt { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Prompt { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index af6f0ad3..47a9030d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -2,7 +2,9 @@ using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; @@ -42,7 +44,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, @@ -64,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, - onAudioTranscriptDone, + onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, onInputAudioTranscriptionCompleted, @@ -125,10 +127,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, + Action onUserAudioTranscriptionCompleted, Action onUserInterrupted) { var buffer = new byte[1024 * 32]; @@ -171,7 +173,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); await Task.Delay(1000); - onAudioTranscriptDone(data.Transcript); + onModelAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { @@ -201,8 +203,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "conversation.item.input_audio_transcription.completed") { _logger.LogInformation($"{response.Type}: {receivedText}"); - var message = await OnInputAudioTranscriptionCompleted(conn, receivedText); - onInputAudioTranscriptionCompleted(message); + var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); + if (!string.IsNullOrEmpty(message.Content)) + { + onUserAudioTranscriptionCompleted(message); + } } else if (response.Type == "input_audio_buffer.speech_started") { @@ -309,6 +314,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return fn; }).ToArray(); + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var sessionUpdate = new { type = "session.update", @@ -320,6 +328,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { Model = "whisper-1", Language = "en", + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }, Voice = "alloy", Instructions = instruction, @@ -663,7 +672,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return outputs; } - public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) { var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index bc1bad36..cd755306 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -45,7 +45,7 @@ public class HangupPhoneCallFn : IFunctionCallback pathSid: callSid ); - message.Content = "The call has ended."; + message.Content = "The call has been ended."; message.StopCompletion = true; }); From 7646030f782b279a353604199022955fa09f935a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 12 Mar 2025 16:33:20 -0500 Subject: [PATCH 68/84] add vector storage snapshots --- .../Files/Utilities/FileUtility.cs | 11 ++ .../Knowledges/IKnowledgeService.cs | 8 + .../VectorStorage/IVectorDb.cs | 42 +++-- .../Snapshot/VectorCollectionSnapshot.cs | 9 + .../Controllers/DashboardController.cs | 10 +- .../Controllers/KnowledgeBaseController.cs | 54 +++++- .../{ => Request}/AgentCreationModel.cs | 0 .../{ => Request}/AgentTaskCreateModel.cs | 0 .../{ => Request}/AgentTaskUpdateModel.cs | 0 .../{ => Request}/AgentTemplatePatchModel.cs | 0 .../Agents/{ => Request}/AgentUpdateModel.cs | 0 .../{ => Request}/RoutingRuleUpdateModel.cs | 0 .../Agents/{ => View}/AgentTaskViewModel.cs | 0 .../Agents/{ => View}/AgentViewModel.cs | 0 .../ConversationCreationModel.cs | 0 .../{ => Request}/ConversationSummaryModel.cs | 0 .../{ => Request}/InputMessageFiles.cs | 0 .../MigrateLatestStateRequest.cs | 0 .../{ => Request}/NewMessageModel.cs | 0 .../UpdateConversationRequest.cs | 0 .../UpdateConversationTitleAliasModel.cs | 0 .../UpdateConversationTitleModel.cs | 0 .../{ => Request}/UpdateMessageModel.cs | 0 .../{ => Response}/ChatResponseModel.cs | 0 .../{ => View}/ConversationViewModel.cs | 0 .../{ => Request}/EmbeddingInputModel.cs | 0 .../Files/{ => View}/MessageFileViewModel.cs | 0 .../{ => Request}/InstructBaseRequest.cs | 0 .../{ => Request}/InstructMessageModel.cs | 0 .../{ => View}/ImageGenerationViewModel.cs | 0 .../{ => View}/InstructBaseViewModel.cs | 0 .../{ => View}/InstructionLogViewModel.cs | 0 .../{ => View}/MultiModalViewModel.cs | 0 .../{ => View}/PdfCompletionViewModel.cs | 0 .../{ => View}/SpeechToTextViewModel.cs | 0 .../CreateVectorCollectionRequest.cs | 0 .../DeleteVectorCollectionSnapshotRequest.cs | 9 + .../{ => Request}/GetKnowledgeDocsRequest.cs | 0 .../SearchGraphKnowledgeRequest.cs | 0 .../SearchVectorKnowledgeRequest.cs | 0 .../VectorKnowledgeCreateRequest.cs | 0 .../VectorKnowledgeUpdateRequest.cs | 0 .../VectorKnowledgeUploadRequest.cs | 0 .../{ => View}/GraphKnowledgeViewModel.cs | 0 .../{ => View}/KnowledgeFileViewModel.cs | 0 .../VectorCollectionConfigViewModel.cs | 0 .../View/VectorCollectionSnapshotViewModel.cs | 35 ++++ .../{ => View}/VectorKnowledgeViewModel.cs | 0 .../{ => Request}/RoleAgentActionViewModel.cs | 0 .../Roles/{ => Request}/RoleViewModel.cs | 0 .../Roles/{ => View}/RoleUpdateModel.cs | 0 .../Users/{ => Request}/UserAvatarModel.cs | 0 .../Users/{ => Request}/UserCreationModel.cs | 0 .../{ => Request}/UserResetPasswordModel.cs | 0 .../Users/{ => Request}/UserUpdateModel.cs | 0 .../{ => View}/UserAgentActionViewModel.cs | 0 .../UserDashboardConversationViewModel.cs} | 13 +- .../Users/{ => View}/UserViewModel.cs | 0 src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../Services/KnowledgeService.Snapshot.cs | 64 +++++++ .../Models/RecoverFromSnapshotResponse.cs | 15 ++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 158 ++++++++++++++++++ 62 files changed, 403 insertions(+), 27 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => Request}/AgentCreationModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => Request}/AgentTaskCreateModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => Request}/AgentTaskUpdateModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => Request}/AgentTemplatePatchModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => Request}/AgentUpdateModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => Request}/RoutingRuleUpdateModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => View}/AgentTaskViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/{ => View}/AgentViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/ConversationCreationModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/ConversationSummaryModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/InputMessageFiles.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/MigrateLatestStateRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/NewMessageModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/UpdateConversationRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/UpdateConversationTitleAliasModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/UpdateConversationTitleModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Request}/UpdateMessageModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => Response}/ChatResponseModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/{ => View}/ConversationViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/{ => Request}/EmbeddingInputModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/{ => View}/MessageFileViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => Request}/InstructBaseRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => Request}/InstructMessageModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => View}/ImageGenerationViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => View}/InstructBaseViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => View}/InstructionLogViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => View}/MultiModalViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => View}/PdfCompletionViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/{ => View}/SpeechToTextViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/CreateVectorCollectionRequest.cs (100%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/GetKnowledgeDocsRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/SearchGraphKnowledgeRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/SearchVectorKnowledgeRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/VectorKnowledgeCreateRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/VectorKnowledgeUpdateRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => Request}/VectorKnowledgeUploadRequest.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => View}/GraphKnowledgeViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => View}/KnowledgeFileViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => View}/VectorCollectionConfigViewModel.cs (100%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{ => View}/VectorKnowledgeViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/{ => Request}/RoleAgentActionViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/{ => Request}/RoleViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/{ => View}/RoleUpdateModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{ => Request}/UserAvatarModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{ => Request}/UserCreationModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{ => Request}/UserResetPasswordModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{ => Request}/UserUpdateModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{ => View}/UserAgentActionViewModel.cs (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{UserDashboardConversationModel.cs => View/UserDashboardConversationViewModel.cs} (57%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/{ => View}/UserViewModel.cs (100%) create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs create mode 100644 src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index bf2644dd..db9ccb65 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -47,6 +47,17 @@ public static class FileUtility return $"data:{contentType};base64,{base64}"; } + public static BinaryData BuildBinaryDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var binary = BinaryData.FromStream(stream); + stream.Close(); + + return binary; + } + public static string GetFileContentType(string fileName) { string contentType; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 3fc90816..fb368b0b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -60,6 +60,14 @@ public interface IKnowledgeService Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion + #region Snapshot + Task> GetVectorCollectionSnapshots(string collectionName); + Task CreateVectorCollectionSnapshot(string collectionName); + Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName); + Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData); + Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); + #endregion + #region Common Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 66f7727a..ff090da6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,14 +6,36 @@ public interface IVectorDb { string Provider { get; } - Task DoesCollectionExist(string collectionName); - Task> GetCollections(); - Task> GetPagedCollectionData(string collectionName, VectorFilter filter); - Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dimension); - Task DeleteCollection(string collectionName); - Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); - Task DeleteCollectionData(string collectionName, List ids); - Task DeleteCollectionAllData(string collectionName); + Task DoesCollectionExist(string collectionName) + => throw new NotImplementedException(); + Task> GetCollections() + => throw new NotImplementedException(); + Task> GetPagedCollectionData(string collectionName, VectorFilter filter) + => throw new NotImplementedException(); + Task> GetCollectionData(string collectionName, IEnumerable ids, + bool withPayload = false, bool withVector = false) + => throw new NotImplementedException(); + Task CreateCollection(string collectionName, int dimension) + => throw new NotImplementedException(); + Task DeleteCollection(string collectionName) + => throw new NotImplementedException(); + Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) + => throw new NotImplementedException(); + Task> Search(string collectionName, float[] vector, IEnumerable? fields, + int limit = 5, float confidence = 0.5f, bool withVector = false) + => throw new NotImplementedException(); + Task DeleteCollectionData(string collectionName, List ids) + => throw new NotImplementedException(); + Task DeleteCollectionAllData(string collectionName) + => throw new NotImplementedException(); + Task> GetCollectionSnapshots(string collectionName) + => throw new NotImplementedException(); + Task CreateCollectionShapshot(string collectionName) + => throw new NotImplementedException(); + Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName) + => throw new NotImplementedException(); + Task RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) + => throw new NotImplementedException(); + Task DeleteCollectionShapshot(string collectionName, string snapshotName) + => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs new file mode 100644 index 00000000..a8405a80 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionSnapshot +{ + public string Name { get; set; } = default!; + public long Size { get; set; } + public DateTime CreatedTime { get; set; } + public string? CheckSum { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs index c95d819d..93cfefde 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs @@ -19,16 +19,16 @@ public class DashboardController : ControllerBase } #region User Components [HttpGet("/dashboard/components")] - public async Task GetComponents() + public async Task GetComponents() { var userService = _services.GetRequiredService(); var dashboardProfile = await userService.GetDashboard(); - if (dashboardProfile == null) return new UserDashboardModel(); + if (dashboardProfile == null) return new(); - var result = new UserDashboardModel + var result = new UserDashboardViewModel { ConversationList = dashboardProfile.ConversationList.Select( - x => new UserDashboardConversationModel + x => new UserDashboardConversationViewModel { Name = x.Name, ConversationId = x.ConversationId, @@ -40,7 +40,7 @@ public class DashboardController : ControllerBase } [HttpPost("/dashboard/component/conversation")] - public async Task UpdateDashboardConversationInstruction(UserDashboardConversationModel dashConv) + public async Task UpdateDashboardConversationInstruction(UserDashboardConversationViewModel dashConv) { if (string.IsNullOrEmpty(dashConv.Name) && string.IsNullOrEmpty(dashConv.Instruction)) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 50b7c528..f5c67070 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -12,7 +12,9 @@ public class KnowledgeBaseController : ControllerBase private readonly IKnowledgeService _knowledgeService; private readonly IServiceProvider _services; - public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvider services) + public KnowledgeBaseController( + IKnowledgeService knowledgeService, + IServiceProvider services) { _knowledgeService = knowledgeService; _services = services; @@ -117,6 +119,46 @@ public class KnowledgeBaseController : ControllerBase #endregion + #region Snapshot + [HttpGet("/knowledge/vector/{collection}/snapshots")] + public async Task> GetVectorCollectionSnapshots([FromRoute] string collection) + { + var snapshots = await _knowledgeService.GetVectorCollectionSnapshots(collection); + return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x)); + } + + [HttpPost("/knowledge/vector/{collection}/snapshot")] + public async Task CreateVectorCollectionSnapshot([FromRoute] string collection) + { + var snapshot = await _knowledgeService.CreateVectorCollectionSnapshot(collection); + return VectorCollectionSnapshotViewModel.From(snapshot); + } + + [HttpGet("/knowledge/vector/{collection}/snapshot")] + public async Task GetVectorCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName) + { + var snapshot = await _knowledgeService.DownloadVectorCollectionSnapshot(collection, snapshotFileName); + return BuildFileResult(snapshotFileName, snapshot); + } + + [HttpPost("/knowledge/vector/{collection}/snapshot/recover")] + public async Task RecoverVectorCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile) + { + var fileName = snapshotFile.FileName; + var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile); + var done = await _knowledgeService.RecoverVectorCollectionFromSnapshot(collection, fileName, binary); + return done; + } + + [HttpDelete("/knowledge/vector/{collection}/snapshot")] + public async Task DeleteVectorCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteVectorCollectionSnapshotRequest request) + { + var done = await _knowledgeService.DeleteVectorCollectionSnapshot(collection, request.SnapshotName); + return done; + } + #endregion + + #region Document [HttpPost("/knowledge/document/{collection}/upload")] public async Task UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) @@ -187,7 +229,6 @@ public class KnowledgeBaseController : ControllerBase #endregion - #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) @@ -214,4 +255,13 @@ public class KnowledgeBaseController : ControllerBase return saved ? "Success" : "Fail"; } #endregion + + #region Private methods + private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) + { + var stream = fileData.ToStream(); + stream.Position = 0; + return File(stream, "application/octet-stream", Path.GetFileName(fileName)); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentCreationModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskCreateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskCreateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTaskUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTemplatePatchModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTemplatePatchModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTemplatePatchModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentTemplatePatchModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/AgentUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/RoutingRuleUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/RoutingRuleUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/RoutingRuleUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/Request/RoutingRuleUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentTaskViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentTaskViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationCreationModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationCreationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationCreationModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationSummaryModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationSummaryModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationSummaryModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/ConversationSummaryModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/InputMessageFiles.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/InputMessageFiles.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/MigrateLatestStateRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/MigrateLatestStateRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/NewMessageModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/NewMessageModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleAliasModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleAliasModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleAliasModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleAliasModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateConversationTitleModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationTitleModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateMessageModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateMessageModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/EmbeddingInputModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/Request/EmbeddingInputModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/EmbeddingInputModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/Request/EmbeddingInputModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/View/MessageFileViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/View/MessageFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/ImageGenerationViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/ImageGenerationViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructBaseViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructBaseViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructionLogViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/InstructionLogViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/MultiModalViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/MultiModalViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/PdfCompletionViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/PdfCompletionViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/SpeechToTextViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/View/SpeechToTextViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs new file mode 100644 index 00000000..30398b2c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteVectorCollectionSnapshotRequest +{ + [JsonPropertyName("snapshot_name")] + public string SnapshotName { get; set; } = default!; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/GraphKnowledgeViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/GraphKnowledgeViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeFileViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorCollectionConfigViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorCollectionConfigViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs new file mode 100644 index 00000000..95b8a325 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs @@ -0,0 +1,35 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorCollectionSnapshotViewModel +{ + [JsonPropertyName("name")] + public string Name { get; set; } = default!; + + [JsonPropertyName("size")] + public long Size { get; set; } + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } + + [JsonPropertyName("check_sum")] + public string? CheckSum { get; set; } + + public static VectorCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model) + { + if (model == null) + { + return null; + } + + return new VectorCollectionSnapshotViewModel + { + Name = model.Name, + Size = model.Size, + CreatedTime = model.CreatedTime, + CheckSum = model.CheckSum + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleAgentActionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleAgentActionViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleAgentActionViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleAgentActionViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/Request/RoleViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/View/RoleUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/RoleUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Roles/View/RoleUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserAvatarModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserAvatarModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserCreationModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserCreationModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserResetPasswordModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserResetPasswordModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserUpdateModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserUpdateModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/Request/UserUpdateModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAgentActionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserAgentActionViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAgentActionViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserAgentActionViewModel.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserDashboardConversationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs similarity index 57% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserDashboardConversationModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs index 3a5f3491..7165fa87 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserDashboardConversationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs @@ -1,19 +1,14 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.OpenAPI.ViewModels.Users; -public class UserDashboardModel -{ +public class UserDashboardViewModel +{ [JsonPropertyName("conversation_list")] - public IList ConversationList { get; set; } = []; + public IList ConversationList { get; set; } = []; } -public class UserDashboardConversationModel +public class UserDashboardConversationViewModel { [JsonPropertyName("name")] public string? Name { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserViewModel.cs similarity index 100% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserViewModel.cs diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 0465dc4e..99378ddc 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -60,9 +60,9 @@ public class GraphDb : IGraphDb using (var client = http.CreateClient()) { - var uri = new Uri(url); try { + var uri = new Uri(url); var data = JsonSerializer.Serialize(request, _jsonOptions); var message = new HttpRequestMessage { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs new file mode 100644 index 00000000..4b839332 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs @@ -0,0 +1,64 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetVectorCollectionSnapshots(string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return Enumerable.Empty(); + } + + var db = GetVectorDb(); + var snapshots = await db.GetCollectionSnapshots(collectionName); + return snapshots; + } + + public async Task CreateVectorCollectionSnapshot(string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return null; + } + + var db = GetVectorDb(); + var snapshot = await db.CreateCollectionShapshot(collectionName); + return snapshot; + } + + public async Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) + { + return BinaryData.Empty; + } + + var db = GetVectorDb(); + var snapshot = await db.DownloadCollectionSnapshot(collectionName, snapshotFileName); + return snapshot; + } + + public async Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + var done = await db.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData); + return done; + } + + public async Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) + { + return false; + } + + var db = GetVectorDb(); + var done = await db.DeleteCollectionShapshot(collectionName, snapshotName); + return done; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs new file mode 100644 index 00000000..f346aa1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Qdrant.Models; + +public class RecoverFromSnapshotResponse +{ + [JsonPropertyName("time")] + public decimal Time { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("result")] + public bool Result { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index ef9faa93..3c5d61e5 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,8 +1,14 @@ +using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Plugin.Qdrant.Models; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; +using System.Net.Http; +using System.Net.Mime; +using System.Text.Json; namespace BotSharp.Plugin.Qdrant; @@ -10,15 +16,18 @@ public class QdrantDb : IVectorDb { private QdrantClient _client; private readonly QdrantSetting _setting; + private readonly BotSharpOptions _options; private readonly IServiceProvider _services; private readonly ILogger _logger; public QdrantDb( QdrantSetting setting, + BotSharpOptions options, ILogger logger, IServiceProvider services) { _setting = setting; + _options = options; _logger = logger; _services = services; } @@ -39,6 +48,7 @@ public class QdrantDb : IVectorDb return _client; } + #region Collection public async Task DoesCollectionExist(string collectionName) { var client = GetClient(); @@ -86,7 +96,9 @@ public class QdrantDb : IVectorDb var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } + #endregion + #region Collection data public async Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { var exist = await DoesCollectionExist(collectionName); @@ -332,4 +344,150 @@ public class QdrantDb : IVectorDb var result = await client.DeleteAsync(collectionName, new Filter()); return result.Status == UpdateStatus.Completed; } + #endregion + + #region Snapshots + public async Task> GetCollectionSnapshots(string collectionName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return Enumerable.Empty(); + } + + var client = GetClient(); + var data = await client.ListSnapshotsAsync(collectionName); + var snapshots = data.Select(x => new VectorCollectionSnapshot + { + Name = x.Name, + Size = x.Size, + CreatedTime = x.CreationTime.ToDateTime(), + CheckSum = x.Checksum + }); + return snapshots; + } + + public async Task CreateCollectionShapshot(string collectionName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return null; + } + + var client = GetClient(); + var desc = await client.CreateSnapshotAsync(collectionName); + if (desc == null) + { + return null; + } + + return new VectorCollectionSnapshot + { + Name = desc.Name, + Size = desc.Size, + CreatedTime = desc.CreationTime.ToDateTime(), + CheckSum = desc.Checksum + }; + } + + public async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return BinaryData.Empty; + } + + var domain = $"https://{_setting.Url}:6333"; + var url = $"{domain}/collections/{collectionName}/snapshots/{snapshotFileName}"; + + var http = _services.GetRequiredService(); + using (var client = http.CreateClient()) + { + try + { + var uri = new Uri(url); + var message = new HttpRequestMessage + { + Method = HttpMethod.Get, + RequestUri = uri + }; + + client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey); + var rawResponse = await client.SendAsync(message); + rawResponse.EnsureSuccessStatusCode(); + + using var contentStream = await rawResponse.Content.ReadAsStreamAsync(); + return BinaryData.FromStream(contentStream); + } + catch (Exception ex) + { + _logger.LogError($"Error when downloading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}"); + return BinaryData.Empty; + } + } + } + + public async Task RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) + { + var domain = $"https://{_setting.Url}:6333"; + var url = $"{domain}/collections/{collectionName}/snapshots/upload"; + + var http = _services.GetRequiredService(); + using (var client = http.CreateClient()) + { + try + { + var uri = new Uri(url); + var data = new MultipartFormDataContent + { + { new StringContent(snapshotFileName), "name" }, + { new StringContent(MediaTypeNames.Application.Octet), "type" }, + { new StreamContent(snapshotData.ToStream()), "snapshot", snapshotFileName } + }; + + var message = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = uri, + Content = data + }; + + client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey); + var rawResponse = await client.SendAsync(message); + rawResponse.EnsureSuccessStatusCode(); + + var responseStr = await rawResponse.Content.ReadAsStringAsync(); + var response = JsonSerializer.Deserialize(responseStr, _options.JsonSerializerOptions); + return response?.Result == true; + } + catch (Exception ex) + { + _logger.LogError($"Error when uploading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + } + + public async Task DeleteCollectionShapshot(string collectionName, string snapshotName) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return false; + } + + try + { + var client = GetClient(); + await client.DeleteSnapshotAsync(collectionName, snapshotName); + return true; + } + catch + { + return false; + } + } + #endregion } From 14c476ce0da38e7419dcc24ce2b4453d7c1881e7 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 12 Mar 2025 18:30:53 -0500 Subject: [PATCH 69/84] OnUserDisconnected --- .../BotSharp.Abstraction/Conversations/ConversationHookBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index 0f115d5d..a84c8693 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -79,6 +79,6 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; - public Task OnUserDisconnected(Conversation conversation) + public virtual Task OnUserDisconnected(Conversation conversation) => Task.CompletedTask; } From f0ab14bb4db95bab20bff4e6fd411ab4cd803372 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 12 Mar 2025 19:20:30 -0500 Subject: [PATCH 70/84] Avoid duplicate message if there is init_audio_file --- .../Services/RealtimeHub.cs | 29 ++++++++++++------- .../Controllers/TwilioStreamController.cs | 8 ++++- .../Models/ConversationalVoiceRequest.cs | 1 + 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 0e6027da..7dc34f27 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -98,24 +98,33 @@ public class RealtimeHub : IRealtimeHub routing.Context.SetDialogs(dialogs); + var states = _services.GetRequiredService(); + await _completer.Connect(_conn, onModelReady: async () => { - // Control initial session, prevent initial response interruption - await _completer.UpdateSession(_conn, turnDetection: false); - - if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) + if (states.ContainsState("init_audio_file")) { - await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await _completer.UpdateSession(_conn, turnDetection: true); } else { - await _completer.TriggerModelInference("Reply based on the conversation context."); - } + // Control initial session, prevent initial response interruption + await _completer.UpdateSession(_conn, turnDetection: false); - // Start turn detection - await Task.Delay(1000 * 8); - await _completer.UpdateSession(_conn, turnDetection: true); + if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) + { + await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + } + else + { + await _completer.TriggerModelInference("Reply based on the conversation context."); + } + + // Start turn detection + await Task.Delay(1000 * 8); + await _completer.UpdateSession(_conn, turnDetection: true); + } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index dcb33959..7fb7e0e0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -44,7 +44,8 @@ public class TwilioStreamController : TwilioController if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file")) { - instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]); + request.InitAudioFile = _context.HttpContext.Request.Query["init_audio_file"]; + instruction.SpeechPaths.Add(request.InitAudioFile); } if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) @@ -104,6 +105,11 @@ public class TwilioStreamController : TwilioController new(StateConst.ROUTING_MODE, "lazy"), }; + if (request.InitAudioFile != null) + { + states.Add(new("init_audio_file", request.InitAudioFile)); + } + convService.SetConversationId(conversation.Id, states); convService.SaveStates(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 33ee7633..a17b344f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -18,6 +18,7 @@ public class ConversationalVoiceRequest : VoiceRequest public string? AIResponseErrorMessage { get; set; } = string.Empty; public string Intent { get; set; } = string.Empty; + public string? InitAudioFile { get; set; } public List States { get; set; } = []; } From d2ba1a3dc0f23b467e4a4e5d2b6ca2a5634a66a5 Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Wed, 12 Mar 2025 21:35:30 -0500 Subject: [PATCH 71/84] Update default wait time for web driver --- .../Browsing/Settings/WebBrowsingSettings.cs | 2 ++ .../UtilFunctions/UtilWebGoToPageFn.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 7521a3e4..f876c449 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -7,4 +7,6 @@ public class WebBrowsingSettings // Default timeout in milliseconds public float DefaultTimeout { get; set; } = 30000; public bool IsEnableScreenshot { get; set; } + // Default wait time in seconds after page is opened + public int DefaultWaitTime { get; set; } = 5; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index a29af212..0c5507bf 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -26,7 +26,7 @@ public class UtilWebGoToPageFn : IFunctionCallback var _webDriver = _services.GetRequiredService(); args.Timeout = _webDriver.DefaultTimeout; args.WaitForNetworkIdle = false; - args.WaitTime = 5; + args.WaitTime = _webDriver.DefaultWaitTime; args.OpenNewTab = true; var conv = _services.GetRequiredService(); From c4838da1fbd95c4d51c781dc1cfe767ac3343233 Mon Sep 17 00:00:00 2001 From: vguruparan Date: Thu, 13 Mar 2025 11:39:37 -0500 Subject: [PATCH 72/84] Update no sandbox argument for playwright instance --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 755d4faf..283ad314 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -77,7 +77,8 @@ public class PlaywrightInstance : IDisposable Args = [ "--disable-infobars", - "--test-type" + "--test-type", + "--no-sandbox" // "--start-maximized" ] }); From a2e81a9b81d56571624e3f1cc6bff9e8b94fcd17 Mon Sep 17 00:00:00 2001 From: visagang <47051333+visagang@users.noreply.github.com> Date: Thu, 13 Mar 2025 16:44:14 +0000 Subject: [PATCH 73/84] Update no-sandbox argument for playwright instance --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 755d4faf..283ad314 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -77,7 +77,8 @@ public class PlaywrightInstance : IDisposable Args = [ "--disable-infobars", - "--test-type" + "--test-type", + "--no-sandbox" // "--start-maximized" ] }); From 9a8f559785bb94b50cd3d1efbd3abef59432f09a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 13 Mar 2025 13:06:00 -0500 Subject: [PATCH 74/84] temp save --- .../Conversations/IConversationService.cs | 2 +- .../Repositories/IBotSharpRepository.cs | 2 +- .../Conversations/Services/ConversationService.cs | 4 ++-- .../FileRepository/FileRepository.Conversation.cs | 7 +++++-- .../BotSharp.OpenAPI/Controllers/ConversationController.cs | 2 +- .../Conversations/Request/UpdateConversationRequest.cs | 3 ++- .../Repository/MongoRepository.Conversation.cs | 7 ++++++- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 44ceebf1..15f7e89e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -13,7 +13,7 @@ public interface IConversationService Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); Task UpdateConversationTitleAlias(string id, string titleAlias); - Task UpdateConversationTags(string conversationId, List tags); + Task UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags); Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request); Task> GetLastConversations(); Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 9bccb82e..332ca340 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -132,7 +132,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); void UpdateConversationTitleAlias(string conversationId, string titleAlias) => throw new NotImplementedException(); - bool UpdateConversationTags(string conversationId, List tags) + bool UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) => throw new NotImplementedException(); bool AppendConversationTags(string conversationId, List tags) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index a1a0868a..9fc97a72 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -59,10 +59,10 @@ public partial class ConversationService : IConversationService return conversation; } - public async Task UpdateConversationTags(string conversationId, List tags) + public async Task UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) { var db = _services.GetRequiredService(); - return db.UpdateConversationTags(conversationId, tags); + return db.UpdateConversationTags(conversationId, toAddTags, toDeleteTags); } public async Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 3d70264c..4e5998fb 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -158,7 +158,7 @@ public partial class FileRepository } } - public bool UpdateConversationTags(string conversationId, List tags) + public bool UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) { if (string.IsNullOrEmpty(conversationId)) return false; @@ -170,7 +170,10 @@ public partial class FileRepository var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); - conv.Tags = tags ?? new(); + + var tags = conv.Tags.Concat(toAddTags).ToList(); + conv.Tags = tags.Where(x => !toDeleteTags.Contains(x, StringComparer.OrdinalIgnoreCase)).ToList(); + conv.UpdatedTime = DateTime.UtcNow; File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); return true; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 6cb42ac6..f032f504 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -255,7 +255,7 @@ public class ConversationController : ControllerBase public async Task UpdateConversationTags([FromRoute] string conversationId, [FromBody] UpdateConversationRequest request) { var conv = _services.GetRequiredService(); - return await conv.UpdateConversationTags(conversationId, request.Tags); + return await conv.UpdateConversationTags(conversationId, request.ToAddTags, request.ToDeleteTags); } [HttpPut("/conversation/{conversationId}/update-message")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs index c9b89747..c177221c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Request/UpdateConversationRequest.cs @@ -2,5 +2,6 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations; public class UpdateConversationRequest { - public List Tags { get; set; } = []; + public List ToAddTags { get; set; } = []; + public List ToDeleteTags { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 908e66b5..393846bf 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -134,11 +134,16 @@ public partial class MongoRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } - public bool UpdateConversationTags(string conversationId, List tags) + public bool UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) { if (string.IsNullOrEmpty(conversationId)) return false; var filter = Builders.Filter.Eq(x => x.Id, conversationId); + var conv = _dc.Conversations.Find(filter).FirstOrDefault(); + if (conv == null) return false; + + + var update = Builders.Update .Set(x => x.Tags, tags ?? new()) .Set(x => x.UpdatedTime, DateTime.UtcNow); From f6994a3a6fe44e710b03945dcc10a57a905145f9 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 13 Mar 2025 14:05:06 -0500 Subject: [PATCH 75/84] ITwilioCallStatusHook --- .../Services/RealtimeHub.cs | 1 - .../Controllers/TwilioStreamController.cs | 38 +++++++++++++++---- .../Controllers/TwilioVoiceController.cs | 20 +++++----- .../Interfaces/ITwilioCallStatusHook.cs | 8 ++++ .../Models/ConversationalVoiceRequest.cs | 22 ++++++++++- .../Functions/OutboundPhoneCallFn.cs | 7 +++- .../Services/TwilioService.cs | 1 + 7 files changed, 75 insertions(+), 22 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 7dc34f27..76302262 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -26,7 +26,6 @@ public class RealtimeHub : IRealtimeHub { var buffer = new byte[1024 * 16]; WebSocketReceiveResult result; - do diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 7fb7e0e0..e69f79f6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -35,24 +35,28 @@ public class TwilioStreamController : TwilioController throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } - VoiceResponse response = null; + VoiceResponse response = default!; + + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + response = new VoiceResponse(); + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}")); + return TwiML(response); + } + var instruction = new ConversationalVoiceResponse { SpeechPaths = [], ActionOnEmptyResult = true }; - if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file")) + if (request.InitAudioFile != null) { - request.InitAudioFile = _context.HttpContext.Request.Query["init_audio_file"]; instruction.SpeechPaths.Add(request.InitAudioFile); } - if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) - { - request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; - } - await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); @@ -78,6 +82,24 @@ public class TwilioStreamController : TwilioController return TwiML(response); } + [ValidateRequest] + [HttpPost("twilio/stream/status")] + public async Task StreamConversationStatus(ConversationalVoiceRequest request) + { + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null && + request.CallStatus == "completed") + { + // voicemail + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailLeft(request.ConversationId); + }); + } + return Ok(); + } + private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 807d7af8..6507a105 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -66,7 +66,7 @@ public class TwilioVoiceController : TwilioController }); request.ConversationId = $"TwilioVoice_{request.CallSid}"; - instruction.CallbackPath = $"twilio/voice/{request.ConversationId}/receive/0?{GenerateStatesParameter(request.States)}"; + instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"; var twilio = _services.GetRequiredService(); if (string.IsNullOrWhiteSpace(request.Intent)) @@ -89,7 +89,7 @@ public class TwilioVoiceController : TwilioController }; await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post); } await HookEmitter.Emit(_services, async hook => @@ -109,7 +109,7 @@ public class TwilioVoiceController : TwilioController /// /// [ValidateRequest] - [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] + [HttpPost("twilio/voice/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { var twilio = _services.GetRequiredService(); @@ -142,7 +142,7 @@ public class TwilioVoiceController : TwilioController await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); await HookEmitter.Emit(_services, async hook => { @@ -173,7 +173,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { SpeechPaths = new List(), - CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{request.SeqNum}?{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", + CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", ActionOnEmptyResult = true }; @@ -203,7 +203,7 @@ public class TwilioVoiceController : TwilioController /// /// [ValidateRequest] - [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] + [HttpPost("twilio/voice/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { var nextSeqNum = request.SeqNum + 1; @@ -276,7 +276,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { SpeechPaths = speechPaths, - CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; @@ -315,7 +315,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { SpeechPaths = instructions, - CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; @@ -361,7 +361,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], - CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{nextSeqNum}?{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}", ActionOnEmptyResult = true, Hints = reply.Hints }; @@ -388,7 +388,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { ActionOnEmptyResult = true, - CallbackPath = $"twilio/voice/{conversationId}/receive/1", + CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}", SpeechPaths = new List { $"twilio/voice/speeches/{conversationId}/intial.mp3" diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs new file mode 100644 index 00000000..d35a2a35 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -0,0 +1,8 @@ +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Interfaces; + +public interface ITwilioCallStatusHook +{ + Task OnVoicemailLeft(string conversationId); +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index a17b344f..25e3ba0d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -7,7 +7,7 @@ public class ConversationalVoiceRequest : VoiceRequest [FromQuery(Name = "agent-id")] public string AgentId { get; set; } = string.Empty; - [FromRoute] + [FromQuery(Name = "conversation-id")] public string ConversationId { get; set; } = string.Empty; [FromRoute] @@ -18,7 +18,27 @@ public class ConversationalVoiceRequest : VoiceRequest public string? AIResponseErrorMessage { get; set; } = string.Empty; public string Intent { get; set; } = string.Empty; + + [FromQuery(Name = "init-audio-file")] public string? InitAudioFile { get; set; } public List States { get; set; } = []; + + [FromForm] + public string? CallbackSource { get; set; } + + /// + /// machine_start + /// + [FromForm] + public string? AnsweredBy { get; set; } + + [FromForm] + public int MachineDetectionDuration { get; set; } + + [FromForm] + public int Duration { get; set; } + + [FromForm] + public int CallDuration { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index df97cd44..cb707474 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -66,9 +66,12 @@ public class OutboundPhoneCallFn : IFunctionCallback // Make outbound call var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={newConversationId}&init_audio_file={fileName}"), + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"), to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber)); + from: new PhoneNumber(_twilioSetting.PhoneNumber), + statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"), + // https://www.twilio.com/docs/voice/answering-machine-detection + machineDetection: "Enable"); var convService = _services.GetRequiredService(); var routing = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 6f0364bb..b17f287a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -199,6 +199,7 @@ public class TwilioService } } } + var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}"); From f4cc8f41fa347366064b4dc2e3f4ca92849952b5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 13 Mar 2025 15:36:14 -0500 Subject: [PATCH 76/84] refine update conv tags --- .../Conversations/ConversationHookBase.cs | 2 +- .../FileRepository/FileRepository.Conversation.cs | 3 ++- .../Repository/MongoRepository.Conversation.cs | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index 0f115d5d..a84c8693 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -79,6 +79,6 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; - public Task OnUserDisconnected(Conversation conversation) + public virtual Task OnUserDisconnected(Conversation conversation) => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 4e5998fb..647282a4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -171,7 +171,8 @@ public partial class FileRepository var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); - var tags = conv.Tags.Concat(toAddTags).ToList(); + var tags = conv.Tags ?? []; + tags = tags.Concat(toAddTags).Distinct().ToList(); conv.Tags = tags.Where(x => !toDeleteTags.Contains(x, StringComparer.OrdinalIgnoreCase)).ToList(); conv.UpdatedTime = DateTime.UtcNow; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 393846bf..ebae075b 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -142,10 +142,12 @@ public partial class MongoRepository var conv = _dc.Conversations.Find(filter).FirstOrDefault(); if (conv == null) return false; - + var tags = conv.Tags ?? []; + tags = tags.Concat(toAddTags).Distinct().ToList(); + tags = tags.Where(x => !toDeleteTags.Contains(x, StringComparer.OrdinalIgnoreCase)).ToList(); var update = Builders.Update - .Set(x => x.Tags, tags ?? new()) + .Set(x => x.Tags, tags) .Set(x => x.UpdatedTime, DateTime.UtcNow); var res = _dc.Conversations.UpdateOne(filter, update); From a49b75f4953376dade9243110fc0a24df236ab4d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 14 Mar 2025 13:59:13 -0500 Subject: [PATCH 77/84] Add Twilio settings. --- .../Conversations/ConversationHookBase.cs | 3 -- .../Conversations/IConversationHook.cs | 7 --- .../BotSharp.Core.Realtime.csproj | 6 ++- .../Services/RealtimeHub.cs | 8 +--- .../Controllers/TwilioStreamController.cs | 22 +-------- .../Controllers/TwilioVoiceController.cs | 48 ++++++++++++++----- .../Interfaces/ITwilioCallStatusHook.cs | 4 +- .../Models/AssistantMessage.cs | 2 +- .../Functions/OutboundPhoneCallFn.cs | 47 ++++++++++++++---- .../Services/TwilioService.cs | 15 +++++- .../Settings/TwilioSetting.cs | 28 +++++++++-- 11 files changed, 125 insertions(+), 65 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index a84c8693..da87646e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -78,7 +78,4 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; - - public virtual Task OnUserDisconnected(Conversation conversation) - => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index f99078ea..c764f391 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -25,13 +25,6 @@ public interface IConversationHook /// Task OnUserAgentConnectedInitially(Conversation conversation); - /// - /// Triggered when user disconnects with agent. - /// - /// - /// - Task OnUserDisconnected(Conversation conversation); - /// /// Triggered once for every new conversation. /// diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index c004dc50..25218b08 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -1,7 +1,11 @@  - net8.0 + $(TargetFramework) + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(SolutionDir)packages enable enable diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 76302262..8adf63f0 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,7 +1,3 @@ -using BotSharp.Abstraction.Utilities; -using BotSharp.Core.Infrastructures; -using Microsoft.AspNetCore.Cors.Infrastructure; - namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub @@ -257,9 +253,7 @@ public class RealtimeHub : IRealtimeHub private async Task HandleUserDisconnected() { - var convService = _services.GetRequiredService(); - var conversation = await convService.GetConversation(_conn.ConversationId); - await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation)); + } private async Task SendEventToUser(WebSocket webSocket, object message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e69f79f6..fb35c25c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -42,7 +42,7 @@ public class TwilioStreamController : TwilioController request.InitAudioFile != null) { response = new VoiceResponse(); - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}")); + response.Play(new Uri(request.InitAudioFile)); return TwiML(response); } @@ -54,7 +54,7 @@ public class TwilioStreamController : TwilioController if (request.InitAudioFile != null) { - instruction.SpeechPaths.Add(request.InitAudioFile); + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); } await HookEmitter.Emit(_services, async hook => @@ -82,24 +82,6 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - [ValidateRequest] - [HttpPost("twilio/stream/status")] - public async Task StreamConversationStatus(ConversationalVoiceRequest request) - { - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api" && - request.InitAudioFile != null && - request.CallStatus == "completed") - { - // voicemail - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailLeft(request.ConversationId); - }); - } - return Ok(); - } - private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 6507a105..0193cae9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,13 +1,11 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Repositories; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using System.ComponentModel.DataAnnotations; using Twilio.Http; namespace BotSharp.Plugin.Twilio.Controllers; @@ -382,21 +380,21 @@ public class TwilioVoiceController : TwilioController } [ValidateRequest] - [HttpPost("twilio/voice/init-call")] - public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) + [HttpPost("twilio/voice/init-outbound-call")] + public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request) { var instruction = new ConversationalVoiceResponse { ActionOnEmptyResult = true, - CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}", - SpeechPaths = new List - { - $"twilio/voice/speeches/{conversationId}/intial.mp3" - } + CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}", }; - string tag = $"twilio:{Request.Form["AnsweredBy"]}"; - var db = _services.GetRequiredService(); - db.AppendConversationTags(conversationId, new List { tag }); + + if (request.InitAudioFile != null) + { + instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}"; + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); + } + var twilio = _services.GetRequiredService(); var response = twilio.ReturnNoninterruptedInstructions(instruction); return TwiML(response); @@ -415,6 +413,32 @@ public class TwilioVoiceController : TwilioController return result; } + [ValidateRequest] + [HttpPost("twilio/voice/status")] + public async Task PhoneCallStatus(ConversationalVoiceRequest request) + { + if (request.CallStatus == "completed") + { + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + // voicemail + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailLeft(request); + }); + } + else + { + // phone call completed + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); + } + } + + return Ok(); + } + private Dictionary ParseStates(List states) { var result = new Dictionary(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index d35a2a35..904f16a1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -1,8 +1,10 @@ +using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; public interface ITwilioCallStatusHook { - Task OnVoicemailLeft(string conversationId); + Task OnVoicemailLeft(ConversationalVoiceRequest request); + Task OnUserDisconnected(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 2587fed9..547b09d9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.Twilio.Models public bool HumanIntervationNeeded { get; set; } public string Content { get; set; } public string MessageId { get; set; } - public string SpeechFileName { get; set; } + public string? SpeechFileName { get; set; } public string Hints { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index cb707474..488ad79b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -1,8 +1,11 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; using Twilio.Types; @@ -58,20 +61,48 @@ public class OutboundPhoneCallFn : IFunctionCallback var newConversationId = Guid.NewGuid().ToString(); states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + var processUrl = $"{_twilioSetting.CallbackHost}/twilio"; + var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}"; + // Generate initial assistant audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(newConversationId, fileName, data); + string initAudioUrl = null; + if (!string.IsNullOrEmpty(args.InitialMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + initAudioUrl = "intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data); + + statusUrl += $"&init-audio-file={initAudioUrl}"; + } + + // Set up process URL streaming or synchronous + if (_twilioSetting.StreamingEnabled) + { + processUrl += "/stream"; + } + else + { + var sessionManager = _services.GetRequiredService(); + await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage + { + Content = args.InitialMessage, + SpeechFileName = initAudioUrl + }); + + processUrl += "/voice/init-outbound-call"; + } + + processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}"; // Make outbound call var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"), + url: new Uri(processUrl), to: new PhoneNumber(args.PhoneNumber), from: new PhoneNumber(_twilioSetting.PhoneNumber), - statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"), + statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection - machineDetection: "Enable"); + machineDetection: _twilioSetting.MachineDetection); var convService = _services.GetRequiredService(); var routing = _services.GetRequiredService(); @@ -80,7 +111,7 @@ public class OutboundPhoneCallFn : IFunctionCallback await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); - message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]"; + message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]"; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index b17f287a..d64c34c2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -109,7 +109,14 @@ public class TwilioService { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } } } var gather = new Gather() @@ -193,9 +200,13 @@ public class TwilioService { response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } else { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 4c65481f..5d34299f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -2,14 +2,36 @@ namespace BotSharp.Plugin.Twilio.Settings; public class TwilioSetting { - public string PhoneNumber { get; set; } + /// + /// Outbound phone number + /// + public string? PhoneNumber { get; set; } + + /// + /// Enable streaming for outbound phone call + /// + public bool StreamingEnabled { get; set; } = false; public string AccountSID { get; set; } public string AuthToken { get; set; } public string AppSID { get; set; } public string ApiKeySID { get; set; } public string ApiSecret { get; set; } public string CallbackHost { get; set; } - public string AgentId { get; set; } - public string CsrAgentNumber { get; set; } + + /// + /// Default Agent Id to handle inbound phone call + /// + public string? AgentId { get; set; } + + /// + /// Human agent phone number if AI can't handle the call + /// + public string? CsrAgentNumber { get; set; } + public int MaxGatherAttempts { get; set; } = 4; + + public string? MachineDetection { get; set; } + + public bool RecordingEnabled { get; set; } = false; + public bool RecordingTranscribe { get; set; } = false; } From 4cde28f09509686e9e6a4a00bb8db59f11670d7b Mon Sep 17 00:00:00 2001 From: vguruparan Date: Fri, 14 Mar 2025 14:43:23 -0500 Subject: [PATCH 78/84] Add customization for web driver. --- .../Browsing/Models/PageActionArgs.cs | 7 ++++++- .../PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs | 10 ++++++++-- .../UtilFunctions/UtilWebGoToPageFn.cs | 6 ++++-- .../functions/util-web-go_to_page.json | 8 ++++++++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index fe575462..1abf7994 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -16,7 +16,10 @@ public class PageActionArgs /// /// This value has to be set to true if you want to get the page XHR/ Fetch responses /// - public bool OpenNewTab { get; set; } = false; + [JsonPropertyName("open_new_tab")] + public bool OpenNewTab { get; set; } = true; + [JsonPropertyName("open_blank_page")] + public bool OpenBlankPage { get; set; } = true; public bool EnableResponseCallback { get; set; } = false; @@ -47,4 +50,6 @@ public class PageActionArgs public int WaitTime { get; set; } public bool ReadInnerHTMLAsBody { get; set; } = false; + [JsonPropertyName("keep_browser_open")] + public bool KeepBrowserOpen { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 7327bb40..922e8f58 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -15,7 +15,14 @@ public partial class PlaywrightWebDriver if (page != null) { - if (page.Url != "about:blank") + if (!args.OpenBlankPage) + { + if (!_instance.Pages[message.ContextId].Contains(page)) + { + _instance.Pages[message.ContextId].Add(page); + } + } + if (args.OpenBlankPage && page.Url != "about:blank") { await page.EvaluateAsync(@"() => { window.open('', '_blank'); @@ -48,7 +55,6 @@ public partial class PlaywrightWebDriver // Active current tab await page.BringToFrontAsync(); - var response = await page.GotoAsync(args.Url, new PageGotoOptions { Timeout = args.Timeout > 0 ? args.Timeout : 30000 diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index 0c5507bf..7a5d8f89 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -27,7 +27,6 @@ public class UtilWebGoToPageFn : IFunctionCallback args.Timeout = _webDriver.DefaultTimeout; args.WaitForNetworkIdle = false; args.WaitTime = _webDriver.DefaultWaitTime; - args.OpenNewTab = true; var conv = _services.GetRequiredService(); @@ -38,7 +37,10 @@ public class UtilWebGoToPageFn : IFunctionCallback MessageId = message.MessageId, ContextId = message.CurrentAgentId, }; - await browser.CloseCurrentPage(msg); + if (!args.KeepBrowserOpen) + { + await browser.CloseCurrentPage(msg); + } var result = await browser.GoToPage(msg, args); if (!result.IsSuccess) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json index 65f088bd..4f6b21fe 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json @@ -7,6 +7,14 @@ "url": { "type": "string", "description": "Web URL" + }, + "open_new_tab": { + "type": "boolean", + "description": "Open new tab" + }, + "open_blank_page": { + "type": "boolean", + "description": "Open blank page" } }, "required": [ "url" ] From addb1b1bba1052a449e6177bf79e3695f9d927ea Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 14 Mar 2025 15:08:34 -0500 Subject: [PATCH 79/84] Outbound call phone_recording_url --- .../Controllers/TwilioRecordController.cs | 44 +++++++++++++++++++ .../Controllers/TwilioStreamController.cs | 1 + .../Controllers/TwilioVoiceController.cs | 18 +++++++- .../Interfaces/ITwilioCallStatusHook.cs | 1 + .../Models/ConversationalVoiceResponse.cs | 1 + .../Functions/OutboundPhoneCallFn.cs | 5 ++- .../Services/TwilioService.cs | 18 ++++---- .../Settings/TwilioSetting.cs | 1 - 8 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs new file mode 100644 index 00000000..65b2a5ad --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -0,0 +1,44 @@ +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioRecordController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TwilioRecordController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/record/status")] + public async Task PhoneRecordingStatus(ConversationalVoiceRequest request) + { + if (request.RecordingStatus == "completed") + { + _logger.LogInformation($"Recording completed for {request.CallSid}, the record URL is {request.RecordingUrl}"); + + // Set the recording URL to the conversation state + var convService = _services.GetRequiredService(); + convService.SetConversationId(request.ConversationId, new List + { + new("phone_recording_url", request.RecordingUrl) + }); + convService.SaveStates(); + + // recording completed + await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request)); + } + + return Ok(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index fb35c25c..8fea9bb4 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -48,6 +48,7 @@ public class TwilioStreamController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = [], ActionOnEmptyResult = true }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 0193cae9..18225d09 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -52,6 +52,7 @@ public class TwilioVoiceController : TwilioController VoiceResponse response = null; var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = ["twilio/welcome.mp3"], ActionOnEmptyResult = true }; @@ -170,6 +171,7 @@ public class TwilioVoiceController : TwilioController { var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = new List(), CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", ActionOnEmptyResult = true @@ -273,6 +275,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = speechPaths, CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true @@ -312,6 +315,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = instructions, CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true @@ -358,6 +362,7 @@ public class TwilioVoiceController : TwilioController { var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}", ActionOnEmptyResult = true, @@ -383,8 +388,19 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/init-outbound-call")] public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request) { + VoiceResponse response = default!; + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + response = new VoiceResponse(); + response.Play(new Uri(request.InitAudioFile)); + return TwiML(response); + } + var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, ActionOnEmptyResult = true, CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}", }; @@ -396,7 +412,7 @@ public class TwilioVoiceController : TwilioController } var twilio = _services.GetRequiredService(); - var response = twilio.ReturnNoninterruptedInstructions(instruction); + response = twilio.ReturnNoninterruptedInstructions(instruction); return TwiML(response); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index 904f16a1..c9ed4710 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -7,4 +7,5 @@ public interface ITwilioCallStatusHook { Task OnVoicemailLeft(ConversationalVoiceRequest request); Task OnUserDisconnected(ConversationalVoiceRequest request); + Task OnRecordingCompleted(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs index ea8072d6..7870f961 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceResponse { + public string ConversationId { get; set; } = null!; public List SpeechPaths { get; set; } = []; public string CallbackPath { get; set; } public bool ActionOnEmptyResult { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 488ad79b..3bcd0260 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -63,6 +63,7 @@ public class OutboundPhoneCallFn : IFunctionCallback var processUrl = $"{_twilioSetting.CallbackHost}/twilio"; var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}"; + var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}"; // Generate initial assistant audio string initAudioUrl = null; @@ -102,7 +103,9 @@ public class OutboundPhoneCallFn : IFunctionCallback from: new PhoneNumber(_twilioSetting.PhoneNumber), statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection - machineDetection: _twilioSetting.MachineDetection); + machineDetection: _twilioSetting.MachineDetection, + record: _twilioSetting.RecordingEnabled, + recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}"); var convService = _services.GetRequiredService(); var routing = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index d64c34c2..8ef9f8e6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; -using Twilio.TwiML.Messaging; using Token = Twilio.Jwt.AccessToken.Token; namespace BotSharp.Plugin.Twilio.Services; @@ -101,13 +100,13 @@ public class TwilioService return response; } - public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse) { var response = new VoiceResponse(); - response.Pause(2); - if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + + if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any()) { - foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + foreach (var speechPath in voiceResponse.SpeechPaths) { if (speechPath.StartsWith(_settings.CallbackHost)) { @@ -119,6 +118,7 @@ public class TwilioService } } } + var gather = new Gather() { Input = new List() @@ -126,14 +126,15 @@ public class TwilioService Gather.InputEnum.Speech, Gather.InputEnum.Dtmf }, - Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"), + Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"), Enhanced = true, SpeechModel = Gather.SpeechModelEnum.PhoneCall, SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3", - Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3, - ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult + Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3, + ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult, }; response.Append(gather); + return response; } @@ -192,6 +193,7 @@ public class TwilioService public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 5d34299f..81629a9e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -33,5 +33,4 @@ public class TwilioSetting public string? MachineDetection { get; set; } public bool RecordingEnabled { get; set; } = false; - public bool RecordingTranscribe { get; set; } = false; } From ace2e906f04923e7d8da8c431db473b0b2100dc7 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 14 Mar 2025 16:44:19 -0500 Subject: [PATCH 80/84] util-twilio-text_message --- .../BotSharp.Plugin.Twilio.csproj | 3 ++ .../Functions/HangupPhoneCallFn.cs | 1 - .../Functions/TextMessageFn.cs | 49 +++++++++++++++++++ .../OutboundPhoneCallHandlerUtilityHook.cs | 4 +- .../Settings/TwilioSetting.cs | 2 + .../functions/util-twilio-text_message.json | 18 +++++++ 6 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 72714388..5b83d77d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -13,6 +13,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index cd755306..5a7eb3f3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,5 +1,4 @@ using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; -using Microsoft.VisualBasic; using Twilio.Rest.Api.V2010.Account; using Task = System.Threading.Tasks.Task; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs new file mode 100644 index 00000000..c7c6fc88 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs @@ -0,0 +1,49 @@ +using BotSharp.Abstraction.Options; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class TextMessageFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-text_message"; + public string Indication => "Sending text message"; + + public TextMessageFn( + IServiceProvider services, + ILogger logger, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + // Send the message + var twilioMessage = MessageResource.Create( + to: args.PhoneNumber, + from: _twilioSetting.MessagingShortCode, + body: args.InitialMessage + ); + + if (twilioMessage.Status == MessageResource.StatusEnum.Queued) + { + message.Content = $"Queued message to {args.PhoneNumber}: {args.InitialMessage} [MESSAGING SID: {twilioMessage.Sid}]"; + message.StopCompletion = true; + } + else + { + message.Content = twilioMessage.ErrorMessage; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index 692610ff..fa0b3179 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -8,6 +8,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook private static string PREFIX = "util-twilio-"; private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; + public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; public void AddUtilities(List utilities) { @@ -17,7 +18,8 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook Functions = [ new($"{OUTBOUND_PHONE_CALL_FN}"), - new($"{HANGUP_PHONE_CALL_FN}") + new($"{HANGUP_PHONE_CALL_FN}"), + new($"{TEXT_MESSAGE_FN}") ], Templates = [ diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 81629a9e..2de42ec8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -18,6 +18,8 @@ public class TwilioSetting public string ApiSecret { get; set; } public string CallbackHost { get; set; } + public string? MessagingShortCode { get; set; } + /// /// Default Agent Id to handle inbound phone call /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json new file mode 100644 index 00000000..a4340410 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json @@ -0,0 +1,18 @@ +{ + "name": "util-twilio-text_message", + "description": "If the user wants to send SMS message to a phone.", + "parameters": { + "type": "object", + "properties": { + "phone_number": { + "type": "string", + "description": "The phone number which will receive message. It needs to be a valid phone number starting with +1." + }, + "initial_message": { + "type": "string", + "description": "The initial message which will be sent." + } + }, + "required": [ "phone_number", "initial_message" ] + } +} \ No newline at end of file From 3c23e58351446059c666d0895ee28d646ef1de7b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 14 Mar 2025 21:06:40 -0500 Subject: [PATCH 81/84] Optimize phone hang-up --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Realtime/Models/ModelTurnDetection.cs | 10 +++++++ .../Realtime/Models/RealtimeModelSettings.cs | 8 ++++++ .../BotSharp.Core.Realtime/RealtimePlugin.cs | 7 +++++ .../Services/RealtimeHub.cs | 23 ++++++++-------- .../Models/Realtime/RealtimeSessionBody.cs | 3 +++ .../Realtime/RealTimeCompletionProvider.cs | 20 +++++++------- .../BotSharp.Plugin.Twilio.csproj | 10 +++---- .../Controllers/TwilioVoiceController.cs | 9 +++++++ .../Functions/HangupPhoneCallFn.cs | 26 ++++++++++++------- .../Functions/OutboundPhoneCallFn.cs | 16 +++++++----- .../LlmContexts/HangupPhoneCallArgs.cs | 4 +-- .../util-twilio-hangup_phone_call.json | 12 ++++++--- .../util-twilio-hangup_phone_call.fn.liquid | 2 +- 14 files changed, 99 insertions(+), 53 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 075d7291..68a96dfa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); + Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs new file mode 100644 index 00000000..a65a8985 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class ModelTurnDetection +{ + public int PrefixPadding { get; set; } = 300; + + public int SilenceDuration { get; set; } = 800; + + public float Threshold { get; set; } = 0.8f; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs new file mode 100644 index 00000000..8755be9d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeModelSettings +{ + public float Temperature { get; set; } = 0.6f; + public int MaxResponseOutputTokens { get; set; } = 512; + public ModelTurnDetection TurnDetection { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs index a45325de..fa9ee268 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; using BotSharp.Core.Realtime.Hooks; using BotSharp.Core.Realtime.Services; using Microsoft.Extensions.Configuration; @@ -14,6 +15,12 @@ public class RealtimePlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("RealtimeModel"); + }); + services.AddScoped(); services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 8adf63f0..22f331ee 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -98,15 +98,18 @@ public class RealtimeHub : IRealtimeHub await _completer.Connect(_conn, onModelReady: async () => { - if (states.ContainsState("init_audio_file")) - { - await _completer.UpdateSession(_conn, turnDetection: true); - } - else - { - // Control initial session, prevent initial response interruption - await _completer.UpdateSession(_conn, turnDetection: false); + // Not TriggerModelInference, waiting for user utter. + await _completer.UpdateSession(_conn); + // Push dialogs into model context + foreach (var message in dialogs) + { + await _completer.InsertConversationItem(message); + } + + // Trigger model inference if there is no audio file in the conversation + if (!states.ContainsState("init_audio_file")) + { if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); @@ -115,10 +118,6 @@ public class RealtimeHub : IRealtimeHub { await _completer.TriggerModelInference("Reply based on the conversation context."); } - - // Start turn detection - await Task.Delay(1000 * 8); - await _completer.UpdateSession(_conn, turnDetection: true); } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index f6a6322c..2f7f79c6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -52,6 +52,9 @@ public class RealtimeSessionBody public class RealtimeSessionTurnDetection { + [JsonPropertyName("interrupt_response")] + public bool InterruptResponse { get; set; } = true; + /// /// Milliseconds /// diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 47a9030d..2510c852 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -290,7 +290,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) + public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -317,6 +317,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var realitmeModelSettings = _services.GetRequiredService(); + var sessionUpdate = new { type = "session.update", @@ -335,22 +337,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), - MaxResponseOutputTokens = 512, + Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f), + MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.9f, - PrefixPadding = 300, - SilenceDuration = 800 + InterruptResponse = interruptResponse, + Threshold = realitmeModelSettings.TurnDetection.Threshold, + PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding, + SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration } } }; - if (!turnDetection) - { - sessionUpdate.session.TurnDetection = null; - } - await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 5b83d77d..8fe2ecce 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -19,12 +19,6 @@ PreserveNewest - - PreserveNewest - - - PreserveNewest - @@ -39,4 +33,8 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 18225d09..079e13c5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -429,6 +429,15 @@ public class TwilioVoiceController : TwilioController return result; } + [ValidateRequest] + [HttpPost("twilio/voice/hang-up")] + public async Task Hangup(ConversationalVoiceRequest request) + { + var twilio = _services.GetRequiredService(); + var response = twilio.HangUp("twilio/bye.mp3"); + return TwiML(response); + } + [ValidateRequest] [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index 5a7eb3f3..9fa7d03c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,6 +1,6 @@ +using BotSharp.Abstraction.Routing; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -8,21 +8,27 @@ public class HangupPhoneCallFn : IFunctionCallback { private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; public string Name => "util-twilio-hangup_phone_call"; public string Indication => "Hangup"; public HangupPhoneCallFn( IServiceProvider services, - ILogger logger) + ILogger logger, + TwilioSetting twilioSetting) { _services = services; _logger = logger; + _twilioSetting = twilioSetting; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; var states = _services.GetRequiredService(); var callSid = states.GetState("twilio_call_sid"); @@ -33,20 +39,20 @@ public class HangupPhoneCallFn : IFunctionCallback return false; } - message.Content = args.GoodbyeMessage; - - _ = Task.Run(async () => + if (args.AnythingElseToHelp) + { + message.Content = "Tell me how I can help."; + } + else { - await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400); - // Have to find the SID by the phone number var call = CallResource.Update( - status: CallResource.UpdateStatusEnum.Completed, + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"), pathSid: callSid ); - message.Content = "The call has been ended."; + message.Content = "The call is ending."; message.StopCompletion = true; - }); + } return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 3bcd0260..1e10a135 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -66,15 +66,15 @@ public class OutboundPhoneCallFn : IFunctionCallback var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}"; // Generate initial assistant audio - string initAudioUrl = null; + string initAudioFile = null; if (!string.IsNullOrEmpty(args.InitialMessage)) { var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - initAudioUrl = "intial.mp3"; - fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data); + initAudioFile = "intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data); - statusUrl += $"&init-audio-file={initAudioUrl}"; + statusUrl += $"&init-audio-file={initAudioFile}"; } // Set up process URL streaming or synchronous @@ -88,13 +88,17 @@ public class OutboundPhoneCallFn : IFunctionCallback await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage { Content = args.InitialMessage, - SpeechFileName = initAudioUrl + SpeechFileName = initAudioFile }); processUrl += "/voice/init-outbound-call"; } - processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}"; + processUrl += $"?conversation-id={newConversationId}"; + if (!string.IsNullOrEmpty(initAudioFile)) + { + processUrl += $"&init-audio-file={initAudioFile}"; + } // Make outbound call var call = await CallResource.CreateAsync( diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs index ea2075d9..efd8114a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -4,6 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class HangupPhoneCallArgs { - [JsonPropertyName("goodbye_message")] - public string? GoodbyeMessage { get; set; } + [JsonPropertyName("anything_else_to_help")] + public bool AnythingElseToHelp { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 773fac23..be5f3ca6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -1,15 +1,19 @@ { "name": "util-twilio-hangup_phone_call", - "description": "Call this function if the user wants to end the phone call", + "description": "Call this function if the user wants to end the phone call or conversation", "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { - "goodbye_message": { + "reason": { "type": "string", - "description": "A polite closing statement for ending a conversation." + "description": "The reason why user wants to end the phone call." + }, + "anything_else_to_help": { + "type": "boolean", + "description": "Check if user has anything else to help." } }, - "required": [ "goodbye_message" ] + "required": [ "reason", "anything_else_to_help" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid index 4a6b5034..5e89cbd2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -1,3 +1,3 @@ {% if channel == 'phone' %} -** Please call util-twilio-hangup_phone_call if user wants to end the phone call. +** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call. {% endif %} \ No newline at end of file From 292da2aee7244619be2d13de85d4e478c3cb6663 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 14 Mar 2025 21:08:29 -0500 Subject: [PATCH 82/84] Change template default to 0.8f --- .../Realtime/Models/RealtimeModelSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index 8755be9d..30b6cfb8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { - public float Temperature { get; set; } = 0.6f; + public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; public ModelTurnDetection TurnDetection { get; set; } = new(); } From a1c73d045a819ff1468a4bcbea8b1a139746782c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 15 Mar 2025 09:29:46 -0500 Subject: [PATCH 83/84] Fix Push dialogs into model context --- .../Services/RealtimeHub.cs | 18 +++++++++++------- .../Controllers/TwilioStreamController.cs | 2 +- .../Services/TwilioService.cs | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 22f331ee..55378b01 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -99,13 +99,7 @@ public class RealtimeHub : IRealtimeHub onModelReady: async () => { // Not TriggerModelInference, waiting for user utter. - await _completer.UpdateSession(_conn); - - // Push dialogs into model context - foreach (var message in dialogs) - { - await _completer.InsertConversationItem(message); - } + var instruction = await _completer.UpdateSession(_conn); // Trigger model inference if there is no audio file in the conversation if (!states.ContainsState("init_audio_file")) @@ -119,6 +113,16 @@ public class RealtimeHub : IRealtimeHub await _completer.TriggerModelInference("Reply based on the conversation context."); } } + else + { + // Push dialogs into model context + foreach (var message in dialogs) + { + await _completer.InsertConversationItem(message); + } + + await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement."); + } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 8fea9bb4..6960f489 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -55,7 +55,7 @@ public class TwilioStreamController : TwilioController if (request.InitAudioFile != null) { - instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); + instruction.SpeechPaths.Add(request.InitAudioFile); } await HookEmitter.Emit(_services, async hook => diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 8ef9f8e6..082a0721 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -208,7 +208,7 @@ public class TwilioService } else { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}")); } } } From cc6f4d609cd752278cf0b29a8118fd4b71a2d459 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 16 Mar 2025 09:45:18 -0500 Subject: [PATCH 84/84] Release realtime v4.1 --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 6716a3c7..3cb973b5 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ net8.0 12.0 - 4.0.0 + 4.1.0 true false