From 8cbd82a0ff7ac3c220ffd826a25059715ba0c429 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 10:35:03 -0500 Subject: [PATCH 01/13] optimize twilio --- .../Realtime/IRealtimeHook.cs | 4 + .../Realtime/Models/TranscriptionData.cs | 6 + .../Services/RealtimeHub.cs | 36 +--- .../Agents/Services/AgentService.GetAgents.cs | 6 +- .../Controllers/TwilioInboundController.cs | 186 ++++++++++++++++++ .../Controllers/TwilioRecordController.cs | 31 --- .../Controllers/TwilioStreamController.cs | 125 ------------ .../Controllers/TwilioTranscribeController.cs | 58 ++++++ .../Controllers/TwilioVoiceController.cs | 15 +- .../Interfaces/ITwilioCallStatusHook.cs | 3 +- .../Models/ConversationalVoiceRequest.cs | 5 - .../Functions/OutboundPhoneCallFn.cs | 8 +- .../Services/TwilioService.cs | 11 +- .../Settings/TwilioSetting.cs | 4 - .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 2 - .../Stream => }/TwilioStreamMiddleware.cs | 2 +- 16 files changed, 293 insertions(+), 209 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs rename src/Plugins/BotSharp.Plugin.Twilio/{Services/Stream => }/TwilioStreamMiddleware.cs (99%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs index 2ad8f1df..c7409f59 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -1,6 +1,10 @@ +using BotSharp.Abstraction.MLTasks; + namespace BotSharp.Abstraction.Realtime; public interface IRealtimeHook { + Task OnModeReady(Agent agent, IRealTimeCompletion completer); string[] OnModelTranscriptPrompt(Agent agent); + Task OnTranscribeCompleted(RoleDialogModel message, TranscriptionData data); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs new file mode 100644 index 00000000..c76be9cd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs @@ -0,0 +1,6 @@ +public class TranscriptionData +{ + public string Transcript { get; set; } = null!; + public float Confidence { get; set; } + public string Language { get; set; } = null!; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index b334b873..4879eb54 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,5 @@ +using BotSharp.Core.Infrastructures; + namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub @@ -70,8 +72,8 @@ public class RealtimeHub : IRealtimeHub _conn.CurrentAgentId = agent.Id; // Set model - var model = agent.LlmConfig.Model; - if (!model.Contains("-realtime-")) + var model = "gpt-4o-mini-realtime"; + if (agent.Profiles.Contains("realtime")) { var llmProviderService = _services.GetRequiredService(); model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name; @@ -85,14 +87,14 @@ public class RealtimeHub : IRealtimeHub var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); - if (dialogs.Count == 0) + /*if (dialogs.Count == 0) { dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); storage.Append(_conn.ConversationId, dialogs.First()); - } + }*/ routing.Context.SetDialogs(dialogs); - routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); + // routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); @@ -102,29 +104,7 @@ public class RealtimeHub : IRealtimeHub // Not TriggerModelInference, waiting for user utter. var instruction = await _completer.UpdateSession(_conn); - // Trigger model inference if there is no audio file in the conversation - if (!states.ContainsState("init_audio_file")) - { - if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) - { - await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); - } - else - { - await _completer.TriggerModelInference("Reply based on the conversation context."); - } - } - else - { - // Append dialogs into model context - var history = "[CONVERSATION HISTORY]\r\n"; - foreach (var message in dialogs) - { - history += $"{message.Role}: {message.Content}\r\n"; - } - - await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement."); - } + await HookEmitter.Emit(_services, async hook => await hook.OnModeReady(agent, _completer)); }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => { diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index 092273fa..4cac4c86 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Agents.Services; public partial class AgentService { #if !DEBUG - [SharpCache(10, perInstanceCache: true)] + [SharpCache(10)] #endif public async Task> GetAgents(AgentFilter filter) { @@ -28,7 +28,7 @@ public partial class AgentService } #if !DEBUG - [SharpCache(10, perInstanceCache: true)] + [SharpCache(10)] #endif public async Task> GetAgentOptions(List? agentIds) { @@ -40,7 +40,7 @@ public partial class AgentService } #if !DEBUG - [SharpCache(10, perInstanceCache: true)] + [SharpCache(10)] #endif public async Task GetAgent(string id) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs new file mode 100644 index 00000000..a3a11c8c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -0,0 +1,186 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Twilio.Http; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioInboundController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly ILogger _logger; + + public TwilioInboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _context = context; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/inbound")] + public async Task InitiateStreamConversation(ConversationalVoiceRequest request) + { + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + + var twilio = _services.GetRequiredService(); + VoiceResponse response = default!; + + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + SpeechPaths = [], + ActionOnEmptyResult = true, + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + + // Load agent profile + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(request.AgentId); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreating(request, instruction); + }); + + request.ConversationId = await InitConversation(request, agent); + instruction.AgentId = request.AgentId; + instruction.ConversationId = request.ConversationId; + + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api") + { + response = new VoiceResponse(); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailStarting(request); + }); + + var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); + response.Play(new Uri(url)); + } + else + { + if (agent.Profiles.Contains("realtime")) + { + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent); + } + else + { + if (string.IsNullOrWhiteSpace(request.Intent)) + { + instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"; + response = twilio.ReturnNoninterruptedInstructions(instruction); + } + else + { + int seqNum = 0; + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); + var callerMessage = new CallerMessage() + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + SeqNumber = seqNum, + Content = request.Intent, + From = request.From, + States = ParseStates(request.States) + }; + await messageQueue.EnqueueAsync(callerMessage); + response = new VoiceResponse(); + // delay 3 seconds to wait for the first message reply and caller is listening dudu sound + await Task.Delay(1000 * 3); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post); + } + } + } + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreated(request); + }); + + return TwiML(response); + } + + protected Dictionary ParseStates(List states) + { + var result = new Dictionary(); + if (states is null || !states.Any()) + { + return result; + } + foreach (var kvp in states) + { + var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) + { + result.Add(parts[0], parts[1]); + } + } + return result; + } + + private async Task InitConversation(ConversationalVoiceRequest request, Agent agent) + { + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(request.ConversationId); + if (conversation == null) + { + var conv = new Conversation + { + AgentId = request.AgentId, + Channel = ConversationChannel.Phone, + ChannelId = request.CallSid, + Title = $"Incoming phone call from {request.From}", + Tags = [], + }; + + conversation = await convService.NewConversation(conv); + } + + var states = new List + { + new("channel", ConversationChannel.Phone), + new("calling_phone", request.From), + new("phone_direction", request.Direction), + new("twilio_call_sid", request.CallSid), + }; + + // Enable lazy routing mode to optimize realtime experience + if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) + { + states.Add(new(StateConst.ROUTING_MODE, "lazy")); + } + + if (request.InitAudioFile != null) + { + states.Add(new("init_audio_file", request.InitAudioFile)); + } + + convService.SetConversationId(conversation.Id, states); + convService.SaveStates(); + + return conversation.Id; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs index e8f055fe..65b2a5ad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -1,12 +1,8 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; -using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; -using System.IO; namespace BotSharp.Plugin.Twilio.Controllers; @@ -45,31 +41,4 @@ public class TwilioRecordController : TwilioController return Ok(); } - - [ValidateRequest] - [HttpPost("twilio/record/transcribe")] - public async Task PhoneRecordingTranscribe(ConversationalVoiceRequest request) - { - if (request.Final == "true") - { - _logger.LogError($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}"); - - // transcription completed - await HookEmitter.Emit(_services, x => x.OnTranscribeCompleted(request)); - - // Append the transcription to the dialog history - var transcript = JsonConvert.DeserializeObject(request.TranscriptionData); - if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript)) - { - var storage = _services.GetRequiredService(); - var message = new RoleDialogModel(AgentRole.User, transcript.Transcript) - { - CurrentAgentId = request.AgentId - }; - storage.Append(request.ConversationId, message); - } - } - - return Ok(); - } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs deleted file mode 100644 index 717f4fa6..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ /dev/null @@ -1,125 +0,0 @@ -using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.Twilio.Interfaces; -using BotSharp.Plugin.Twilio.Models; -using BotSharp.Plugin.Twilio.Services; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; - -namespace BotSharp.Plugin.Twilio.Controllers; - -public class TwilioStreamController : TwilioController -{ - private readonly TwilioSetting _settings; - private readonly IServiceProvider _services; - private readonly IHttpContextAccessor _context; - private readonly ILogger _logger; - - public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) - { - _settings = settings; - _services = services; - _context = context; - _logger = logger; - } - - [ValidateRequest] - [HttpPost("twilio/stream")] - public async Task InitiateStreamConversation(ConversationalVoiceRequest request) - { - var text = JsonSerializer.Serialize(request); - if (request?.CallSid == null) - { - throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); - } - - var twilio = _services.GetRequiredService(); - VoiceResponse response = default!; - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - SpeechPaths = [], - ActionOnEmptyResult = true - }; - - if (request.InitAudioFile != null) - { - instruction.SpeechPaths.Add(request.InitAudioFile); - } - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnSessionCreating(request, instruction); - }); - - request.ConversationId = await InitConversation(request); - instruction.AgentId = request.AgentId; - instruction.ConversationId = request.ConversationId; - - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") - { - response = new VoiceResponse(); - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailStarting(request); - }); - - var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); - response.Play(new Uri(url)); - } - else - { - response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); - } - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnSessionCreated(request); - }); - - return TwiML(response); - } - - private async Task InitConversation(ConversationalVoiceRequest request) - { - var convService = _services.GetRequiredService(); - var conversation = await convService.GetConversation(request.ConversationId); - if (conversation == null) - { - var conv = new Conversation - { - AgentId = request.AgentId, - Channel = ConversationChannel.Phone, - ChannelId = request.CallSid, - Title = $"Incoming phone call from {request.From}", - Tags = [], - }; - - conversation = await convService.NewConversation(conv); - } - - var states = new List - { - new("channel", ConversationChannel.Phone), - new("calling_phone", request.From), - new("twilio_call_sid", request.CallSid), - // Enable lazy routing mode to optimize realtime experience - new(StateConst.ROUTING_MODE, "lazy"), - }; - - if (request.InitAudioFile != null) - { - states.Add(new("init_audio_file", request.InitAudioFile)); - } - - convService.SetConversationId(conversation.Id, states); - convService.SaveStates(); - - return conversation.Id; - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs new file mode 100644 index 00000000..1fa020c0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs @@ -0,0 +1,58 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioTranscribeController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TwilioTranscribeController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/transcribe")] + public async Task PhoneRecordingTranscribe(ConversationalVoiceRequest request) + { + if (request.Final == "true") + { + _logger.LogInformation($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}"); + + // Append the transcription to the dialog history + var transcript = JsonConvert.DeserializeObject(request.TranscriptionData); + if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript)) + { + var storage = _services.GetRequiredService(); + var message = new RoleDialogModel(AgentRole.User, transcript.Transcript) + { + CurrentAgentId = request.AgentId + }; + storage.Append(request.ConversationId, message); + + var routing = _services.GetRequiredService(); + routing.Context.SetMessageId(request.ConversationId, message.MessageId); + + var convService = _services.GetRequiredService(); + convService.SetConversationId(request.ConversationId, []); + + // transcription completed + transcript.Language = request.LanguageCode; + await HookEmitter.Emit(_services, async x => await x.OnTranscribeCompleted(message, transcript)); + } + } + + return Ok(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 6078dd7b..8f042ca4 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -12,10 +12,10 @@ namespace BotSharp.Plugin.Twilio.Controllers; public class TwilioVoiceController : TwilioController { - private readonly TwilioSetting _settings; - private readonly IServiceProvider _services; - private readonly IHttpContextAccessor _context; - private readonly ILogger _logger; + protected readonly TwilioSetting _settings; + protected readonly IServiceProvider _services; + protected readonly IHttpContextAccessor _context; + protected readonly ILogger _logger; public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) { @@ -32,6 +32,7 @@ public class TwilioVoiceController : TwilioController /// /// /// + [Obsolete("Use twilio/inbound for streaming and non-streaming instead of this endpoint")] [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) @@ -352,11 +353,15 @@ public class TwilioVoiceController : TwilioController { await HookEmitter.Emit(_services, x => x.OnCallBusyStatus(request)); } + else if (request.CallStatus == "no-answer") + { + await HookEmitter.Emit(_services, x => x.OnCallNoAnswerStatus(request)); + } return Ok(); } - private Dictionary ParseStates(List states) + protected Dictionary ParseStates(List states) { var result = new Dictionary(); if (states is null || !states.Any()) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index 899cc526..747679cc 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -8,7 +8,6 @@ public interface ITwilioCallStatusHook Task OnVoicemailLeft(ConversationalVoiceRequest request); Task OnUserDisconnected(ConversationalVoiceRequest request); Task OnRecordingCompleted(ConversationalVoiceRequest request); - Task OnTranscribeCompleted(ConversationalVoiceRequest request); Task OnVoicemailStarting(ConversationalVoiceRequest request); /// @@ -19,4 +18,6 @@ public interface ITwilioCallStatusHook /// /// Task OnCallBusyStatus(ConversationalVoiceRequest request); + + Task OnCallNoAnswerStatus(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index d9dde562..df08df0a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -64,9 +64,4 @@ public class ConversationalVoiceRequest : VoiceRequest [FromForm] public string? TranscriptionEvent { get; set; } #endregion -} - -public class TranscriptionData -{ - public string Transcript { get; set; } = null!; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index afe6b89c..6d1e2d11 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -77,8 +77,12 @@ public class OutboundPhoneCallFn : IFunctionCallback statusUrl += $"&init-audio-file={initAudioFile}"; } + // load agent profile + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + // Set up process URL streaming or synchronous - if (_twilioSetting.StreamingEnabled) + if (agent.Profiles.Contains("realtime")) { processUrl += "/stream"; } @@ -119,7 +123,7 @@ public class OutboundPhoneCallFn : IFunctionCallback await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); - message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]"; + message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, RECORDING: {_twilioSetting.RecordingEnabled}]"; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index fda27954..fb637677 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Utilities; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; @@ -199,7 +201,7 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse, Agent agent) { var response = new VoiceResponse(); @@ -207,11 +209,16 @@ public class TwilioService if (_settings.TranscribeEnabled) { + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var hints = string.Join(", ", words); var start = new Start(); start.Transcription( track: "inbound_track", partialResults: false, - statusCallbackUrl: $"{_settings.CallbackHost}/twilio/record/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", name: conversationId); + statusCallbackUrl: $"{_settings.CallbackHost}/twilio/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", + name: conversationId, + hints: hints); response.Append(start); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index f0d845dc..a7cf6f9e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -7,10 +7,6 @@ public class TwilioSetting /// public string? PhoneNumber { get; set; } - /// - /// Enable streaming for outbound phone call - /// - public bool StreamingEnabled { get; set; } = false; public string AccountSID { get; set; } public string AuthToken { get; set; } public string AppSID { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index a31dbb02..d78489ad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,9 +1,7 @@ -using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; using BotSharp.Plugin.Twilio.Services; -using BotSharp.Plugin.Twilio.Services.Stream; using StackExchange.Redis; using Twilio; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs similarity index 99% rename from src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs rename to src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 35118ab3..7ff5ab98 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Http; using System.Net.WebSockets; using Task = System.Threading.Tasks.Task; -namespace BotSharp.Plugin.Twilio.Services.Stream; +namespace BotSharp.Plugin.Twilio; /// /// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js From 4d64b06570b1af90fda397d9b1ffd35662270182 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 12:02:02 -0500 Subject: [PATCH 02/13] Change to modelType when getting realtime model. --- .../MLTasks/ILlmProviderService.cs | 2 +- .../MLTasks/Settings/LlmConfigOptions.cs | 1 - .../MLTasks/Settings/LlmModelSetting.cs | 8 ++------ .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 3 ++- .../Infrastructures/CompletionProvider.cs | 10 +++++++--- .../Infrastructures/LlmProviderService.cs | 12 +++++------- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index e28f61ae..f70a9496 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,7 +6,7 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false); List GetProviderModels(string provider); List GetLlmConfigs(LlmConfigOptions? options = null); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs index 81668059..eae417a4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs @@ -4,6 +4,5 @@ public class LlmConfigOptions { public LlmModelType? Type { get; set; } public bool? MultiModal { get; set; } - public bool? RealTime { get; set; } public bool? ImageGeneration { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 2d98b254..b10f88f3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -37,11 +37,6 @@ public class LlmModelSetting /// public bool MultiModal { get; set; } - /// - /// If true, allow real-time interaction - /// - public bool RealTime { get; set; } - /// /// If true, allow generating images /// @@ -84,5 +79,6 @@ public enum LlmModelType Chat = 2, Image = 3, Embedding = 4, - Audio = 5 + Audio = 5, + Realtime = 6, } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 4879eb54..abe5f937 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Core.Infrastructures; namespace BotSharp.Core.Realtime.Services; @@ -76,7 +77,7 @@ public class RealtimeHub : IRealtimeHub if (agent.Profiles.Contains("realtime")) { var llmProviderService = _services.GetRequiredService(); - model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name; + model = llmProviderService.GetProviderModel("openai", "gpt-4o", modelType: LlmModelType.Realtime).Name; } _completer.SetModelName(model); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 386cce9e..4bf45d86 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -32,6 +32,10 @@ public class CompletionProvider { return GetAudioTranscriber(services, provider: provider, model: model); } + else if (settings.Type == LlmModelType.Realtime) + { + return GetRealTimeCompletion(services, provider: provider, model: model); + } else { return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig); @@ -172,7 +176,7 @@ public class CompletionProvider var completions = services.GetServices(); (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, multiModal: multiModal, - realTime: true, + modelType: LlmModelType.Realtime, agentConfig: agentConfig); var completer = completions.FirstOrDefault(x => x.Provider == provider); @@ -191,7 +195,7 @@ public class CompletionProvider string? model = null, string? modelId = null, bool? multiModal = null, - bool realTime = false, + LlmModelType? modelType = null, bool imageGenerate = false, AgentLlmConfig? agentConfig = null) { @@ -217,7 +221,7 @@ public class CompletionProvider var llmProviderService = services.GetRequiredService(); model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal, - realTime: realTime, + modelType: modelType, imageGenerate: imageGenerate)?.Name; } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 3baf4fb4..95cfd9c0 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false) { var models = GetProviderModels(provider) .Where(x => x.Id == id); @@ -54,7 +54,10 @@ public class LlmProviderService : ILlmProviderService models = models.Where(x => x.MultiModal == multiModal); } - models = models.Where(x => x.RealTime == realTime); + if (modelType.HasValue) + { + models = models.Where(x => x.Type == modelType.Value); + } models = models.Where(x => x.ImageGeneration == imageGenerate); @@ -130,11 +133,6 @@ public class LlmProviderService : ILlmProviderService models = models.Where(x => x.ImageGeneration == options.ImageGeneration.Value).ToList(); } - if (options.RealTime.HasValue) - { - models = models.Where(x => x.RealTime == options.RealTime.Value).ToList(); - } - if (models.IsNullOrEmpty()) { continue; From 06491b1a2836679ce3bc0418820007ce7ef3400a Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 16:25:37 -0500 Subject: [PATCH 03/13] Fix output endpoint. --- .../OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 6d1e2d11..22ce213b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -84,7 +84,7 @@ public class OutboundPhoneCallFn : IFunctionCallback // Set up process URL streaming or synchronous if (agent.Profiles.Contains("realtime")) { - processUrl += "/stream"; + processUrl += "/inbound"; } else { From 88539fbbb5b6481831b1adef4a2608478081da66 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 17:03:05 -0500 Subject: [PATCH 04/13] fix Dialogs is null --- .../BotSharp.Logger/Hooks/RateLimitConversationHook.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/RateLimitConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/RateLimitConversationHook.cs index 82fb0411..97308f69 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/RateLimitConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/RateLimitConversationHook.cs @@ -36,6 +36,11 @@ public class RateLimitConversationHook : ConversationHookBase return; } + if (Dialogs == null) + { + return; + } + // Check message sending frequency var userSents = Dialogs.Where(x => x.Role == AgentRole.User) .TakeLast(2).ToList(); From cc362b20d1466b0e3a7f1435b05963c71b31934f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 18:25:04 -0500 Subject: [PATCH 05/13] Fix conversation id in HangupPhoneCallFn --- .../OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index c84666dd..89d770ac 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -29,8 +29,8 @@ public class HangupPhoneCallFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var fileStorage = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - var conversationId = routing.Context.ConversationId; + var convService = _services.GetRequiredService(); + var conversationId = convService.ConversationId; var states = _services.GetRequiredService(); var callSid = states.GetState("twilio_call_sid"); From 5a166e7e21d481670de792e66268ee3664d87739 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 2 Apr 2025 20:47:32 -0500 Subject: [PATCH 06/13] fix gemini function call --- .../Chat/GeminiChatCompletionProvider.cs | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 8d5d9423..6b4ef08b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class GeminiChatCompletionProvider : IChatCompletion var text = part?.Text ?? string.Empty; RoleDialogModel responseMessage; - if (response.GetFunction()!=null) + if (response.GetFunction() != null) { responseMessage = new RoleDialogModel(AgentRole.Function, text) { @@ -78,7 +78,10 @@ public class GeminiChatCompletionProvider : IChatCompletion { Prompt = prompt, Provider = Provider, - Model = _model + Model = _model, + PromptCount = response.UsageMetadata?.PromptTokenCount ?? 0, + CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0, + CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0 }); } @@ -96,8 +99,8 @@ public class GeminiChatCompletionProvider : IChatCompletion } var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - var chatClient = client.CreateGeminiModel(_model); - var (prompt, messages) = PrepareOptions(chatClient,agent, conversations); + var chatClient = client.CreateGenerativeModel(_model); + var (prompt, messages) = PrepareOptions(chatClient, agent, conversations); var response = await chatClient.GenerateContentAsync(messages); @@ -120,11 +123,12 @@ public class GeminiChatCompletionProvider : IChatCompletion Provider = Provider, Model = _model, PromptCount = response?.UsageMetadata?.PromptTokenCount ?? 0, - CompletionCount = response?.UsageMetadata?.CandidatesTokenCount ?? 0 + CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0, + CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0 }); } - if (response.GetFunction()!=null) + if (response.GetFunction() != null) { var toolCall = response.GetFunction(); _logger.LogInformation($"[{agent.Name}]: {toolCall?.Name}({toolCall?.Args?.ToJsonString()})"); @@ -167,10 +171,10 @@ public class GeminiChatCompletionProvider : IChatCompletion await foreach (var response in asyncEnumerable) { - if (response.GetFunction()!=null) + if (response.GetFunction() != null) { var func = response.GetFunction(); - var update =func?.Args?.ToJsonString().ToString() ?? string.Empty; + var update = func?.Args?.ToJsonString().ToString() ?? string.Empty; _logger.LogInformation(update); await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) @@ -212,6 +216,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { AutoCallFunction = false }; + // Assembly messages var contents = new List(); var tools = new List(); @@ -221,7 +226,6 @@ public class GeminiChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); - contents.Add(new Content(instruction, AgentRole.User)); renderedInstructions.Add(instruction); systemPrompts.Add(instruction); } @@ -237,7 +241,7 @@ public class GeminiChatCompletionProvider : IChatCompletion var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}" ? new Schema() { Type = "object", - Properties = JsonSerializer.Deserialize(props), + Properties = JsonSerializer.Deserialize>(props), Required = def?.Required ?? [] } : null; @@ -261,23 +265,32 @@ public class GeminiChatCompletionProvider : IChatCompletion { if (message.Role == AgentRole.Function) { - contents.Add( new Content(message.Content,AgentRole.Function) - { - Role = AgentRole.Function, - Parts = - [ - new Part() + contents.Add(new Content([ + new Part() + { + FunctionCall = new FunctionCall { - FunctionCall = new FunctionCall + Name = message.FunctionName, + Args = JsonNode.Parse(message.FunctionArgs ?? "{}") + } + } + ], AgentRole.Model)); + + contents.Add(new Content([ + new Part() + { + FunctionResponse = new FunctionResponse + { + Name = message.FunctionName, + Response = new JsonObject() { - Name = message.FunctionName, - Args = JsonNode.Parse(message.FunctionArgs ?? "{}") + ["result"] = message.Content ?? string.Empty } } - ] - }); + } + ], AgentRole.Function)); - convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs})"); + convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}"); } else if (message.Role == AgentRole.User) { @@ -299,6 +312,7 @@ public class GeminiChatCompletionProvider : IChatCompletion : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; var request = new GenerateContentRequest { + SystemInstruction = !systemPrompts.IsNullOrEmpty() ? new Content(systemPrompts[0], AgentRole.System) : null, Contents = contents, Tools = tools, GenerationConfig = new() From 3d6607622e38d794c9de2eb409365e213f6f4627 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 2 Apr 2025 20:51:34 -0500 Subject: [PATCH 07/13] minor change --- .../Providers/Text/GeminiTextCompletionProvider.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index 4c956c97..340853a3 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -51,11 +51,8 @@ public class GeminiTextCompletionProvider : ITextCompletion var aiModel = client.CreateGenerativeModel(_model); PrepareOptions(aiModel); - _tokenStatistics.StartTimer(); var response = await aiModel.GenerateContentAsync(text); - _tokenStatistics.StopTimer(); - - var completion = response.Text ?? string.Empty; + var completion = response?.Text ?? string.Empty; // After completion hook foreach (var hook in contentHooks) @@ -64,7 +61,10 @@ public class GeminiTextCompletionProvider : ITextCompletion { Prompt = text, Provider = Provider, - Model = _model + Model = _model, + PromptCount = response.UsageMetadata?.PromptTokenCount ?? 0, + CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0, + CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0 }); } @@ -79,6 +79,7 @@ public class GeminiTextCompletionProvider : ITextCompletion private void PrepareOptions(GenerativeModel aiModel) { var settings = _services.GetRequiredService(); + aiModel.UseGoogleSearch = settings.Gemini.UseGoogleSearch; aiModel.UseGrounding = settings.Gemini.UseGrounding; aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour() From 8fcfd489a17eef167acdd6bd0c05336908aeb5d0 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 21:44:39 -0500 Subject: [PATCH 08/13] computer use draft --- BotSharp.sln | 22 +++ Directory.Packages.props | 30 ++-- .../BotSharp.Abstraction.ComputerUse.csproj | 28 ++++ .../MLTasks/IComputerUse.cs | 40 +++++ .../MLTasks/IScreenshot.cs | 7 + .../Models/ComputerUseArgs.cs | 14 ++ .../Models/ComputerUseOutput.cs | 9 + .../BotSharp.Abstraction.ComputerUse/Using.cs | 4 + .../BotSharp.Plugin.AudioHandler.csproj | 1 - .../Provider/NativeWhisperProvider.cs | 9 +- .../BotSharp.Plugin.MetaAI.csproj | 1 - .../BotSharp.Plugin.OsDriver.csproj | 23 +++ .../CoordinateScaler.cs | 94 +++++++++++ .../KeyboardSimulator.cs | 105 ++++++++++++ .../OsDriverPlugin.cs | 15 ++ .../ScreenshotWinOS.cs | 155 ++++++++++++++++++ src/Plugins/BotSharp.Plugin.OsDriver/Using.cs | 4 + .../BotSharp.Plugin.OsDriver/WinOSDriver.cs | 85 ++++++++++ .../WindowsMouseController.cs | 135 +++++++++++++++ .../BotSharp.Plugin.SqlDriver.csproj | 1 - .../Controllers/SqlDriverController.cs | 24 --- .../BotSharp.Plugin.WebDriver.csproj | 2 +- .../BotSharp.Test.ComputerUse.csproj | 19 +++ tests/BotSharp.Test.ComputerUse/Program.cs | 135 +++++++++++++++ 24 files changed, 918 insertions(+), 44 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj create mode 100644 src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/Using.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs create mode 100644 src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs delete mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs create mode 100644 tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj create mode 100644 tests/BotSharp.Test.ComputerUse/Program.cs diff --git a/BotSharp.sln b/BotSharp.sln index c70f6c74..92e50aeb 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -133,6 +133,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -549,6 +553,22 @@ Global {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.Build.0 = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.Build.0 = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.ActiveCfg = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.Build.0 = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.ActiveCfg = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.Build.0 = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -613,6 +633,8 @@ Global {781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} + {B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/Directory.Packages.props b/Directory.Packages.props index 1a8a7353..7f1d8b1d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,8 +18,10 @@ + + - + @@ -33,20 +35,20 @@ - + - - + + - + - + @@ -54,8 +56,8 @@ - - + + @@ -67,23 +69,23 @@ - + - + - - + + - + @@ -109,7 +111,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj new file mode 100644 index 00000000..7632b664 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj @@ -0,0 +1,28 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + Icon.png + $(GeneratePackageOnBuild) + $(SolutionDir)packages + + + + + \ + True + + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs new file mode 100644 index 00000000..fcaa8b77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs @@ -0,0 +1,40 @@ +/***************************************************************************** + Copyright 2025 Written by Haiping Chen. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +using System.Threading.Tasks; + +namespace BotSharp.Abstraction.ComputerUse.MLTasks; + +/// +/// Interface for computer use +/// +public interface IComputerUse +{ + /// + /// Capture screen of the specified display + /// + /// + /// + Task CaptureScreen(ComputerUseArgs args); + + Task MouseMove(ComputerUseArgs args); + + Task MouseClick(ComputerUseArgs args); + + Task InputText(ComputerUseArgs args); + + Task KeyPress(ComputerUseArgs args); +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs new file mode 100644 index 00000000..0460f523 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.ComputerUse.MLTasks; + +public interface IScreenshot +{ + (int x, int y) GetScreenSize(int screenIndex); + byte[] CaptureScreen(int displayId); +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs new file mode 100644 index 00000000..8d5812c7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Abstraction.ComputerUse.Models; + +public class ComputerUseArgs +{ + /// + /// Number of multi-screens + /// + public int DisplayId { get; set; } + public int X { get; set; } + public int Y { get; set; } + public string Text { get; set; } = string.Empty; + public MouseButton MouseButton { get; set; } + public KeyCode KeyCode { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs new file mode 100644 index 00000000..ba5c3935 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.ComputerUse.Models; + +public class ComputerUseOutput +{ + /// + /// Base64 encoded image + /// + public string? Screenshot { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs new file mode 100644 index 00000000..a0da56f0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs @@ -0,0 +1,4 @@ +global using System.Threading.Tasks; +global using BotSharp.Abstraction.ComputerUse.Models; + +global using SharpHook.Native; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index abc9749a..f90367c5 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index aa35d17b..529e2943 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -89,7 +89,7 @@ public class NativeWhisperProvider : IAudioTranscription } var bytes = _fileStorage.GetFileBytes(modelLoc); - _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); + _whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { @@ -101,7 +101,12 @@ public class NativeWhisperProvider : IAudioTranscription private void DownloadModel(GgmlType modelType, string modelDir) { - using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + // Create an instance of WhisperGgmlDownloader + var downloader = WhisperGgmlDownloader.Default; + + // Use the instance to call GetGgmlModelAsync + using var modelStream = downloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); modelStream.Close(); } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 231b4b3f..69c6a697 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -16,7 +16,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj b/src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj new file mode 100644 index 00000000..26449635 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj @@ -0,0 +1,23 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs b/src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs new file mode 100644 index 00000000..9a96111a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs @@ -0,0 +1,94 @@ +namespace BotSharp.Plugin.OsDriver; + +public enum ScalingSource +{ + COMPUTER, + API, + // Add other possible values if needed +} + +public class Dimension +{ + public int Width { get; set; } + public int Height { get; set; } + + public Dimension(int width, int height) + { + Width = width; + Height = height; + } +} + +public class ToolError : Exception +{ + public ToolError(string message) : base(message) + { + } +} + +public class CoordinateScaler +{ + private bool _scalingEnabled; + private int _width; + private int _height; + + private static readonly List MaxScalingTargets = new List + { + new Dimension(1024, 768), //XGA 4:3 + new Dimension(1280, 800), //WXGA 16:10 + new Dimension(1366, 768), //FWXGA 16:9 + // Add more dimensions as needed + }; + + public CoordinateScaler(bool scalingEnabled, int width, int height) + { + _scalingEnabled = scalingEnabled; + _width = width; + _height = height; + } + + public (int x, int y) ScaleCoordinates(ScalingSource source, int x, int y) + { + if (!_scalingEnabled) + return (x, y); + + double ratio = (double)_width / _height; + Dimension targetDimension = null; + + foreach (var dimension in MaxScalingTargets) + { + if (Math.Abs((double)dimension.Width / dimension.Height - ratio) < 0.02) + { + if (dimension.Width < _width) + { + targetDimension = dimension; + } + break; + } + } + + if (targetDimension == null) + return (x, y); + + double xScalingFactor = (double)targetDimension.Width / _width; + double yScalingFactor = (double)targetDimension.Height / _height; + + if (source == ScalingSource.API) + { + if (x > _width || y > _height) + throw new ToolError($"Coordinates {x}, {y} are out of bounds"); + + // Scale up + int newX = (int)Math.Round(x / xScalingFactor); + int newY = (int)Math.Round(y / yScalingFactor); + return (newX, newY); + } + else + { + // Scale down + int newX = (int)Math.Round(x * xScalingFactor); + int newY = (int)Math.Round(y * yScalingFactor); + return (newX, newY); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs b/src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs new file mode 100644 index 00000000..ff8e6b5c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs @@ -0,0 +1,105 @@ +using SharpHook.Native; +using SharpHook; + +namespace BotSharp.Plugin.OsDriver; + +public class KeyboardSimulator +{ + public static void SimulateTextInput(string text) + { + if (!string.IsNullOrWhiteSpace(text)) + { + var simulator = new EventSimulator(); + simulator.SimulateTextEntry(text); + } + } + + public static void SimulateKeyCombination(string keyCombination) + { + if (string.IsNullOrWhiteSpace(keyCombination)) + throw new ArgumentException("Key combination cannot be null or empty."); + + // Split the key combination into individual keys + var keys = keyCombination.Split(new[] { '+', ' ' }, StringSplitOptions.RemoveEmptyEntries); + + IEventSimulator simulator = new EventSimulator(); + + // Stack to keep track of keys pressed + Stack keysDown = new Stack(); + ModifierMask currentMask = ModifierMask.None; + + // Map key names to KeyCode enums + var keyMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "Shift", KeyCode.VcLeftShift }, + { "Ctrl", KeyCode.VcLeftControl }, + { "Control", KeyCode.VcLeftControl }, + { "Alt", KeyCode.VcLeftAlt }, + { "LWin", KeyCode.VcLeftMeta }, + { "RWin", KeyCode.VcRightMeta }, + { "Enter", KeyCode.VcEnter }, + { "Return", KeyCode.VcEnter }, + { "Super", KeyCode.VcLeftMeta }, + { "Windows", KeyCode.VcLeftMeta }, + // Add other keys as needed + }; + + // Map modifier keys to ModifierMask enums + var modifierMap = new Dictionary + { + { KeyCode.VcLeftShift, ModifierMask.Shift }, + { KeyCode.VcLeftControl, ModifierMask.Ctrl }, + { KeyCode.VcLeftAlt, ModifierMask.Alt }, + { KeyCode.VcLeftMeta, ModifierMask.Meta }, + { KeyCode.VcRightMeta, ModifierMask.Meta } + }; + + // Press down all keys + foreach (var key in keys) + { + KeyCode keyCode; + + // Check if the key is in the keyMap + if (keyMap.TryGetValue(key, out keyCode)) + { + simulator.SimulateKeyPress(keyCode); + + // Update the modifier mask if it's a modifier key + if (modifierMap.TryGetValue(keyCode, out ModifierMask modifier)) + { + currentMask |= modifier; + } + + keysDown.Push(keyCode); + } + else if (key.Length == 1) + { + // Handle single character keys + char c = key.ToUpper()[0]; + KeyCode charKeyCode = (KeyCode)Enum.Parse(typeof(KeyCode), $"Vc{c}", true); + + simulator.SimulateKeyPress(charKeyCode); + + keysDown.Push(charKeyCode); + } + else + { + throw new ArgumentException($"Unknown key: {key}"); + } + } + + // Release keys in reverse order + while (keysDown.Count > 0) + { + KeyCode keyCode = keysDown.Pop(); + + // Update the modifier mask if it's a modifier key + if (modifierMap.ContainsKey(keyCode)) + { + currentMask &= ~modifierMap[keyCode]; + } + + simulator.SimulateKeyRelease(keyCode); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs new file mode 100644 index 00000000..5f41fda4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs @@ -0,0 +1,15 @@ +using BotSharp.Abstraction.Plugins; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace BotSharp.Plugin.OsDriver; + +public class OsDriverPlugin : IBotSharpPlugin +{ + public string Id => "5aef4940-2f95-464b-ad37-43dbf89febf0"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs b/src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs new file mode 100644 index 00000000..c05a2384 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs @@ -0,0 +1,155 @@ +using BotSharp.Abstraction.ComputerUse.MLTasks; +using System.Runtime.InteropServices; +using System.Drawing; +using System.Drawing.Imaging; + +namespace BotSharp.Plugin.OsDriver; + +public class ScreenshotWinOS : IScreenshot +{ + // Define RECT structure + [StructLayout(LayoutKind.Sequential)] + public struct RECT + { + public int Left, Top, Right, Bottom; + } + + // Define MONITORINFO structure + [StructLayout(LayoutKind.Sequential)] + public struct MONITORINFO + { + public uint cbSize; + public RECT rcMonitor; + public RECT rcWork; + public uint dwFlags; + } + + // Delegate for monitor enumeration callback + private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData); + + // P/Invoke declarations + [DllImport("user32.dll")] + private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip, + MonitorEnumDelegate lpfnEnum, IntPtr dwData); + + [DllImport("user32.dll", CharSet = CharSet.Auto)] + private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateDC(string lpszDriver, string lpszDevice, + string lpszOutput, IntPtr lpInitData); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateCompatibleDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight); + + [DllImport("gdi32.dll")] + private static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj); + + [DllImport("gdi32.dll")] + private static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth, + int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, CopyPixelOperation dwRop); + + [DllImport("gdi32.dll")] + private static extern bool DeleteDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + private static extern bool DeleteObject(IntPtr hObject); + + // Class to hold monitor information + public class MonitorInfo + { + public IntPtr MonitorHandle; + public RECT MonitorArea; + } + + public byte[] CaptureScreen(int displayId) + { + // Get all monitors + var monitors = GetMonitors(); + + // Validate the screen index + if (displayId < 0 || displayId >= monitors.Count) + { + throw new ArgumentOutOfRangeException(nameof(displayId), "Invalid screen index."); + } + + var monitor = monitors[displayId]; + using var bitmap = CaptureMonitor(monitor); + + using var ms = new System.IO.MemoryStream(); + bitmap.Save(ms, ImageFormat.Jpeg); + + return ms.ToArray(); + } + + // Method to capture a specific monitor + public static Bitmap CaptureMonitor(MonitorInfo monitor) + { + int width = monitor.MonitorArea.Right - monitor.MonitorArea.Left; + int height = monitor.MonitorArea.Bottom - monitor.MonitorArea.Top; + + IntPtr hdcSrc = CreateDC("DISPLAY", null, null, IntPtr.Zero); + IntPtr hdcDest = CreateCompatibleDC(hdcSrc); + IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, width, height); + IntPtr hOld = SelectObject(hdcDest, hBitmap); + + bool success = BitBlt(hdcDest, 0, 0, width, height, hdcSrc, + monitor.MonitorArea.Left, monitor.MonitorArea.Top, CopyPixelOperation.SourceCopy | CopyPixelOperation.CaptureBlt); + + Bitmap bmp = null; + + if (success) + { + bmp = Image.FromHbitmap(hBitmap); + } + + // Clean up + SelectObject(hdcDest, hOld); + DeleteObject(hBitmap); + DeleteDC(hdcDest); + DeleteDC(hdcSrc); + + return bmp; + } + + public (int x, int y) GetScreenSize(int screenIndex) + { + // Get all monitors + var monitors = GetMonitors(); + + // Validate the screen index + if (screenIndex < 0 || screenIndex >= monitors.Count) + { + throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index."); + } + + var monitor = monitors[screenIndex]; + return (monitor.MonitorArea.Right - monitor.MonitorArea.Left, monitor.MonitorArea.Bottom - monitor.MonitorArea.Top); + } + + // Method to get all connected monitors + public static List GetMonitors() + { + var monitors = new List(); + + bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData) + { + var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) }; + if (GetMonitorInfo(hMonitor, ref mi)) + { + monitors.Add(new MonitorInfo + { + MonitorHandle = hMonitor, + MonitorArea = mi.rcMonitor + }); + } + return true; // Continue enumeration + } + + EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero); + return monitors; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/Using.cs b/src/Plugins/BotSharp.Plugin.OsDriver/Using.cs new file mode 100644 index 00000000..5372a211 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/Using.cs @@ -0,0 +1,4 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Threading.Tasks; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs b/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs new file mode 100644 index 00000000..c4e18f2c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs @@ -0,0 +1,85 @@ +using BotSharp.Abstraction.ComputerUse.MLTasks; +using BotSharp.Abstraction.ComputerUse.Models; +using SharpHook; +using SixLabors.ImageSharp.Formats.Jpeg; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; + +namespace BotSharp.Plugin.OsDriver; + +public class WinOSDriver : IComputerUse +{ + public static void TakeAction(string action, string? text, Tuple? coordinate, int monitorIndex, CoordinateScaler coordScaler) + { + switch (action) + { + case "type": + KeyboardSimulator.SimulateTextInput(text); + break; + case "key": + KeyboardSimulator.SimulateKeyCombination(text); + break; + case "mouse_move": + var scaledCoord = coordScaler.ScaleCoordinates(ScalingSource.API, coordinate.Item1, coordinate.Item2); + WindowsMouseController.SetCursorPositionOnMonitor(monitorIndex, scaledCoord.Item1, scaledCoord.Item2); + break; + case "left_click": + WindowsMouseController.LeftClick(); + break; + case "right_click": + WindowsMouseController.RightClick(); + break; + default: + throw new ToolError($"Action {action} is not supported"); + } + } + + + public static string DownscaleScreenshot(byte[] screenshot, int scaledX, int scaledY) + { + // Convert Bitmap to MemoryStream + using var memoryStream = new MemoryStream(screenshot); + + memoryStream.Position = 0; // Reset stream position + + // Load the image into ImageSharp + using var image = SixLabors.ImageSharp.Image.Load(memoryStream); + // Resize the image to scaled dimensions + image.Mutate(x => x.Resize(scaledX, scaledY)); + + // Save the image + using var ms = new MemoryStream(); + image.Save(ms, new JpegEncoder()); + ms.Position = 0; // Reset stream position + //convert to byte 64 string + byte[] imageBytes = ms.ToArray(); + return Convert.ToBase64String(imageBytes); + } + + public Task CaptureScreen(ComputerUseArgs args) + { + throw new NotImplementedException(); + } + + public Task MouseMove(ComputerUseArgs args) + { + throw new NotImplementedException(); + } + + public void MouseClick(ComputerUseArgs args) + { + var simulator = new EventSimulator(); + simulator.SimulateMousePress(args.MouseButton); + simulator.SimulateMouseRelease(args.MouseButton); + } + + public Task InputText(ComputerUseArgs args) + { + throw new NotImplementedException(); + } + + public Task KeyPress(ComputerUseArgs args) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs b/src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs new file mode 100644 index 00000000..c604780f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs @@ -0,0 +1,135 @@ +using SharpHook; +using SharpHook.Native; +using System.Drawing; +using System.Runtime.InteropServices; + +namespace BotSharp.Plugin.OsDriver; + +public class WindowsMouseController +{ + [StructLayout(LayoutKind.Sequential)] + public struct RECT + { + public int Left, Top, Right, Bottom; + } + + // Define MONITORINFO structure + [StructLayout(LayoutKind.Sequential)] + public struct MONITORINFO + { + public uint cbSize; + public RECT rcMonitor; + public RECT rcWork; + public uint dwFlags; + } + + // Delegate for monitor enumeration callback + private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData); + + // P/Invoke declarations + [DllImport("user32.dll")] + private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip, + MonitorEnumDelegate lpfnEnum, IntPtr dwData); + + [DllImport("user32.dll", CharSet = CharSet.Auto)] + private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi); + + // Class to hold monitor information + public class MonitorInfo + { + public IntPtr MonitorHandle; + public RECT MonitorArea; + } + + // Method to get all connected monitors + public static List GetMonitors() + { + var monitors = new List(); + + bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData) + { + var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) }; + if (GetMonitorInfo(hMonitor, ref mi)) + { + monitors.Add(new MonitorInfo + { + MonitorHandle = hMonitor, + MonitorArea = mi.rcMonitor + }); + } + return true; // Continue enumeration + } + + EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero); + return monitors; + } + + public static (int virtualX, int virtualY) GetVirtualCoordinates(int screenIndex, int x, int y) + { + // Get all screens + var monitors = GetMonitors(); + + // Validate the screen index + if (screenIndex < 0 || screenIndex >= monitors.Count) + { + throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index."); + } + + var monitor = monitors[screenIndex]; + var monitorBounds = new Rectangle(monitor.MonitorArea.Left, monitor.MonitorArea.Top, + monitor.MonitorArea.Right - monitor.MonitorArea.Left, + monitor.MonitorArea.Bottom - monitor.MonitorArea.Top); + + + // Validate coordinates + if (x < 0 || x >= monitorBounds.Width || y < 0 || y >= monitorBounds.Height) + { + throw new ArgumentOutOfRangeException("Coordinates are out of bounds for the specified monitor."); + } + + // Convert to virtual screen coordinates + int virtualX = (int)(monitorBounds.X + x); + int virtualY = (int)(monitorBounds.Y + y); + + return (virtualX, virtualY); + } + + // Method to set the cursor position on a specific monitor + public static void SetCursorPositionOnMonitor(int monitorIndex, int x, int y) + { + var (virtualX, virtualY) = GetVirtualCoordinates(monitorIndex, x, y); + + IEventSimulator simulator = new EventSimulator(); + + // Move the cursor to the specified position + simulator.SimulateMouseMovement(Convert.ToInt16(virtualX), Convert.ToInt16(virtualY)); + } + + // Method to perform a left-click at the current cursor position + public static void LeftClick() + { + IEventSimulator simulator = new EventSimulator(); + simulator.SimulateMousePress(MouseButton.Button1); + simulator.SimulateMouseRelease(MouseButton.Button1); + + } + + public static void RightClick() + { + IEventSimulator simulator = new EventSimulator(); + simulator.SimulateMousePress(MouseButton.Button2); + simulator.SimulateMouseRelease(MouseButton.Button2); + + } + + + // Method to move the cursor and perform a left-click at specified coordinates on a specific monitor + public static void ClickAtPositionOnMonitor(int monitorIndex, int x, int y) + { + SetCursorPositionOnMonitor(monitorIndex, x, y); + System.Threading.Thread.Sleep(50); // Optional delay to ensure the cursor has moved + LeftClick(); + } + + +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index ec2d91c1..d7626c47 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -100,7 +100,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs deleted file mode 100644 index 6f16aafa..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs +++ /dev/null @@ -1,24 +0,0 @@ -using BotSharp.Plugin.SqlDriver.Models; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; - -namespace BotSharp.Plugin.SqlDriver.Controllers; - -[Authorize] -[ApiController] -public class SqlDriverController : ControllerBase -{ - private readonly IServiceProvider _services; - - public SqlDriverController(IServiceProvider services) - { - _services = services; - } - - [HttpPost("/knowledge/database/import")] - public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) - { - var dbKnowledge = _services.GetRequiredService(); - return await dbKnowledge.Import(request); - } -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index 03f1a9b9..2f391853 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj new file mode 100644 index 00000000..605a93c8 --- /dev/null +++ b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + diff --git a/tests/BotSharp.Test.ComputerUse/Program.cs b/tests/BotSharp.Test.ComputerUse/Program.cs new file mode 100644 index 00000000..9fc96de5 --- /dev/null +++ b/tests/BotSharp.Test.ComputerUse/Program.cs @@ -0,0 +1,135 @@ +using Anthropic.SDK.Constants; +using Anthropic.SDK.Messaging; +using Anthropic.SDK; +using System.Text.Json.Nodes; +using BotSharp.Plugin.OsDriver; +using Anthropic.SDK.Common; + +Console.WriteLine("Open Google Chrome in Incognito Mode in a given window. Enter which disaplay ID you are using:"); +var displayId = Convert.ToInt32(Console.ReadLine()); + +var capturer = new ScreenshotWinOS(); + +var (width, height) = capturer.GetScreenSize(displayId - 1); +Console.WriteLine($"Screen Size: {width}x{height}"); + +var coordScaler = new CoordinateScaler(true, width, height); + +var (scaledX, scaledY) = coordScaler.ScaleCoordinates(ScalingSource.COMPUTER, width, height); + +Console.WriteLine($"Scaled Screen Size: {scaledX}x{scaledY}"); +var client = new AnthropicClient(new APIAuthentication("")); + +var messages = new List(); +messages.Add(new Message() +{ + Role = RoleType.User, + Content = new List() + { + new TextContent() + { + Text = """ + Open google.com to search blogs about BotSharp, find the most recent updates on it and output the summary. + """ + } + } +}); + +var tools = new List() + { + new Function("computer", "computer_20241022",new Dictionary() + { + {"display_width_px", scaledX }, + {"display_height_px", scaledY }, + {"display_number", displayId } + }) + }; +var parameters = new MessageParameters() +{ + Messages = messages, + Model = AnthropicModels.Claude35Sonnet, + Stream = false, + Tools = tools, + System = new List() + { + new SystemMessage($"""" + A Google Chrome Incognito window is already open and maximized in the appropriate monitor. Use that instance. + """") + } +}; + +var isRunning = true; + +var res = await client.Messages.GetClaudeMessageAsync(parameters); +messages.Add(res.Message); +while (isRunning) +{ + var toolUse = res.Content.OfType().ToList(); + + if (toolUse.Count == 0) + { + isRunning = false; + break; + } + var cb = new List(); + foreach (var tool in toolUse) + { + var action = tool.Input["action"].ToString(); + var text = tool.Input["text"]?.ToString(); + var coordinate = tool.Input["coordinate"] as JsonArray; + + switch (action) + { + case "screenshot": + messages.Add(new Message() + { + Role = RoleType.User, + Content = new List() + { + new ToolResultContent() + { + ToolUseId = tool.Id, + Content =new List() { new ImageContent() + { + Source = new ImageSource() { + Data = WinOSDriver.DownscaleScreenshot(capturer.CaptureScreen(displayId -1), scaledX, scaledY), + MediaType = "image/jpeg" + } + } } + } + } + }); + break; + default: + WinOSDriver.TakeAction(action, text, + coordinate == null ? null : new Tuple(Convert.ToInt32(coordinate[0].ToString()), + Convert.ToInt32(coordinate[1].ToString())), displayId - 1, coordScaler); + await Task.Delay(1000); + cb.Add(new ToolResultContent() + { + ToolUseId = tool.Id, + Content = new List() + { + new TextContent() + { + Text = "Action completed" + } + } + }); + break; + } + + } + messages.Add(new Message() + { + Role = RoleType.User, + Content = cb + }); + res = await client.Messages.GetClaudeMessageAsync(parameters); + messages.Add(res.Message); +} + +Console.WriteLine("----------------------------------------------"); +Console.WriteLine("Final Result:"); +Console.WriteLine(messages.Last().Content.OfType().First().Text); +Console.ReadLine(); \ No newline at end of file From 6f938f6366558c343fb5dd8d020007ebfe6a094a Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 21:53:20 -0500 Subject: [PATCH 09/13] fix compile issue. --- BotSharp.sln | 22 +++++++++++++++++++ .../BotSharp.Plugin.OsDriver/WinOSDriver.cs | 4 +++- .../BotSharp.Test.BrowserUse.csproj | 10 +++++++++ .../BotSharp.Test.BrowserUse/Program.cs | 2 ++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj create mode 100644 tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/Program.cs diff --git a/BotSharp.sln b/BotSharp.sln index 92e50aeb..52055258 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -137,6 +137,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Abstraction.ComputerUse", "src\Infrastructure\BotSharp.Abstraction.ComputerUse\BotSharp.Abstraction.ComputerUse.csproj", "{970BE341-9AC8-99A5-6572-E703C1E02FCB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -569,6 +573,22 @@ Global {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|x64.ActiveCfg = Debug|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|x64.Build.0 = Debug|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|Any CPU.Build.0 = Release|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|x64.ActiveCfg = Release|Any CPU + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|x64.Build.0 = Release|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.ActiveCfg = Debug|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.Build.0 = Debug|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.Build.0 = Release|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.ActiveCfg = Release|Any CPU + {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -635,6 +655,8 @@ Global {534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F} + {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} + {970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs b/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs index c4e18f2c..45a2572d 100644 --- a/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs +++ b/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs @@ -66,11 +66,13 @@ public class WinOSDriver : IComputerUse throw new NotImplementedException(); } - public void MouseClick(ComputerUseArgs args) + public Task MouseClick(ComputerUseArgs args) { var simulator = new EventSimulator(); simulator.SimulateMousePress(args.MouseButton); simulator.SimulateMouseRelease(args.MouseButton); + + return null; } public Task InputText(ComputerUseArgs args) diff --git a/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj new file mode 100644 index 00000000..2150e379 --- /dev/null +++ b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj @@ -0,0 +1,10 @@ + + + + Exe + net8.0 + enable + enable + + + diff --git a/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/Program.cs b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/Program.cs new file mode 100644 index 00000000..3751555c --- /dev/null +++ b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/Program.cs @@ -0,0 +1,2 @@ +// See https://aka.ms/new-console-template for more information +Console.WriteLine("Hello, World!"); From f5bfdb56ddf964e471b3ea55ec24b37a3b3a7f16 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 2 Apr 2025 21:58:13 -0500 Subject: [PATCH 10/13] BotSharp.Test.BrowserUser --- BotSharp.sln | 22 +++++++++---------- .../BotSharp.Test.BrowserUse.csproj | 0 .../BotSharp.Test.BrowserUse/Program.cs | 0 3 files changed, 11 insertions(+), 11 deletions(-) rename tests/{BotSharp.Test.ComputerUse => }/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj (100%) rename tests/{BotSharp.Test.ComputerUse => }/BotSharp.Test.BrowserUse/Program.cs (100%) diff --git a/BotSharp.sln b/BotSharp.sln index 52055258..2bacbcdb 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -137,10 +137,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Abstraction.ComputerUse", "src\Infrastructure\BotSharp.Abstraction.ComputerUse\BotSharp.Abstraction.ComputerUse.csproj", "{970BE341-9AC8-99A5-6572-E703C1E02FCB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -573,14 +573,6 @@ Global {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|x64.ActiveCfg = Debug|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Debug|x64.Build.0 = Debug|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|Any CPU.Build.0 = Release|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|x64.ActiveCfg = Release|Any CPU - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F}.Release|x64.Build.0 = Release|Any CPU {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -589,6 +581,14 @@ Global {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.Build.0 = Release|Any CPU {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.ActiveCfg = Release|Any CPU {970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.Build.0 = Release|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|x64.ActiveCfg = Debug|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|x64.Build.0 = Debug|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.Build.0 = Release|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.ActiveCfg = Release|Any CPU + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -655,8 +655,8 @@ Global {534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F} - {1177D2E9-23B1-4D6E-8CD7-972B47C7BC4F} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} + {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj b/tests/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj similarity index 100% rename from tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj rename to tests/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj diff --git a/tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/Program.cs b/tests/BotSharp.Test.BrowserUse/Program.cs similarity index 100% rename from tests/BotSharp.Test.ComputerUse/BotSharp.Test.BrowserUse/Program.cs rename to tests/BotSharp.Test.BrowserUse/Program.cs From 23df4067327ba552a472115ee1ff63a2ae3fb93c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 3 Apr 2025 12:13:42 -0500 Subject: [PATCH 11/13] SetMessageId --- .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index abe5f937..49337253 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -88,14 +88,14 @@ public class RealtimeHub : IRealtimeHub var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); - /*if (dialogs.Count == 0) + if (dialogs.Count == 0) { dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); storage.Append(_conn.ConversationId, dialogs.First()); - }*/ + } routing.Context.SetDialogs(dialogs); - // routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); + routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); From eeae8d348dfbfc4aadeef47dd6db6ea1b82d1873 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 3 Apr 2025 14:32:04 -0500 Subject: [PATCH 12/13] Add route_to_agent log --- .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 8 ++++++++ .../Providers/Realtime/RealTimeCompletionProvider.cs | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 49337253..bf813034 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.MLTasks.Settings; +using BotSharp.Abstraction.Options; using BotSharp.Core.Infrastructures; namespace BotSharp.Core.Realtime.Services; @@ -144,6 +146,12 @@ public class RealtimeHub : IRealtimeHub if (message.MessageType == MessageTypeName.FunctionCall && !string.IsNullOrEmpty(message.FunctionName)) { + if (message.FunctionName == "route_to_agent") + { + var instruction = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message)); + } + await routing.InvokeFunction(message.FunctionName, message); } else diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 4499f851..ba2dc522 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -163,8 +163,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (response.Type == "error") { - var error = JsonSerializer.Deserialize(receivedText); - _logger.LogError($"Error: {error.Body.Message}"); + _logger.LogError($"{response.Type}: {receivedText}"); } else if (response.Type == "session.created") { @@ -677,6 +676,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { Provider = Provider, Model = _model, + Prompt = "[hook.AfterGenerated] [UNCHANGED PROMPT]", CompletionCount = data.Usage.OutputTokens, PromptCount = data.Usage.InputTokens }); From 11d315d33a9e64e9c96e5d500cf93f3f83164d86 Mon Sep 17 00:00:00 2001 From: Haiping Date: Thu, 3 Apr 2025 16:26:03 -0500 Subject: [PATCH 13/13] Upgrade OpeAI SDK to 2.2.0-beta.4 --- Directory.Packages.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 7f1d8b1d..d08f7269 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -42,8 +42,8 @@ - - + + @@ -143,4 +143,4 @@ - \ No newline at end of file +