SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into upgrademcp

Browse source
This commit is contained in:
geffzhang 2025-04-04 11:00:07 +08:00 committed by GitHub
parent 38d1d4fcba 11d315d33a
commit 3ea5b3a31d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
52 changed files with 1323 additions and 303 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
BotSharp.sln
View file

@ -133,6 +133,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Abstraction.ComputerUse", "src\Infrastructure\BotSharp.Abstraction.ComputerUse\BotSharp.Abstraction.ComputerUse.csproj", "{970BE341-9AC8-99A5-6572-E703C1E02FCB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -549,6 +557,38 @@ Global
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.Build.0 = Debug|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.Build.0 = Release|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.ActiveCfg = Release|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.Build.0 = Release|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.ActiveCfg = Debug|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.Build.0 = Debug|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.ActiveCfg = Debug|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.Build.0 = Debug|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.Build.0 = Release|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.ActiveCfg = Release|Any CPU
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.Build.0 = Release|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|x64.ActiveCfg = Debug|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|x64.Build.0 = Debug|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.Build.0 = Release|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.ActiveCfg = Release|Any CPU
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -613,6 +653,10 @@ Global
{781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
{970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

34
Directory.Packages.props
View file

@ -18,8 +18,10 @@
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="SharpHook" Version="5.3.8" />
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
<PackageVersion Include="System.Memory.Data" Version="8.0.0" />
<PackageVersion Include="System.Text.Json" Version="8.0.5" />
<PackageVersion Include="Serilog.Sinks.Console" Version="6.0.0" />
@ -33,20 +35,20 @@
<PackageVersion Include="Nanoid" Version="3.1.0" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
<PackageVersion Include="Anthropic.SDK" Version="4.7.1" />
<PackageVersion Include="Anthropic.SDK" Version="5.1.1" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
<PackageVersion Include="NAudio" Version="2.2.1" />
<PackageVersion Include="NAudio.Core" Version="2.2.1" />
<PackageVersion Include="Whisper.net" Version="1.5.0" />
<PackageVersion Include="Whisper.net.Runtime" Version="1.5.0" />
<PackageVersion Include="Whisper.net" Version="1.8.1" />
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
<PackageVersion Include="NCrontab" Version="3.3.3" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.1.0" />
<PackageVersion Include="OpenAI" Version="2.1.0" />
<PackageVersion Include="MailKit" Version="4.7.0" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.4" />
<PackageVersion Include="OpenAI" Version="2.2.0-beta.4" />
<PackageVersion Include="MailKit" Version="4.11.0" />
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageVersion Include="MySql.Data" Version="9.0.0" />
<PackageVersion Include="NPOI" Version="2.7.1" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.3.0" />
<PackageVersion Include="Refit" Version="8.0.0" />
<PackageVersion Include="Refit.HttpClientFactory" Version="8.0.0" />
<PackageVersion Include="Jint" Version="4.1.0" />
@ -54,8 +56,8 @@
<PackageVersion Include="TensorFlow.Keras" Version="0.15.0" />
<PackageVersion Include="LangChain.Providers.Google.VertexAI" Version="0.15.3-dev.58" />
<PackageVersion Include="LLamaSharp" Version="0.21.0" />
<PackageVersion Include="FaissMask" Version="0.2.0" />
<PackageVersion Include="FastText.NetWrapper" Version="1.3.0" />
<PackageVersion Include="FaissMask" Version="0.4.2" />
<PackageVersion Include="FastText.NetWrapper" Version="1.3.1" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25161.3" />
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.1.0" />
@ -67,23 +69,23 @@
<PackageVersion Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
<PackageVersion Include="Sdcb.PaddleOCR" Version="2.7.0.1" />
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
<PackageVersion Include="System.Drawing.Common" Version="8.0.7" />
<PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
<PackageVersion Include="pythonnet" Version="3.0.4" />
<PackageVersion Include="Qdrant.Client" Version="1.13.0" />
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
<PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
<PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
<PackageVersion Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.16.0-alpha" />
<PackageVersion Include="Microsoft.VisualStudio.Validation" Version="17.8.8" />
<PackageVersion Include="Sdcb.SparkDesk" Version="3.1.0" />
<PackageVersion Include="MySqlConnector" Version="2.3.7" />
<PackageVersion Include="Npgsql" Version="8.0.5" />
<PackageVersion Include="MySqlConnector" Version="2.4.0" />
<PackageVersion Include="Npgsql" Version="8.0.7" />
<PackageVersion Include="Tencent.QCloud.Cos.Sdk" Version="5.4.39" />
<PackageVersion Include="StackExchange.Redis" Version="2.7.27" />
<PackageVersion Include="StrongGrid" Version="0.108.0" />
<PackageVersion Include="Twilio.AspNet.Common" Version="8.1.1" />
<PackageVersion Include="Twilio.AspNet.Core" Version="8.1.1" />
<PackageVersion Include="Microsoft.Playwright" Version="1.49.0" />
<PackageVersion Include="Microsoft.Playwright" Version="1.51.0" />
<PackageVersion Include="Senparc.Weixin.MP.Middleware" Version="1.2.1" />
<PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
@ -142,4 +144,4 @@
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" />
<PackageVersion Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="6.0.26" />
</ItemGroup>
</Project>
</Project>

28
src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj Normal file
View file

@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
<Nullable>enable</Nullable>
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<PackageIcon>Icon.png</PackageIcon>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>
<None Include="..\..\..\arts\Icon.png" Link="Icon.png">
<PackagePath>\</PackagePath>
<Pack>True</Pack>
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="SharpHook" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
</Project>

40
src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs Normal file
View file

@ -0,0 +1,40 @@
/*****************************************************************************
Copyright 2025 Written by Haiping Chen. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
******************************************************************************/
using System.Threading.Tasks;
namespace BotSharp.Abstraction.ComputerUse.MLTasks;
/// <summary>
/// Interface for computer use
/// </summary>
public interface IComputerUse
{
/// <summary>
/// Capture screen of the specified display
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
Task<ComputerUseOutput> CaptureScreen(ComputerUseArgs args);
Task<ComputerUseOutput> MouseMove(ComputerUseArgs args);
Task<ComputerUseOutput> MouseClick(ComputerUseArgs args);
Task<ComputerUseOutput> InputText(ComputerUseArgs args);
Task<ComputerUseOutput> KeyPress(ComputerUseArgs args);
}

7
src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Abstraction.ComputerUse.MLTasks;
public interface IScreenshot
{
(int x, int y) GetScreenSize(int screenIndex);
byte[] CaptureScreen(int displayId);
}

14
src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs Normal file
View file

@ -0,0 +1,14 @@
namespace BotSharp.Abstraction.ComputerUse.Models;
public class ComputerUseArgs
{
/// <summary>
/// Number of multi-screens
/// </summary>
public int DisplayId { get; set; }
public int X { get; set; }
public int Y { get; set; }
public string Text { get; set; } = string.Empty;
public MouseButton MouseButton { get; set; }
public KeyCode KeyCode { get; set; }
}

9
src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs Normal file
View file

@ -0,0 +1,9 @@
namespace BotSharp.Abstraction.ComputerUse.Models;
public class ComputerUseOutput
{
/// <summary>
/// Base64 encoded image
/// </summary>
public string? Screenshot { get; set; }
}

4
src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs Normal file
View file

@ -0,0 +1,4 @@
global using System.Threading.Tasks;
global using BotSharp.Abstraction.ComputerUse.Models;
global using SharpHook.Native;

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
View file

@ -6,7 +6,7 @@ public interface ILlmProviderService
{
LlmModelSetting GetSetting(string provider, string model);
List<string> GetProviders();
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false);
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false);
List<LlmModelSetting> GetProviderModels(string provider);
List<LlmProviderSetting> GetLlmConfigs(LlmConfigOptions? options = null);
}

1
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs
View file

@ -4,6 +4,5 @@ public class LlmConfigOptions
{
public LlmModelType? Type { get; set; }
public bool? MultiModal { get; set; }
public bool? RealTime { get; set; }
public bool? ImageGeneration { get; set; }
}

8
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -37,11 +37,6 @@ public class LlmModelSetting
/// </summary>
public bool MultiModal { get; set; }
/// <summary>
/// If true, allow real-time interaction
/// </summary>
public bool RealTime { get; set; }
/// <summary>
/// If true, allow generating images
/// </summary>
@ -84,5 +79,6 @@ public enum LlmModelType
Chat = 2,
Image = 3,
Embedding = 4,
Audio = 5
Audio = 5,
Realtime = 6,
}

4
src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs
View file

@ -1,6 +1,10 @@
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Abstraction.Realtime;
public interface IRealtimeHook
{
Task OnModeReady(Agent agent, IRealTimeCompletion completer);
string[] OnModelTranscriptPrompt(Agent agent);
Task OnTranscribeCompleted(RoleDialogModel message, TranscriptionData data);
}

6
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs Normal file
View file

@ -0,0 +1,6 @@
public class TranscriptionData
{
public string Transcript { get; set; } = null!;
public float Confidence { get; set; }
public string Language { get; set; } = null!;
}

41
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -1,3 +1,8 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.MLTasks.Settings;
using BotSharp.Abstraction.Options;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Core.Realtime.Services;
public class RealtimeHub : IRealtimeHub
@ -70,11 +75,11 @@ public class RealtimeHub : IRealtimeHub
_conn.CurrentAgentId = agent.Id;
// Set model
var model = agent.LlmConfig.Model;
if (!model.Contains("-realtime-"))
var model = "gpt-4o-mini-realtime";
if (agent.Profiles.Contains("realtime"))
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name;
model = llmProviderService.GetProviderModel("openai", "gpt-4o", modelType: LlmModelType.Realtime).Name;
}
_completer.SetModelName(model);
@ -102,29 +107,7 @@ public class RealtimeHub : IRealtimeHub
// Not TriggerModelInference, waiting for user utter.
var instruction = await _completer.UpdateSession(_conn);
// Trigger model inference if there is no audio file in the conversation
if (!states.ContainsState("init_audio_file"))
{
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{
await _completer.TriggerModelInference("Reply based on the conversation context.");
}
}
else
{
// Append dialogs into model context
var history = "[CONVERSATION HISTORY]\r\n";
foreach (var message in dialogs)
{
history += $"{message.Role}: {message.Content}\r\n";
}
await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement.");
}
await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModeReady(agent, _completer));
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
{
@ -163,6 +146,12 @@ public class RealtimeHub : IRealtimeHub
if (message.MessageType == MessageTypeName.FunctionCall &&
!string.IsNullOrEmpty(message.FunctionName))
{
if (message.FunctionName == "route_to_agent")
{
var instruction = JsonSerializer.Deserialize<FunctionCallFromLlm>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message));
}
await routing.InvokeFunction(message.FunctionName, message);
}
else

6
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs
View file

@ -6,7 +6,7 @@ namespace BotSharp.Core.Agents.Services;
public partial class AgentService
{
#if !DEBUG
[SharpCache(10, perInstanceCache: true)]
[SharpCache(10)]
#endif
public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
{
@ -28,7 +28,7 @@ public partial class AgentService
}
#if !DEBUG
[SharpCache(10, perInstanceCache: true)]
[SharpCache(10)]
#endif
public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
{
@ -40,7 +40,7 @@ public partial class AgentService
}
#if !DEBUG
[SharpCache(10, perInstanceCache: true)]
[SharpCache(10)]
#endif
public async Task<Agent> GetAgent(string id)
{

10
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -32,6 +32,10 @@ public class CompletionProvider
{
return GetAudioTranscriber(services, provider: provider, model: model);
}
else if (settings.Type == LlmModelType.Realtime)
{
return GetRealTimeCompletion(services, provider: provider, model: model);
}
else
{
return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig);
@ -172,7 +176,7 @@ public class CompletionProvider
var completions = services.GetServices<IRealTimeCompletion>();
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
multiModal: multiModal,
realTime: true,
modelType: LlmModelType.Realtime,
agentConfig: agentConfig);
var completer = completions.FirstOrDefault(x => x.Provider == provider);
@ -191,7 +195,7 @@ public class CompletionProvider
string? model = null,
string? modelId = null,
bool? multiModal = null,
bool realTime = false,
LlmModelType? modelType = null,
bool imageGenerate = false,
AgentLlmConfig? agentConfig = null)
{
@ -217,7 +221,7 @@ public class CompletionProvider
var llmProviderService = services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel(provider, modelIdentity,
multiModal: multiModal,
realTime: realTime,
modelType: modelType,
imageGenerate: imageGenerate)?.Name;
}
}

12
src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
View file

@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService
?.Models ?? new List<LlmModelSetting>();
}
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false)
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false)
{
var models = GetProviderModels(provider)
.Where(x => x.Id == id);
@ -54,7 +54,10 @@ public class LlmProviderService : ILlmProviderService
models = models.Where(x => x.MultiModal == multiModal);
}
models = models.Where(x => x.RealTime == realTime);
if (modelType.HasValue)
{
models = models.Where(x => x.Type == modelType.Value);
}
models = models.Where(x => x.ImageGeneration == imageGenerate);
@ -130,11 +133,6 @@ public class LlmProviderService : ILlmProviderService
models = models.Where(x => x.ImageGeneration == options.ImageGeneration.Value).ToList();
}
if (options.RealTime.HasValue)
{
models = models.Where(x => x.RealTime == options.RealTime.Value).ToList();
}
if (models.IsNullOrEmpty())
{
continue;

5
src/Infrastructure/BotSharp.Logger/Hooks/RateLimitConversationHook.cs
View file

@ -36,6 +36,11 @@ public class RateLimitConversationHook : ConversationHookBase
return;
}
if (Dialogs == null)
{
return;
}
// Check message sending frequency
var userSents = Dialogs.Where(x => x.Role == AgentRole.User)
.TakeLast(2).ToList();

1
src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj
View file

@ -12,7 +12,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc" />
<PackageReference Include="NAudio" />
<PackageReference Include="NAudio.Core" />
<PackageReference Include="Whisper.net" />

9
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -89,7 +89,7 @@ public class NativeWhisperProvider : IAudioTranscription
}
var bytes = _fileStorage.GetFileBytes(modelLoc);
_whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build();
_whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build();
}
catch (Exception ex)
{
@ -101,7 +101,12 @@ public class NativeWhisperProvider : IAudioTranscription
private void DownloadModel(GgmlType modelType, string modelDir)
{
using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult();
// Create an instance of WhisperGgmlDownloader
var downloader = WhisperGgmlDownloader.Default;
// Use the instance to call GetGgmlModelAsync
using var modelStream = downloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult();
_fileStorage.SaveFileStreamToPath(modelDir, modelStream);
modelStream.Close();
}

58
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -49,7 +49,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
var text = part?.Text ?? string.Empty;
RoleDialogModel responseMessage;
if (response.GetFunction()!=null)
if (response.GetFunction() != null)
{
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
@ -78,7 +78,10 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
Prompt = prompt,
Provider = Provider,
Model = _model
Model = _model,
PromptCount = response.UsageMetadata?.PromptTokenCount ?? 0,
CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0,
CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0
});
}
@ -96,8 +99,8 @@ public class GeminiChatCompletionProvider : IChatCompletion
}
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGeminiModel(_model);
var (prompt, messages) = PrepareOptions(chatClient,agent, conversations);
var chatClient = client.CreateGenerativeModel(_model);
var (prompt, messages) = PrepareOptions(chatClient, agent, conversations);
var response = await chatClient.GenerateContentAsync(messages);
@ -120,11 +123,12 @@ public class GeminiChatCompletionProvider : IChatCompletion
Provider = Provider,
Model = _model,
PromptCount = response?.UsageMetadata?.PromptTokenCount ?? 0,
CompletionCount = response?.UsageMetadata?.CandidatesTokenCount ?? 0
CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0,
CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0
});
}
if (response.GetFunction()!=null)
if (response.GetFunction() != null)
{
var toolCall = response.GetFunction();
_logger.LogInformation($"[{agent.Name}]: {toolCall?.Name}({toolCall?.Args?.ToJsonString()})");
@ -167,10 +171,10 @@ public class GeminiChatCompletionProvider : IChatCompletion
await foreach (var response in asyncEnumerable)
{
if (response.GetFunction()!=null)
if (response.GetFunction() != null)
{
var func = response.GetFunction();
var update =func?.Args?.ToJsonString().ToString() ?? string.Empty;
var update = func?.Args?.ToJsonString().ToString() ?? string.Empty;
_logger.LogInformation(update);
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)
@ -212,6 +216,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
AutoCallFunction = false
};
// Assembly messages
var contents = new List<Content>();
var tools = new List<Tool>();
@ -221,7 +226,6 @@ public class GeminiChatCompletionProvider : IChatCompletion
if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
{
var instruction = agentService.RenderedInstruction(agent);
contents.Add(new Content(instruction, AgentRole.User));
renderedInstructions.Add(instruction);
systemPrompts.Add(instruction);
}
@ -237,7 +241,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}" ? new Schema()
{
Type = "object",
Properties = JsonSerializer.Deserialize<dynamic>(props),
Properties = JsonSerializer.Deserialize<Dictionary<string, Schema>>(props),
Required = def?.Required ?? []
} : null;
@ -261,23 +265,32 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
if (message.Role == AgentRole.Function)
{
contents.Add( new Content(message.Content,AgentRole.Function)
{
Role = AgentRole.Function,
Parts =
[
new Part()
contents.Add(new Content([
new Part()
{
FunctionCall = new FunctionCall
{
FunctionCall = new FunctionCall
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
}
}
], AgentRole.Model));
contents.Add(new Content([
new Part()
{
FunctionResponse = new FunctionResponse
{
Name = message.FunctionName,
Response = new JsonObject()
{
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
["result"] = message.Content ?? string.Empty
}
}
]
});
}
], AgentRole.Function));
convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs})");
convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
else if (message.Role == AgentRole.User)
{
@ -299,6 +312,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
{
SystemInstruction = !systemPrompts.IsNullOrEmpty() ? new Content(systemPrompts[0], AgentRole.System) : null,
Contents = contents,
Tools = tools,
GenerationConfig = new()

11
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
View file

@ -51,11 +51,8 @@ public class GeminiTextCompletionProvider : ITextCompletion
var aiModel = client.CreateGenerativeModel(_model);
PrepareOptions(aiModel);
_tokenStatistics.StartTimer();
var response = await aiModel.GenerateContentAsync(text);
_tokenStatistics.StopTimer();
var completion = response.Text ?? string.Empty;
var completion = response?.Text ?? string.Empty;
// After completion hook
foreach (var hook in contentHooks)
@ -64,7 +61,10 @@ public class GeminiTextCompletionProvider : ITextCompletion
{
Prompt = text,
Provider = Provider,
Model = _model
Model = _model,
PromptCount = response.UsageMetadata?.PromptTokenCount ?? 0,
CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0,
CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0
});
}
@ -79,6 +79,7 @@ public class GeminiTextCompletionProvider : ITextCompletion
private void PrepareOptions(GenerativeModel aiModel)
{
var settings = _services.GetRequiredService<GoogleAiSettings>();
aiModel.UseGoogleSearch = settings.Gemini.UseGoogleSearch;
aiModel.UseGrounding = settings.Gemini.UseGrounding;
aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()

1
src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj
View file

@ -16,7 +16,6 @@
<ItemGroup>
<PackageReference Include="FaissMask" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
<PackageReference Include="FastText.NetWrapper" />
</ItemGroup>
</Project>

4
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -163,8 +163,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
if (response.Type == "error")
{
var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
_logger.LogError($"Error: {error.Body.Message}");
_logger.LogError($"{response.Type}: {receivedText}");
}
else if (response.Type == "session.created")
{
@ -677,6 +676,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
Provider = Provider,
Model = _model,
Prompt = "[hook.AfterGenerated] [UNCHANGED PROMPT]",
CompletionCount = data.Usage.OutputTokens,
PromptCount = data.Usage.InputTokens
});

23
src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj Normal file
View file

@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
<Nullable>enable</Nullable>
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SixLabors.ImageSharp" />
<PackageReference Include="System.Drawing.Common" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction.ComputerUse\BotSharp.Abstraction.ComputerUse.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
</Project>

94
src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs Normal file
View file

@ -0,0 +1,94 @@
namespace BotSharp.Plugin.OsDriver;
public enum ScalingSource
{
COMPUTER,
API,
// Add other possible values if needed
}
public class Dimension
{
public int Width { get; set; }
public int Height { get; set; }
public Dimension(int width, int height)
{
Width = width;
Height = height;
}
}
public class ToolError : Exception
{
public ToolError(string message) : base(message)
{
}
}
public class CoordinateScaler
{
private bool _scalingEnabled;
private int _width;
private int _height;
private static readonly List<Dimension> MaxScalingTargets = new List<Dimension>
{
new Dimension(1024, 768), //XGA 4:3
new Dimension(1280, 800), //WXGA 16:10
new Dimension(1366, 768), //FWXGA 16:9
// Add more dimensions as needed
};
public CoordinateScaler(bool scalingEnabled, int width, int height)
{
_scalingEnabled = scalingEnabled;
_width = width;
_height = height;
}
public (int x, int y) ScaleCoordinates(ScalingSource source, int x, int y)
{
if (!_scalingEnabled)
return (x, y);
double ratio = (double)_width / _height;
Dimension targetDimension = null;
foreach (var dimension in MaxScalingTargets)
{
if (Math.Abs((double)dimension.Width / dimension.Height - ratio) < 0.02)
{
if (dimension.Width < _width)
{
targetDimension = dimension;
}
break;
}
}
if (targetDimension == null)
return (x, y);
double xScalingFactor = (double)targetDimension.Width / _width;
double yScalingFactor = (double)targetDimension.Height / _height;
if (source == ScalingSource.API)
{
if (x > _width || y > _height)
throw new ToolError($"Coordinates {x}, {y} are out of bounds");
// Scale up
int newX = (int)Math.Round(x / xScalingFactor);
int newY = (int)Math.Round(y / yScalingFactor);
return (newX, newY);
}
else
{
// Scale down
int newX = (int)Math.Round(x * xScalingFactor);
int newY = (int)Math.Round(y * yScalingFactor);
return (newX, newY);
}
}
}

105
src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs Normal file
View file

@ -0,0 +1,105 @@
using SharpHook.Native;
using SharpHook;
namespace BotSharp.Plugin.OsDriver;
public class KeyboardSimulator
{
public static void SimulateTextInput(string text)
{
if (!string.IsNullOrWhiteSpace(text))
{
var simulator = new EventSimulator();
simulator.SimulateTextEntry(text);
}
}
public static void SimulateKeyCombination(string keyCombination)
{
if (string.IsNullOrWhiteSpace(keyCombination))
throw new ArgumentException("Key combination cannot be null or empty.");
// Split the key combination into individual keys
var keys = keyCombination.Split(new[] { '+', ' ' }, StringSplitOptions.RemoveEmptyEntries);
IEventSimulator simulator = new EventSimulator();
// Stack to keep track of keys pressed
Stack<KeyCode> keysDown = new Stack<KeyCode>();
ModifierMask currentMask = ModifierMask.None;
// Map key names to KeyCode enums
var keyMap = new Dictionary<string, KeyCode>(StringComparer.OrdinalIgnoreCase)
{
{ "Shift", KeyCode.VcLeftShift },
{ "Ctrl", KeyCode.VcLeftControl },
{ "Control", KeyCode.VcLeftControl },
{ "Alt", KeyCode.VcLeftAlt },
{ "LWin", KeyCode.VcLeftMeta },
{ "RWin", KeyCode.VcRightMeta },
{ "Enter", KeyCode.VcEnter },
{ "Return", KeyCode.VcEnter },
{ "Super", KeyCode.VcLeftMeta },
{ "Windows", KeyCode.VcLeftMeta },
// Add other keys as needed
};
// Map modifier keys to ModifierMask enums
var modifierMap = new Dictionary<KeyCode, ModifierMask>
{
{ KeyCode.VcLeftShift, ModifierMask.Shift },
{ KeyCode.VcLeftControl, ModifierMask.Ctrl },
{ KeyCode.VcLeftAlt, ModifierMask.Alt },
{ KeyCode.VcLeftMeta, ModifierMask.Meta },
{ KeyCode.VcRightMeta, ModifierMask.Meta }
};
// Press down all keys
foreach (var key in keys)
{
KeyCode keyCode;
// Check if the key is in the keyMap
if (keyMap.TryGetValue(key, out keyCode))
{
simulator.SimulateKeyPress(keyCode);
// Update the modifier mask if it's a modifier key
if (modifierMap.TryGetValue(keyCode, out ModifierMask modifier))
{
currentMask |= modifier;
}
keysDown.Push(keyCode);
}
else if (key.Length == 1)
{
// Handle single character keys
char c = key.ToUpper()[0];
KeyCode charKeyCode = (KeyCode)Enum.Parse(typeof(KeyCode), $"Vc{c}", true);
simulator.SimulateKeyPress(charKeyCode);
keysDown.Push(charKeyCode);
}
else
{
throw new ArgumentException($"Unknown key: {key}");
}
}
// Release keys in reverse order
while (keysDown.Count > 0)
{
KeyCode keyCode = keysDown.Pop();
// Update the modifier mask if it's a modifier key
if (modifierMap.ContainsKey(keyCode))
{
currentMask &= ~modifierMap[keyCode];
}
simulator.SimulateKeyRelease(keyCode);
}
}
}

15
src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs Normal file
View file

@ -0,0 +1,15 @@
using BotSharp.Abstraction.Plugins;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
namespace BotSharp.Plugin.OsDriver;
public class OsDriverPlugin : IBotSharpPlugin
{
public string Id => "5aef4940-2f95-464b-ad37-43dbf89febf0";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
}
}

155
src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs Normal file
View file

@ -0,0 +1,155 @@
using BotSharp.Abstraction.ComputerUse.MLTasks;
using System.Runtime.InteropServices;
using System.Drawing;
using System.Drawing.Imaging;
namespace BotSharp.Plugin.OsDriver;
public class ScreenshotWinOS : IScreenshot
{
// Define RECT structure
[StructLayout(LayoutKind.Sequential)]
public struct RECT
{
public int Left, Top, Right, Bottom;
}
// Define MONITORINFO structure
[StructLayout(LayoutKind.Sequential)]
public struct MONITORINFO
{
public uint cbSize;
public RECT rcMonitor;
public RECT rcWork;
public uint dwFlags;
}
// Delegate for monitor enumeration callback
private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData);
// P/Invoke declarations
[DllImport("user32.dll")]
private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip,
MonitorEnumDelegate lpfnEnum, IntPtr dwData);
[DllImport("user32.dll", CharSet = CharSet.Auto)]
private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi);
[DllImport("gdi32.dll")]
private static extern IntPtr CreateDC(string lpszDriver, string lpszDevice,
string lpszOutput, IntPtr lpInitData);
[DllImport("gdi32.dll")]
private static extern IntPtr CreateCompatibleDC(IntPtr hdc);
[DllImport("gdi32.dll")]
private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
[DllImport("gdi32.dll")]
private static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
[DllImport("gdi32.dll")]
private static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth,
int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, CopyPixelOperation dwRop);
[DllImport("gdi32.dll")]
private static extern bool DeleteDC(IntPtr hdc);
[DllImport("gdi32.dll")]
private static extern bool DeleteObject(IntPtr hObject);
// Class to hold monitor information
public class MonitorInfo
{
public IntPtr MonitorHandle;
public RECT MonitorArea;
}
public byte[] CaptureScreen(int displayId)
{
// Get all monitors
var monitors = GetMonitors();
// Validate the screen index
if (displayId < 0 || displayId >= monitors.Count)
{
throw new ArgumentOutOfRangeException(nameof(displayId), "Invalid screen index.");
}
var monitor = monitors[displayId];
using var bitmap = CaptureMonitor(monitor);
using var ms = new System.IO.MemoryStream();
bitmap.Save(ms, ImageFormat.Jpeg);
return ms.ToArray();
}
// Method to capture a specific monitor
public static Bitmap CaptureMonitor(MonitorInfo monitor)
{
int width = monitor.MonitorArea.Right - monitor.MonitorArea.Left;
int height = monitor.MonitorArea.Bottom - monitor.MonitorArea.Top;
IntPtr hdcSrc = CreateDC("DISPLAY", null, null, IntPtr.Zero);
IntPtr hdcDest = CreateCompatibleDC(hdcSrc);
IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, width, height);
IntPtr hOld = SelectObject(hdcDest, hBitmap);
bool success = BitBlt(hdcDest, 0, 0, width, height, hdcSrc,
monitor.MonitorArea.Left, monitor.MonitorArea.Top, CopyPixelOperation.SourceCopy | CopyPixelOperation.CaptureBlt);
Bitmap bmp = null;
if (success)
{
bmp = Image.FromHbitmap(hBitmap);
}
// Clean up
SelectObject(hdcDest, hOld);
DeleteObject(hBitmap);
DeleteDC(hdcDest);
DeleteDC(hdcSrc);
return bmp;
}
public (int x, int y) GetScreenSize(int screenIndex)
{
// Get all monitors
var monitors = GetMonitors();
// Validate the screen index
if (screenIndex < 0 || screenIndex >= monitors.Count)
{
throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index.");
}
var monitor = monitors[screenIndex];
return (monitor.MonitorArea.Right - monitor.MonitorArea.Left, monitor.MonitorArea.Bottom - monitor.MonitorArea.Top);
}
// Method to get all connected monitors
public static List<MonitorInfo> GetMonitors()
{
var monitors = new List<MonitorInfo>();
bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData)
{
var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) };
if (GetMonitorInfo(hMonitor, ref mi))
{
monitors.Add(new MonitorInfo
{
MonitorHandle = hMonitor,
MonitorArea = mi.rcMonitor
});
}
return true; // Continue enumeration
}
EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero);
return monitors;
}
}

4
src/Plugins/BotSharp.Plugin.OsDriver/Using.cs Normal file
View file

@ -0,0 +1,4 @@
global using System;
global using System.Collections.Generic;
global using System.IO;
global using System.Threading.Tasks;

87
src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs Normal file
View file

@ -0,0 +1,87 @@
using BotSharp.Abstraction.ComputerUse.MLTasks;
using BotSharp.Abstraction.ComputerUse.Models;
using SharpHook;
using SixLabors.ImageSharp.Formats.Jpeg;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
namespace BotSharp.Plugin.OsDriver;
public class WinOSDriver : IComputerUse
{
public static void TakeAction(string action, string? text, Tuple<int, int>? coordinate, int monitorIndex, CoordinateScaler coordScaler)
{
switch (action)
{
case "type":
KeyboardSimulator.SimulateTextInput(text);
break;
case "key":
KeyboardSimulator.SimulateKeyCombination(text);
break;
case "mouse_move":
var scaledCoord = coordScaler.ScaleCoordinates(ScalingSource.API, coordinate.Item1, coordinate.Item2);
WindowsMouseController.SetCursorPositionOnMonitor(monitorIndex, scaledCoord.Item1, scaledCoord.Item2);
break;
case "left_click":
WindowsMouseController.LeftClick();
break;
case "right_click":
WindowsMouseController.RightClick();
break;
default:
throw new ToolError($"Action {action} is not supported");
}
}
public static string DownscaleScreenshot(byte[] screenshot, int scaledX, int scaledY)
{
// Convert Bitmap to MemoryStream
using var memoryStream = new MemoryStream(screenshot);
memoryStream.Position = 0; // Reset stream position
// Load the image into ImageSharp
using var image = SixLabors.ImageSharp.Image.Load<Rgba32>(memoryStream);
// Resize the image to scaled dimensions
image.Mutate(x => x.Resize(scaledX, scaledY));
// Save the image
using var ms = new MemoryStream();
image.Save(ms, new JpegEncoder());
ms.Position = 0; // Reset stream position
//convert to byte 64 string
byte[] imageBytes = ms.ToArray();
return Convert.ToBase64String(imageBytes);
}
public Task<ComputerUseOutput> CaptureScreen(ComputerUseArgs args)
{
throw new NotImplementedException();
}
public Task<ComputerUseOutput> MouseMove(ComputerUseArgs args)
{
throw new NotImplementedException();
}
public Task<ComputerUseOutput> MouseClick(ComputerUseArgs args)
{
var simulator = new EventSimulator();
simulator.SimulateMousePress(args.MouseButton);
simulator.SimulateMouseRelease(args.MouseButton);
return null;
}
public Task<ComputerUseOutput> InputText(ComputerUseArgs args)
{
throw new NotImplementedException();
}
public Task<ComputerUseOutput> KeyPress(ComputerUseArgs args)
{
throw new NotImplementedException();
}
}

135
src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs Normal file
View file

@ -0,0 +1,135 @@
using SharpHook;
using SharpHook.Native;
using System.Drawing;
using System.Runtime.InteropServices;
namespace BotSharp.Plugin.OsDriver;
public class WindowsMouseController
{
[StructLayout(LayoutKind.Sequential)]
public struct RECT
{
public int Left, Top, Right, Bottom;
}
// Define MONITORINFO structure
[StructLayout(LayoutKind.Sequential)]
public struct MONITORINFO
{
public uint cbSize;
public RECT rcMonitor;
public RECT rcWork;
public uint dwFlags;
}
// Delegate for monitor enumeration callback
private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData);
// P/Invoke declarations
[DllImport("user32.dll")]
private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip,
MonitorEnumDelegate lpfnEnum, IntPtr dwData);
[DllImport("user32.dll", CharSet = CharSet.Auto)]
private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi);
// Class to hold monitor information
public class MonitorInfo
{
public IntPtr MonitorHandle;
public RECT MonitorArea;
}
// Method to get all connected monitors
public static List<MonitorInfo> GetMonitors()
{
var monitors = new List<MonitorInfo>();
bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData)
{
var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) };
if (GetMonitorInfo(hMonitor, ref mi))
{
monitors.Add(new MonitorInfo
{
MonitorHandle = hMonitor,
MonitorArea = mi.rcMonitor
});
}
return true; // Continue enumeration
}
EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero);
return monitors;
}
public static (int virtualX, int virtualY) GetVirtualCoordinates(int screenIndex, int x, int y)
{
// Get all screens
var monitors = GetMonitors();
// Validate the screen index
if (screenIndex < 0 || screenIndex >= monitors.Count)
{
throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index.");
}
var monitor = monitors[screenIndex];
var monitorBounds = new Rectangle(monitor.MonitorArea.Left, monitor.MonitorArea.Top,
monitor.MonitorArea.Right - monitor.MonitorArea.Left,
monitor.MonitorArea.Bottom - monitor.MonitorArea.Top);
// Validate coordinates
if (x < 0 || x >= monitorBounds.Width || y < 0 || y >= monitorBounds.Height)
{
throw new ArgumentOutOfRangeException("Coordinates are out of bounds for the specified monitor.");
}
// Convert to virtual screen coordinates
int virtualX = (int)(monitorBounds.X + x);
int virtualY = (int)(monitorBounds.Y + y);
return (virtualX, virtualY);
}
// Method to set the cursor position on a specific monitor
public static void SetCursorPositionOnMonitor(int monitorIndex, int x, int y)
{
var (virtualX, virtualY) = GetVirtualCoordinates(monitorIndex, x, y);
IEventSimulator simulator = new EventSimulator();
// Move the cursor to the specified position
simulator.SimulateMouseMovement(Convert.ToInt16(virtualX), Convert.ToInt16(virtualY));
}
// Method to perform a left-click at the current cursor position
public static void LeftClick()
{
IEventSimulator simulator = new EventSimulator();
simulator.SimulateMousePress(MouseButton.Button1);
simulator.SimulateMouseRelease(MouseButton.Button1);
}
public static void RightClick()
{
IEventSimulator simulator = new EventSimulator();
simulator.SimulateMousePress(MouseButton.Button2);
simulator.SimulateMouseRelease(MouseButton.Button2);
}
// Method to move the cursor and perform a left-click at specified coordinates on a specific monitor
public static void ClickAtPositionOnMonitor(int monitorIndex, int x, int y)
{
SetCursorPositionOnMonitor(monitorIndex, x, y);
System.Threading.Thread.Sleep(50); // Optional delay to ensure the cursor has moved
LeftClick();
}
}

1
src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
View file

@ -100,7 +100,6 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
<PackageReference Include="MySqlConnector" />
<PackageReference Include="Npgsql" />
</ItemGroup>

24
src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs
View file

@ -1,24 +0,0 @@
using BotSharp.Plugin.SqlDriver.Models;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
namespace BotSharp.Plugin.SqlDriver.Controllers;
[Authorize]
[ApiController]
public class SqlDriverController : ControllerBase
{
private readonly IServiceProvider _services;
public SqlDriverController(IServiceProvider services)
{
_services = services;
}
[HttpPost("/knowledge/database/import")]
public async Task<bool> ImportDbKnowledge(ImportDbKnowledgeRequest request)
{
var dbKnowledge = _services.GetRequiredService<DbKnowledgeService>();
return await dbKnowledge.Import(request);
}
}

186
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs Normal file
View file

@ -0,0 +1,186 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Twilio.Http;
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioInboundController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
public TwilioInboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioOutboundController> logger)
{
_settings = settings;
_services = services;
_context = context;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/inbound")]
public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
{
if (request?.CallSid == null)
{
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true,
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
// Load agent profile
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(request.AgentId);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
request.ConversationId = await InitConversation(request, agent);
instruction.AgentId = request.AgentId;
instruction.ConversationId = request.ConversationId;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
if (agent.Profiles.Contains("realtime"))
{
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent);
}
else
{
if (string.IsNullOrWhiteSpace(request.Intent))
{
instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}";
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
else
{
int seqNum = 0;
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = seqNum,
Content = request.Intent,
From = request.From,
States = ParseStates(request.States)
};
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
// delay 3 seconds to wait for the first message reply and caller is listening dudu sound
await Task.Delay(1000 * 3);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post);
}
}
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
});
return TwiML(response);
}
protected Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();
if (states is null || !states.Any())
{
return result;
}
foreach (var kvp in states)
{
var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 2)
{
result.Add(parts[0], parts[1]);
}
}
return result;
}
private async Task<string> InitConversation(ConversationalVoiceRequest request, Agent agent)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
if (conversation == null)
{
var conv = new Conversation
{
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",
Tags = [],
};
conversation = await convService.NewConversation(conv);
}
var states = new List<MessageState>
{
new("channel", ConversationChannel.Phone),
new("calling_phone", request.From),
new("phone_direction", request.Direction),
new("twilio_call_sid", request.CallSid),
};
// Enable lazy routing mode to optimize realtime experience
if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing)
{
states.Add(new(StateConst.ROUTING_MODE, "lazy"));
}
if (request.InitAudioFile != null)
{
states.Add(new("init_audio_file", request.InitAudioFile));
}
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
return conversation.Id;
}
}

31
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs
View file

@ -1,12 +1,8 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Cors.Infrastructure;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using System.IO;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -45,31 +41,4 @@ public class TwilioRecordController : TwilioController
return Ok();
}
[ValidateRequest]
[HttpPost("twilio/record/transcribe")]
public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
{
if (request.Final == "true")
{
_logger.LogError($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
// transcription completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnTranscribeCompleted(request));
// Append the transcription to the dialog history
var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
{
var storage = _services.GetRequiredService<IConversationStorage>();
var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
{
CurrentAgentId = request.AgentId
};
storage.Append(request.ConversationId, message);
}
}
return Ok();
}
}

125
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -1,125 +0,0 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioStreamController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioStreamController> logger)
{
_settings = settings;
_services = services;
_context = context;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/stream")]
public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
{
var text = JsonSerializer.Serialize(request);
if (request?.CallSid == null)
{
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
request.ConversationId = await InitConversation(request);
instruction.AgentId = request.AgentId;
instruction.ConversationId = request.ConversationId;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
});
return TwiML(response);
}
private async Task<string> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
if (conversation == null)
{
var conv = new Conversation
{
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",
Tags = [],
};
conversation = await convService.NewConversation(conv);
}
var states = new List<MessageState>
{
new("channel", ConversationChannel.Phone),
new("calling_phone", request.From),
new("twilio_call_sid", request.CallSid),
// Enable lazy routing mode to optimize realtime experience
new(StateConst.ROUTING_MODE, "lazy"),
};
if (request.InitAudioFile != null)
{
states.Add(new("init_audio_file", request.InitAudioFile));
}
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
return conversation.Id;
}
}

58
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs Normal file
View file

@ -0,0 +1,58 @@
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioTranscribeController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public TwilioTranscribeController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioRecordController> logger)
{
_settings = settings;
_services = services;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/transcribe")]
public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
{
if (request.Final == "true")
{
_logger.LogInformation($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
// Append the transcription to the dialog history
var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
{
var storage = _services.GetRequiredService<IConversationStorage>();
var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
{
CurrentAgentId = request.AgentId
};
storage.Append(request.ConversationId, message);
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.SetMessageId(request.ConversationId, message.MessageId);
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(request.ConversationId, []);
// transcription completed
transcript.Language = request.LanguageCode;
await HookEmitter.Emit<IRealtimeHook>(_services, async x => await x.OnTranscribeCompleted(message, transcript));
}
}
return Ok();
}
}

15
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -12,10 +12,10 @@ namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioVoiceController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
protected readonly TwilioSetting _settings;
protected readonly IServiceProvider _services;
protected readonly IHttpContextAccessor _context;
protected readonly ILogger _logger;
public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioVoiceController> logger)
{
@ -32,6 +32,7 @@ public class TwilioVoiceController : TwilioController
/// <param name="states"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
[Obsolete("Use twilio/inbound for streaming and non-streaming instead of this endpoint")]
[ValidateRequest]
[HttpPost("twilio/voice/welcome")]
public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest request)
@ -352,11 +353,15 @@ public class TwilioVoiceController : TwilioController
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallBusyStatus(request));
}
else if (request.CallStatus == "no-answer")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallNoAnswerStatus(request));
}
return Ok();
}
private Dictionary<string, string> ParseStates(List<string> states)
protected Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();
if (states is null || !states.Any())

3
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
View file

@ -8,7 +8,6 @@ public interface ITwilioCallStatusHook
Task OnVoicemailLeft(ConversationalVoiceRequest request);
Task OnUserDisconnected(ConversationalVoiceRequest request);
Task OnRecordingCompleted(ConversationalVoiceRequest request);
Task OnTranscribeCompleted(ConversationalVoiceRequest request);
Task OnVoicemailStarting(ConversationalVoiceRequest request);
/// <summary>
@ -19,4 +18,6 @@ public interface ITwilioCallStatusHook
/// <param name="request"></param>
/// <returns></returns>
Task OnCallBusyStatus(ConversationalVoiceRequest request);
Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
}

5
src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs
View file

@ -64,9 +64,4 @@ public class ConversationalVoiceRequest : VoiceRequest
[FromForm]
public string? TranscriptionEvent { get; set; }
#endregion
}
public class TranscriptionData
{
public string Transcript { get; set; } = null!;
}

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -29,8 +29,8 @@ public class HangupPhoneCallFn : IFunctionCallback
var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var convService = _services.GetRequiredService<IConversationService>();
var conversationId = convService.ConversationId;
var states = _services.GetRequiredService<IConversationStateService>();
var callSid = states.GetState("twilio_call_sid");

10
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
View file

@ -77,10 +77,14 @@ public class OutboundPhoneCallFn : IFunctionCallback
statusUrl += $"&init-audio-file={initAudioFile}";
}
// load agent profile
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(message.CurrentAgentId);
// Set up process URL streaming or synchronous
if (_twilioSetting.StreamingEnabled)
if (agent.Profiles.Contains("realtime"))
{
processUrl += "/stream";
processUrl += "/inbound";
}
else
{
@ -119,7 +123,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.StopCompletion = true;
return true;
}

11
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
@ -199,7 +201,7 @@ public class TwilioService
/// </summary>
/// <param name="conversationalVoiceResponse"></param>
/// <returns></returns>
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse, Agent agent)
{
var response = new VoiceResponse();
@ -207,11 +209,16 @@ public class TwilioService
if (_settings.TranscribeEnabled)
{
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var hints = string.Join(", ", words);
var start = new Start();
start.Transcription(
track: "inbound_track",
partialResults: false,
statusCallbackUrl: $"{_settings.CallbackHost}/twilio/record/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", name: conversationId);
statusCallbackUrl: $"{_settings.CallbackHost}/twilio/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}",
name: conversationId,
hints: hints);
response.Append(start);
}

4
src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
View file

@ -7,10 +7,6 @@ public class TwilioSetting
/// </summary>
public string? PhoneNumber { get; set; }
/// <summary>
/// Enable streaming for outbound phone call
/// </summary>
public bool StreamingEnabled { get; set; } = false;
public string AccountSID { get; set; }
public string AuthToken { get; set; }
public string AppSID { get; set; }

2
src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
View file

@ -1,9 +1,7 @@
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
using BotSharp.Plugin.Twilio.Services;
using BotSharp.Plugin.Twilio.Services.Stream;
using StackExchange.Redis;
using Twilio;

2
src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs → src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs
View file

@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Http;
using System.Net.WebSockets;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services.Stream;
namespace BotSharp.Plugin.Twilio;
/// <summary>
/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js

2
src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

10
tests/BotSharp.Test.BrowserUse/BotSharp.Test.BrowserUse.csproj Normal file
View file

@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

2
tests/BotSharp.Test.BrowserUse/Program.cs Normal file
View file

@ -0,0 +1,2 @@
// See https://aka.ms/new-console-template for more information
Console.WriteLine("Hello, World!");

19
tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj Normal file
View file

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Anthropic.SDK" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Plugins\BotSharp.Plugin.AnthropicAI\BotSharp.Plugin.AnthropicAI.csproj" />
<ProjectReference Include="..\..\src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj" />
</ItemGroup>
</Project>

135
tests/BotSharp.Test.ComputerUse/Program.cs Normal file
View file

@ -0,0 +1,135 @@
using Anthropic.SDK.Constants;
using Anthropic.SDK.Messaging;
using Anthropic.SDK;
using System.Text.Json.Nodes;
using BotSharp.Plugin.OsDriver;
using Anthropic.SDK.Common;
Console.WriteLine("Open Google Chrome in Incognito Mode in a given window. Enter which disaplay ID you are using:");
var displayId = Convert.ToInt32(Console.ReadLine());
var capturer = new ScreenshotWinOS();
var (width, height) = capturer.GetScreenSize(displayId - 1);
Console.WriteLine($"Screen Size: {width}x{height}");
var coordScaler = new CoordinateScaler(true, width, height);
var (scaledX, scaledY) = coordScaler.ScaleCoordinates(ScalingSource.COMPUTER, width, height);
Console.WriteLine($"Scaled Screen Size: {scaledX}x{scaledY}");
var client = new AnthropicClient(new APIAuthentication(""));
var messages = new List<Message>();
messages.Add(new Message()
{
Role = RoleType.User,
Content = new List<ContentBase>()
{
new TextContent()
{
Text = """
Open google.com to search blogs about BotSharp, find the most recent updates on it and output the summary.
"""
}
}
});
var tools = new List<Anthropic.SDK.Common.Tool>()
{
new Function("computer", "computer_20241022",new Dictionary<string, object>()
{
{"display_width_px", scaledX },
{"display_height_px", scaledY },
{"display_number", displayId }
})
};
var parameters = new MessageParameters()
{
Messages = messages,
Model = AnthropicModels.Claude35Sonnet,
Stream = false,
Tools = tools,
System = new List<SystemMessage>()
{
new SystemMessage($""""
A Google Chrome Incognito window is already open and maximized in the appropriate monitor. Use that instance.
"""")
}
};
var isRunning = true;
var res = await client.Messages.GetClaudeMessageAsync(parameters);
messages.Add(res.Message);
while (isRunning)
{
var toolUse = res.Content.OfType<ToolUseContent>().ToList();
if (toolUse.Count == 0)
{
isRunning = false;
break;
}
var cb = new List<ContentBase>();
foreach (var tool in toolUse)
{
var action = tool.Input["action"].ToString();
var text = tool.Input["text"]?.ToString();
var coordinate = tool.Input["coordinate"] as JsonArray;
switch (action)
{
case "screenshot":
messages.Add(new Message()
{
Role = RoleType.User,
Content = new List<ContentBase>()
{
new ToolResultContent()
{
ToolUseId = tool.Id,
Content =new List<ContentBase>() { new ImageContent()
{
Source = new ImageSource() {
Data = WinOSDriver.DownscaleScreenshot(capturer.CaptureScreen(displayId -1), scaledX, scaledY),
MediaType = "image/jpeg"
}
} }
}
}
});
break;
default:
WinOSDriver.TakeAction(action, text,
coordinate == null ? null : new Tuple<int, int>(Convert.ToInt32(coordinate[0].ToString()),
Convert.ToInt32(coordinate[1].ToString())), displayId - 1, coordScaler);
await Task.Delay(1000);
cb.Add(new ToolResultContent()
{
ToolUseId = tool.Id,
Content = new List<ContentBase>()
{
new TextContent()
{
Text = "Action completed"
}
}
});
break;
}
}
messages.Add(new Message()
{
Role = RoleType.User,
Content = cb
});
res = await client.Messages.GetClaudeMessageAsync(parameters);
messages.Add(res.Message);
}
Console.WriteLine("----------------------------------------------");
Console.WriteLine("Final Result:");
Console.WriteLine(messages.Last().Content.OfType<TextContent>().First().Text);
Console.ReadLine();