diff --git a/BotSharp.sln b/BotSharp.sln index 8dda48e8..d281450e 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -449,6 +451,14 @@ Global {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -503,6 +513,7 @@ Global {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} + {F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 6e8a2ae3..2a2705db 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 94c9e27c..697f94a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 2018d6a7..8484c5fb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Conversations.Models; @@ -75,6 +74,9 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? RichContent { get; set; } + /// + /// Rich content for secondary language + /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? SecondaryRichContent { get; set; } @@ -86,6 +88,9 @@ public class RoleDialogModel : ITrackableMessage public FunctionCallFromLlm Instruction { get; set; } + /// + /// Files to be used in conversation + /// public List Files { get; set; } = new List(); /// @@ -94,7 +99,8 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); - public float KnowledgeConfidence { get; set; } = 0.5f; + + private RoleDialogModel() { } @@ -134,8 +140,7 @@ public class RoleDialogModel : ITrackableMessage Payload = source.Payload, StopCompletion = source.StopCompletion, Instruction = source.Instruction, - Data = source.Data, - KnowledgeConfidence = source.KnowledgeConfidence + Data = source.Data }; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 87df6137..24b28402 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { - public string Name { get; } + public string Provider { get; } /// /// Convert pdf pages to images, and return a list of image file paths diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index 10ccd1a0..dc4ac40b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public string Pdf2TextConverter { get; set; } - public string Pdf2ImageConverter { get; set; } + public SettingBase Pdf2TextConverter { get; set; } + public SettingBase Pdf2ImageConverter { get; set; } } + +public class SettingBase +{ + public string Provider { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 78a400e1..f8f4c605 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + Task VaryImage(string? provider, string? model, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask); #endregion #region Pdf @@ -17,7 +17,11 @@ public interface IFileInstructService /// /// Pdf files /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + #endregion + + #region Audio + Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index a3e5a32d..373466c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,6 +6,7 @@ public interface IFileStorageService { #region Common string GetDirectory(string conversationId); + IEnumerable GetFiles(string relativePath, string? searchQuery = null); byte[] GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); bool SaveFileBytesToPath(string filePath, byte[] bytes); @@ -16,7 +17,6 @@ public interface IFileStorageService string BuildDirectory(params string[] segments); #endregion - #region Conversation /// /// Get the message file screenshots for specific content types, e.g., pdf @@ -24,7 +24,7 @@ public interface IFileStorageService /// /// /// - Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -37,7 +37,7 @@ public interface IFileStorageService IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); - bool SaveMessageFiles(string conversationId, string messageId, string source, List files); + bool SaveMessageFiles(string conversationId, string messageId, string source, List files); /// /// Delete files under messages @@ -54,10 +54,11 @@ public interface IFileStorageService #region User string GetUserAvatar(); - bool SaveUserAvatar(BotSharpFile file); + bool SaveUserAvatar(InputFileModel file); #endregion + #region Speech - Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); - Task RetrieveSpeechFileAsync(string conversationId, string fileName); + bool SaveSpeechFile(string conversationId, string fileName, BinaryData data); + BinaryData GetSpeechFile(string conversationId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 11a11e46..bc670626 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,12 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileBase +public class BotSharpFile : FileInformation { - + /// + /// File data => format: "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index 3483921a..f952e7f9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models; public class FileBase { - /// - /// External file url - /// - [JsonPropertyName("file_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileUrl { get; set; } = string.Empty; - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileStorageUrl { get; set; } = string.Empty; - /// /// File name without extension /// @@ -29,18 +15,4 @@ public class FileBase [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; - - /// - /// File content type - /// - [JsonPropertyName("content_type")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ContentType { get; set; } = string.Empty; - - /// - /// File extension without dot - /// - [JsonPropertyName("file_extension")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs new file mode 100644 index 00000000..fe767e16 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileInformation +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs new file mode 100644 index 00000000..7eb350e3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InputFileModel : FileBase +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public new string FileName { get; set; } = string.Empty; + + /// + /// File data => format: "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public new string FileData { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs deleted file mode 100644 index 29749bce..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class InputMessageFiles -{ - public List Files { get; set; } = new List(); - public BotSharpFile? Mask { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs new file mode 100644 index 00000000..7eaddd5a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InstructFileModel : FileBase +{ + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; + + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 2a0128b6..da2ddec6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileBase +public class MessageFileModel : FileInformation { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index df33906d..bba60b10 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,4 +1,6 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.StaticFiles; +using System.IO; namespace BotSharp.Abstraction.Files.Utilities; @@ -26,11 +28,30 @@ public static class FileUtility return (contentType, Convert.FromBase64String(base64Str)); } - public static string GetFileContentType(string filePath) + public static string BuildFileDataFromFile(string fileName, byte[] bytes) + { + var contentType = GetFileContentType(fileName); + var base64 = Convert.ToBase64String(bytes); + return $"data:{contentType};base64,{base64}"; + } + + public static string BuildFileDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var contentType = GetFileContentType(file.FileName); + var base64 = Convert.ToBase64String(stream.ToArray()); + stream.Close(); + + return $"data:{contentType};base64,{base64}"; + } + + public static string GetFileContentType(string fileName) { string contentType; var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) + if (!provider.TryGetContentType(fileName, out contentType)) { contentType = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs index 64c5388b..b15503e2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph; public interface IGraphDb { - public string Name { get; } + public string Provider { get; } Task Search(string query, GraphSearchOptions options); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 20825791..1021ea1c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + #region Vector + Task CreateVectorCollection(string collectionName, int dimension); + Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); @@ -12,6 +15,10 @@ public interface IKnowledgeService Task DeleteVectorCollectionData(string collectionName, string id); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); + #endregion + + #region Graph Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index b8f2d47b..d6edad54 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - public string Name { get; } + public string Provider { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs deleted file mode 100644 index 7d06a8a4..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs +++ /dev/null @@ -1,11 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; - -namespace BotSharp.Abstraction.Knowledges; - -/// -/// Chop large content into chunks -/// -public interface ITextChopper -{ - List Chop(string content, ChunkOption option); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 963a2b3b..217dafb9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { - public string VectorDb { get; set; } - public string GraphDb { get; set; } + public SettingBase VectorDb { get; set; } + public SettingBase GraphDb { get; set; } public DefaultKnowledgeBaseSetting Default { get; set; } public List Collections { get; set; } = new(); @@ -23,9 +23,13 @@ public class VectorCollectionSetting public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; } } -public class KnowledgeTextEmbeddingSetting +public class KnowledgeTextEmbeddingSetting : SettingBase { - public string Provider { get; set; } public string Model { get; set; } public int Dimension { get; set; } +} + +public class SettingBase +{ + public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs new file mode 100644 index 00000000..54ed2d7d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -0,0 +1,13 @@ +using System.IO; + +namespace BotSharp.Abstraction.MLTasks; + +public interface IAudioCompletion +{ + string Provider { get; } + + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); + Task GenerateAudioFromTextAsync(string text); + + void SetModelName(string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs deleted file mode 100644 index 9e0dd574..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks; - -public interface ISpeechToText -{ - string Provider { get; } - - Task GenerateTextFromAudioAsync(string filePath); - // Task AudioToTextTranscript(Stream stream); - Task SetModelName(string modelType); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs deleted file mode 100644 index 344fad0e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks -{ - public interface ITextToSpeech - { - /// - /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI - /// - string Provider { get; } - - /// - /// Set model name, one provider can consume different model or version(s) - /// - /// deployment name - void SetModelName(string model); - - Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); - } - - public interface ITextToSpeechOptions - { - - } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index cc7cfba0..008f528b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -68,5 +68,6 @@ public enum LlmModelType Text = 1, Chat = 2, Image = 3, - Embedding = 4 + Embedding = 4, + Audio = 5 } diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs new file mode 100644 index 00000000..f981ba6e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Models; + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs index 24e4152c..9ea5917c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Models; -public class MessageConfig : InputMessageFiles +public class MessageConfig { /// /// Completion Provider @@ -15,7 +15,7 @@ public class MessageConfig : InputMessageFiles public virtual string? Model { get; set; } = null; /// - /// Model name + /// Model id /// [JsonPropertyName("model_id")] public virtual string? ModelId { get; set; } = null; @@ -34,7 +34,7 @@ public class MessageConfig : InputMessageFiles /// /// Conversation states from input /// - public List States { get; set; } = new List(); + public List States { get; set; } = new(); /// /// Agent task id diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs new file mode 100644 index 00000000..ec519f04 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.VectorStorage.Extensions; + +public static class VectorStorageExtension +{ + public static string ToQuestionAnswer(this VectorSearchResult data) + { + if (data?.Data == null) return string.Empty; + + return $"Question: {data.Data[KnowledgePayloadName.Text]}\r\nAnswer: {data.Data[KnowledgePayloadName.Answer]}"; + } + + public static string ToPayloadPair(this VectorSearchResult data, IList payloads) + { + if (data?.Data == null || payloads.IsNullOrEmpty()) return string.Empty; + + var results = data.Data.Where(x => payloads.Contains(x.Key)) + .OrderBy(x => payloads.IndexOf(x.Key)) + .Select(x => + { + return $"{x.Key}: {x.Value}"; + }) + .ToList(); + + return string.Join("\r\n", results.Where(x => !string.IsNullOrWhiteSpace(x))); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 53b921a1..cedacb46 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,12 +4,13 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - string Name { get; } + string Provider { get; } Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dim); + Task CreateCollection(string collectionName, int dimension); + Task DeleteCollection(string collectionName); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, Guid id); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index 91abcc25..85b9dec2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -4,4 +4,7 @@ public class VectorFilter : StringIdPagination { [JsonPropertyName("with_vector")] public bool WithVector { get; set; } -} + + [JsonPropertyName("search_pairs")] + public IEnumerable? SearchPairs { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs index ce39edbf..f2c582ce 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorSearchResult : VectorCollectionData diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 05d485b7..ff10a517 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index f06e2ba2..05653d9c 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -6,6 +6,7 @@ using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Settings; using BotSharp.Abstraction.Templating; using BotSharp.Core.Instructs; +using BotSharp.Core.Knowledges.Services; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; using BotSharp.Core.Templating; @@ -54,6 +55,8 @@ public class ConversationPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } public bool AttachMenu(List menu) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs new file mode 100644 index 00000000..e2aa844c --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -0,0 +1,20 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) + { + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var audioBytes = await DownloadFile(audio); + using var stream = new MemoryStream(); + stream.Write(audioBytes, 0, audioBytes.Length); + stream.Position = 0; + + var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; + var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text); + stream.Close(); + return content; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index c9d35cb7..cd65c16a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -14,10 +14,10 @@ public partial class FileInstructService { new RoleDialogModel(AgentRole.User, text) { - Files = images?.ToList() ?? new List() + Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() } }); - return message; + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text) @@ -30,7 +30,7 @@ public partial class FileInstructService return message; } - public async Task VaryImage(string? provider, string? model, BotSharpFile image) + public async Task VaryImage(string? provider, string? model, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -43,16 +43,17 @@ public partial class FileInstructService stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -65,16 +66,17 @@ public partial class FileInstructService stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask) { if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) @@ -94,10 +96,12 @@ public partial class FileInstructService maskStream.Write(maskBytes, 0, maskBytes.Length); maskStream.Position = 0; + var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; + var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); @@ -105,7 +109,7 @@ public partial class FileInstructService } #region Private methods - private async Task DownloadFile(BotSharpFile file) + private async Task DownloadFile(InstructFileModel file) { var bytes = new byte[0]; if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2aec257b..29dfdaf9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { var content = string.Empty; @@ -50,7 +50,7 @@ public partial class FileInstructService } #region Private methods - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { @@ -80,9 +80,9 @@ public partial class FileInstructService var fileDir = _fileStorage.BuildDirectory(dir, guid); DeleteIfExistDirectory(fileDir, true); - var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileStorage.SaveFileBytesToPath(pdfDir, bytes); - locs.Add(pdfDir); + var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(outputDir, bytes); + locs.Add(outputDir); } } catch (Exception ex) @@ -98,7 +98,7 @@ public partial class FileInstructService { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index acd0ddaa..416d9f30 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index e90c7812..1aaba575 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -1,28 +1,38 @@ using System.IO; -namespace BotSharp.Core.Files.Services +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService { - public partial class LocalFileStorageService + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + try { var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } - var filePath = Path.Combine(dir, fileName); - if (File.Exists(filePath)) return; - using var file = File.Create(filePath); - using var input = data.ToStream(); - await input.CopyToAsync(file); - } - public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + var filePath = Path.Combine(dir, fileName); + if (File.Exists(filePath)) return false; + + using var fs = File.Create(filePath); + using var ds = data.ToStream(); + ds.CopyTo(fs); + return true; + } + catch (Exception ex) { - var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); - using var file = new FileStream(path, FileMode.Open, FileAccess.Read); - return await BinaryData.FromStreamAsync(file); + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; } } + + public BinaryData GetSpeechFile(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return BinaryData.FromStream(file); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c49edbce..b16c2c2c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,6 +14,22 @@ public partial class LocalFileStorageService return dir; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + return Enumerable.Empty(); + } + + var path = Path.Combine(_baseDir, relativePath); + + if (!string.IsNullOrWhiteSpace(searchPattern)) + { + return Directory.GetFiles(path, searchPattern); + } + return Directory.GetFiles(path); + } + public byte[] GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 8c2ed831..5638179c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) @@ -118,7 +118,7 @@ public partial class LocalFileStorageService return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; @@ -276,7 +276,7 @@ public partial class LocalFileStorageService private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index 43ff9eed..d1e962cd 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -16,7 +16,7 @@ public partial class LocalFileStorageService return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 94ee2beb..750803c4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _audioTypes = new List - { - "mp3", - "wav" - }; - private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 874a063e..d6746e09 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -28,6 +28,10 @@ public class CompletionProvider { return GetImageCompletion(services, provider: provider, model: model); } + else if (settings.Type == LlmModelType.Audio) + { + return GetAudioCompletion(services, provider: provider, model: model); + } else { return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig); @@ -108,7 +112,7 @@ public class CompletionProvider if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve completion provider by {provider}"); + logger.LogError($"Can't resolve text-embedding provider by {provider}"); } @@ -120,35 +124,19 @@ public class CompletionProvider return completer; } - public static ITextToSpeech GetTextToSpeech( + public static IAudioCompletion GetAudioCompletion( IServiceProvider services, string provider, string model) { - var completions = services.GetServices(); + var completions = services.GetServices(); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve text2speech provider by {provider}"); + logger.LogError($"Can't resolve audio-completion provider by {provider}"); } - completer.SetModelName(model); - return completer; - } - public static ISpeechToText GetSpeechToText( - IServiceProvider services, - string provider, - string model - ) - { - var completions = services.GetServices(); - var completer = completions.FirstOrDefault(x => x.Provider == provider); - if (completer == null) - { - var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve speech2text provider by {provider}"); - } completer.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs new file mode 100644 index 00000000..a278d7bf --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs @@ -0,0 +1,36 @@ +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Knowledges.Helpers; + +public static class KnowledgeSettingHelper +{ + public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) + { + var settings = services.GetRequiredService(); + var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding; + if (found == null) + { + found = settings.Default.TextEmbedding; + } + + var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); + var dimension = found.Dimension; + + if (found.Dimension <= 0) + { + dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model); + } + + embedding.SetModelName(found.Model); + embedding.SetDimension(dimension); + return embedding; + } + + private static int GetLlmTextEmbeddingDimension(IServiceProvider services, string provider, string model) + { + var settings = services.GetRequiredService(); + var found = settings.GetSetting(provider, model); + return found?.Dimension ?? 0; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs similarity index 78% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs index 88c77641..68b58539 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs @@ -1,17 +1,18 @@ +using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Helpers; -public class TextChopperService : ITextChopper +public static class TextChopper { - public List Chop(string content, ChunkOption option) + public static List Chop(string content, ChunkOption option) { content = Regex.Replace(content, @"\.{2,}", " "); content = Regex.Replace(content, @"_{2,}", " "); return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option); } - private List ChopByWord(string content, ChunkOption option) + private static List ChopByWord(string content, ChunkOption option) { var chunks = new List(); @@ -34,7 +35,7 @@ public class TextChopperService : ITextChopper return chunks; } - private List ChopByChar(string content, ChunkOption option) + private static List ChopByChar(string content, ChunkOption option) { var chunks = new List(); var currentPos = 0; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs similarity index 65% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 9b185867..d8b82fb5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,11 +1,15 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + var lines = TextChopper.Chop(knowledge.Content, new ChunkOption { Size = 1024, Conjunction = 32, @@ -25,6 +29,25 @@ public partial class KnowledgeService } } + public async Task CreateVectorCollection(string collectionName, int dimension) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.CreateCollection(collectionName, dimension); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs similarity index 51% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs index e78990a7..404b4e2c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs @@ -1,7 +1,26 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { + public async Task DeleteVectorCollection(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollection(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task DeleteVectorCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs index ee36780e..69093659 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs index 0fa1f5cd..b9652f61 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs similarity index 64% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index eec49891..2c0853d0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,38 +1,41 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Graph; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.VectorStorage; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly ITextChopper _textChopper; private readonly ILogger _logger; public KnowledgeService( IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper, ILogger logger) { _services = services; _settings = settings; - _textChopper = textChopper; _logger = logger; } private IVectorDb GetVectorDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); return db; } private IGraphDb GetGraphDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.GraphDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.GraphDb.Provider); return db; } private ITextEmbedding GetTextEmbedding(string collection) { - return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection); + return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs index 72549e51..0fcaa5a1 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Core.Routing.Planning; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid index 0332af76..4c9f51c3 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid @@ -1,8 +1,10 @@ -You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name. -the summarized question/answer should: +You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname, data type and comments, generate the related knowledge for DBA and BA. When users ask the question, they don't know the table name. + +The summarized question/answer should: 1. help user to identify the location of tables to find further information 2. identify the table structure and data relationship based on the task description -3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer +3. summarize all the tables to table relationship information based on the FOREIGN KEY, and include both tables in the answer + Go through all the columns and generate multiple question & answer pairs. The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n. diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index cf6985e9..afb8dd01 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -23,10 +23,13 @@ public class VerboseLogHook : IContentGeneratingHook { if (!_convSettings.ShowVerboseLog) return; - var dialog = conversations.Last(); - var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; - _logger.LogInformation(log); - + var dialog = conversations.LastOrDefault(); + if (dialog != null) + { + var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; + _logger.LogInformation(log); + } + await Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 24002225..9094107d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -19,6 +19,7 @@ + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 9b964dff..565677f7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -294,9 +294,7 @@ public class ConversationController : ControllerBase } [HttpPost("/conversation/{agentId}/{conversationId}/sse")] - public async Task SendMessageSse([FromRoute] string agentId, - [FromRoute] string conversationId, - [FromBody] NewMessageModel input) + public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) { var conv = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, input.Text) @@ -391,7 +389,7 @@ public class ConversationController : ControllerBase } [HttpPost("/agent/{agentId}/conversation/{conversationId}/upload")] - public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) + public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] InputMessageFiles input) { var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 48fdafe1..bc02e846 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -20,8 +21,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/{agentId}")] - public async Task InstructCompletion([FromRoute] string agentId, - [FromBody] InstructMessageModel input) + public async Task InstructCompletion([FromRoute] string agentId, [FromBody] InstructMessageModel input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -79,7 +79,7 @@ public class InstructModeController : ControllerBase #region Read image [HttpPost("/instruct/multi-modal")] - public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + public async Task MultiModalCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -87,21 +87,50 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); - return message.Content; + var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); + return content; } catch (Exception ex) { - var error = $"Error in analyzing files. {ex.Message}"; + var error = $"Error in reading images. {ex.Message}"; _logger.LogError(error); return error; } } + + [HttpPost("/instruct/multi-modal/upload")] + public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new MultiModalViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadImages(provider, model, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in reading image upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] IncomingMessageModel input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -127,7 +156,7 @@ public class InstructModeController : ControllerBase #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] IncomingMessageModel input) + public async Task ImageVariation([FromBody] ImageVariationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -135,16 +164,16 @@ public class InstructModeController : ControllerBase try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; } catch (Exception ex) @@ -156,8 +185,43 @@ public class InstructModeController : ControllerBase } } + [HttpPost("/instruct/image-variation/upload")] + public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageVariation(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] IncomingMessageModel input) + public async Task ImageEdit([FromBody] ImageEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -166,12 +230,11 @@ public class InstructModeController : ControllerBase try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -185,8 +248,44 @@ public class InstructModeController : ControllerBase } } + [HttpPost("/instruct/image-edit/upload")] + public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), stream, file.FileName); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -195,11 +294,11 @@ public class InstructModeController : ControllerBase try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var image = input.File; var mask = input.Mask; if (image == null || mask == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; @@ -214,11 +313,52 @@ public class InstructModeController : ControllerBase return imageViewModel; } } + + [HttpPost("/instruct/image-mask-edit/upload")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + using var imageStream = new MemoryStream(); + image.CopyTo(imageStream); + imageStream.Position = 0; + + using var maskStream = new MemoryStream(); + mask.CopyTo(maskStream); + maskStream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageStream.Close(); + maskStream.Close(); + + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } #endregion #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] IncomingMessageModel input) + public async Task PdfCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -239,5 +379,109 @@ public class InstructModeController : ControllerBase return viewModel; } } + + [HttpPost("/instruct/pdf-completion/upload")] + public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new PdfCompletionViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in pdf completion upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + #endregion + + #region Audio + [HttpPost("/instruct/speech-to-text")] + public async Task SpeechToText([FromBody] SpeechToTextRequest input) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + var audio = input.File; + if (audio == null) + { + return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; + } + var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech to text. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/speech-to-text/upload")] + public async Task SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] string? text = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text); + viewModel.Content = content; + stream.Close(); + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech-to-text upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/text-to-speech")] + public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1"); + var binaryData = await completion.GenerateAudioFromTextAsync(input.Text); + var stream = binaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index deb21378..827c2d7c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -18,12 +18,25 @@ public class KnowledgeBaseController : ControllerBase _services = services; } + #region Vector [HttpGet("knowledge/vector/collections")] public async Task> GetVectorCollections() { return await _knowledgeService.GetVectorCollections(); } + [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")] + public async Task CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension) + { + return await _knowledgeService.CreateVectorCollection(collection, dimension); + } + + [HttpDelete("knowledge/vector/{collection}/delete-collection")] + public async Task GetVectorCollections([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollection(collection); + } + [HttpPost("/knowledge/vector/{collection}/search")] public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { @@ -91,7 +104,7 @@ public class KnowledgeBaseController : ControllerBase public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); var filePath = Path.GetTempFileName(); using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) @@ -109,7 +122,10 @@ public class KnowledgeBaseController : ControllerBase System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } + #endregion + + #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) { @@ -124,7 +140,10 @@ public class KnowledgeBaseController : ControllerBase Result = result.Result }; } + #endregion + + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) { @@ -148,4 +167,5 @@ public class KnowledgeBaseController : ControllerBase GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null }; } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 5571d9fa..02282b85 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -25,6 +25,6 @@ public class LlmProviderController : ControllerBase public IEnumerable GetLlmProviderModels([FromRoute] string provider) { var list = _llmProvider.GetProviderModels(provider); - return list.Where(x => !x.ImageGeneration); + return list.Where(x => x.Type == LlmModelType.Chat); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a7d4c449..16399334 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -135,9 +135,14 @@ public class UserController : ControllerBase #region Avatar [HttpPost("/user/avatar")] - public bool UploadUserAvatar([FromBody] BotSharpFile file) + public bool UploadUserAvatar([FromBody] UserAvatarModel input) { var fileStorage = _services.GetRequiredService(); + var file = new InputFileModel + { + FileName = input.FileName, + FileData = input.FileData, + }; return fileStorage.SaveUserAvatar(file); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs new file mode 100644 index 00000000..fdf088bb --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class InputMessageFiles +{ + public List States { get; set; } = new(); + public List Files { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 0050de01..ea8dc076 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -2,18 +2,11 @@ using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class ImageGenerationViewModel +public class ImageGenerationViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - [JsonPropertyName("images")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable Images { get; set; } = new List(); - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } public class ImageViewModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs new file mode 100644 index 00000000..39a6439d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -0,0 +1,75 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructBaseRequest +{ + [JsonPropertyName("provider")] + public virtual string? Provider { get; set; } = null; + + [JsonPropertyName("model")] + public virtual string? Model { get; set; } = null; + + [JsonPropertyName("model_id")] + public virtual string? ModelId { get; set; } = null; + + [JsonPropertyName("states")] + public List States { get; set; } = new(); +} + +public class MultiModalRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("files")] + public List Files { get; set; } = new(); +} + +public class ImageGenerationRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; +} + +public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + +public class ImageEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + +public class ImageMaskEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } + + [JsonPropertyName("mask")] + public InstructFileModel Mask { get; set; } +} + +public class SpeechToTextRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string? Text { get; set; } + + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + +public class TextToSpeechRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs new file mode 100644 index 00000000..0b20fca3 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructBaseViewModel +{ + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs index 3b265e04..b736330f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.OpenAPI.ViewModels.Instructs; public class InstructMessageModel : IncomingMessageModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs new file mode 100644 index 00000000..a7b15262 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class MultiModalViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs index 13ed3eb9..7ac594e6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs @@ -1,13 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class PdfCompletionViewModel +public class PdfCompletionViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs new file mode 100644 index 00000000..d681986d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class SpeechToTextViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs new file mode 100644 index 00000000..0eab88e5 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Users; + +public class UserAvatarModel +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public string FileData { get; set; } = string.Empty; +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 304aa97b..2c289907 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,31 +1,24 @@ using BotSharp.Plugin.AudioHandler.Settings; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Abstraction.Settings; -namespace BotSharp.Plugin.AudioHandler +namespace BotSharp.Plugin.AudioHandler; + +public class AudioHandlerPlugin : IBotSharpPlugin { - public class AudioHandlerPlugin : IBotSharpPlugin + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) { - public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; - public string Name => "Audio Handler"; - public string Description => "Process audio input and transform it into text output."; - public void RegisterDI(IServiceCollection services, IConfiguration config) + services.AddScoped(provider => { - //var settings = new AudioHandlerSettings(); - //config.Bind("AudioHandler", settings); - //services.AddSingleton(x => settings); + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); - services.AddScoped(provider => - { - var settingService = provider.GetRequiredService(); - return settingService.Bind("AudioHandler"); - }); - - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - } + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index 7218d40c..4ae8c7d2 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -22,5 +22,19 @@ + + + + + + + + + PreserveNewest + + + PreserveNewest + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs deleted file mode 100644 index 45f4c8a4..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Provider; -using BotSharp.Core.Infrastructures; - -namespace BotSharp.Plugin.AudioHandler.Controllers -{ -#if DEBUG - [AllowAnonymous] -#endif - [ApiController] - public class AudioController : ControllerBase - { - private readonly ISpeechToText _nativeWhisperProvider; - private readonly IServiceProvider _services; - - public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service) - { - _nativeWhisperProvider = nativeWhisperProvider; - _services = service; - } - - [HttpGet("audio/transcript")] - public async Task GetTextFromAudioController(string audioInputString, string modelType = "") - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - await _nativeWhisperProvider.SetModelName(modelType); - - var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - - [HttpPost("openai/audio/transcript")] - public async Task GetTextFromAudioOpenAiController(string filePath) - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1"); - var result = await client.GenerateTextFromAudioAsync(filePath); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs index 436b2922..7b299a76 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -1,32 +1,23 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Whisper.net.Wave; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public enum AudioType { - public enum AudioType - { - wav, - mp3, - } + wav, + mp3, +} - public static class AudioTypeExtensions +public static class AudioTypeExtensions +{ + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + public static string ToFileType(this AudioType audioType) { - public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; - public static string ToFileType(this AudioType audioType) + string type = audioType switch { - string type = audioType switch - { - AudioType.mp3 => "audio/mpeg", - AudioType.wav => "audio/wav", - _ => throw new NotImplementedException($"No support found for {audioType}") - }; - return type; - } + AudioType.mp3 => "audio/mpeg", + AudioType.wav => "audio/wav", + _ => throw new NotImplementedException($"No support found for {audioType}") + }; + return type; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs index 0deab6e9..d11bd65a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public class UtilityName { - public class UtilityName - { - public const string AudioHandler = "audio-handler"; - } + public const string AudioHandler = "audio-handler"; } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs deleted file mode 100644 index a6544359..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs +++ /dev/null @@ -1,68 +0,0 @@ -using BotSharp.Plugin.AudioHandler.Enums; -using NAudio; -using NAudio.Wave; -using NAudio.Wave.SampleProviders; - -namespace BotSharp.Plugin.AudioHandler.Functions; - -public class AudioProcessUtilities : IAudioProcessUtilities -{ - public AudioProcessUtilities() - { - } - - public Stream ConvertMp3ToStream(string mp3FileName) - { - var fileStream = File.OpenRead(mp3FileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; - - } - - public Stream ConvertWavToStream(string wavFileName) - { - var fileStream = File.OpenRead(wavFileName); - using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; - } - - public Stream ConvertToStream(string fileName) - { - if (string.IsNullOrEmpty(fileName)) - { - throw new ArgumentNullException("fileName is Null"); - } - string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); - if (!Enum.TryParse(fileExtension, out AudioType fileType)) - { - throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); - } - - var stream = fileType switch - { - AudioType.mp3 => ConvertMp3ToStream(fileName), - AudioType.wav => ConvertWavToStream(fileName), - _ => throw new NotSupportedException("File extension not supported"), - }; - - return stream; - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 0c2298c6..34ec530c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using Microsoft.AspNetCore.StaticFiles; @@ -10,9 +9,9 @@ public class HandleAudioRequestFn : IFunctionCallback public string Indication => "Handling audio request"; private readonly IServiceProvider _serviceProvider; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; - private Agent? _agent; private readonly IEnumerable _audioContentType = new List { @@ -20,13 +19,13 @@ public class HandleAudioRequestFn : IFunctionCallback AudioType.wav.ToFileType(), }; - public HandleAudioRequestFn( + IFileStorageService fileStorage, IServiceProvider serviceProvider, ILogger logger, - BotSharpOptions options - ) + BotSharpOptions options) { + _fileStorage = fileStorage; _serviceProvider = serviceProvider; _logger = logger; _options = options; @@ -36,32 +35,31 @@ public class HandleAudioRequestFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); var conv = _serviceProvider.GetRequiredService(); - var isNeedSummary = args?.IsNeedSummary ?? false; var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText; + var response = await GetResponeFromDialogs(dialogs); message.Content = response; return true; } - private async Task> AssembleFiles(string convId, List dialogs) + private List AssembleFiles(string convId, List dialogs) { if (dialogs.IsNullOrEmpty()) + { return new List(); + } - var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); foreach (var dialog in dialogs) { var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList(); - if (found.IsNullOrEmpty()) - continue; + if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { @@ -74,53 +72,48 @@ public class HandleAudioRequestFn : IFunctionCallback return dialogs; } - private bool ParseAudioFileType(string fileType) - { - fileType = fileType.ToLower(); - var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType); - return canParse; - } - private async Task GetResponeFromDialogs(List dialogs) { - var whisperService = await PrepareModel("native"); // openai, native + var audioCompletion = PrepareModel(); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); - int transcribedCount = 0; + var transcripts = new List(); + foreach (var file in dialog.Files) { - if (file == null) - continue; + if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; - string extension = Path.GetExtension(file?.FileStorageUrl); - if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) - { - file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl); - transcribedCount++; - } + var extension = Path.GetExtension(file.FileStorageUrl); + + var fileName = Path.GetFileName(file.FileStorageUrl); + if (!ParseAudioFileType(fileName)) continue; + + var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = new MemoryStream(bytes); + stream.Position = 0; + + var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName); + transcripts.Add(result); + stream.Close(); } - if (transcribedCount == 0) + if (transcripts.IsNullOrEmpty()) { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } - var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); - return string.Join("\n\r", resList); + + return string.Join("\r\n\r\n", transcripts); } - private async Task PrepareModel(string modelName = "native") + private IAudioCompletion PrepareModel() { - var whisperService = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == modelName.ToLower()); - if (whisperService == null) - { - throw new Exception($"Can't resolve speech2text provider by {modelName}"); - } + return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1"); + } - if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase)) - { - return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); - } - await whisperService.SetModelName("Tiny"); - return whisperService; + private bool ParseAudioFileType(string fileName) + { + var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); + var provider = new FileExtensionContentTypeProvider(); + bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _); + return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs deleted file mode 100644 index a3c8243b..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace BotSharp.Plugin.AudioHandler.Functions -{ - public interface IAudioProcessUtilities - { - Stream ConvertMp3ToStream(string mp3FileName); - Stream ConvertWavToStream(string wavFileName); - Stream ConvertToStream(string fileName); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs new file mode 100644 index 00000000..4737a5e9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -0,0 +1,85 @@ +using NAudio.Wave; +using NAudio.Wave.SampleProviders; + +namespace BotSharp.Plugin.AudioHandler.Helpers; + +public static class AudioHelper +{ + private const int DEFAULT_SAMPLE_RATE = 16000; + + public static Stream ConvertToStream(string fileName) + { + if (string.IsNullOrEmpty(fileName)) + { + throw new ArgumentNullException("fileName is Null when converting to stream in audio processor"); + } + + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); + } + + var stream = fileType switch + { + AudioType.mp3 => ConvertMp3ToStream(fileName), + _ => ConvertWavToStream(fileName) + }; + + return stream; + } + + public static Stream Transform(Stream stream, string fileName) + { + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); + } + + Stream resultStream = new MemoryStream(); + stream.CopyTo(resultStream); + resultStream.Seek(0, SeekOrigin.Begin); + + WaveStream reader = fileType switch + { + AudioType.mp3 => new Mp3FileReader(resultStream), + _ => new WaveFileReader(resultStream) + }; + + resultStream = ChangeSampleRate(reader); + reader.Close(); + return resultStream; + } + + private static Stream ConvertMp3ToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + return ChangeSampleRate(reader); + } + + private static Stream ConvertWavToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); + using var reader = new WaveFileReader(fileStream); + return ChangeSampleRate(reader); + } + + private static Stream ChangeSampleRate(WaveStream ws) + { + var ms = new MemoryStream(); + if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE) + { + var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE); + WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16()); + } + else + { + ws.CopyTo(ms); + } + + ms.Seek(0, SeekOrigin.Begin); + return ms; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index c3ff10d0..80acb149 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,73 +1,60 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -namespace BotSharp.Plugin.AudioHandler.Hooks +namespace BotSharp.Plugin.AudioHandler.Hooks; + +public class AudioHandlerHook : AgentHookBase, IAgentHook { - public class AudioHandlerHook : AgentHookBase, IAgentHook + private const string HANDLER_AUDIO = "handle_audio_request"; + + public override string SelfId => string.Empty; + + public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - private const string HANDLER_AUDIO = "handle_audio_request"; + } - public override string SelfId => string.Empty; + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + if (isEnabled && isConvMode) { + AddUtility(agent, HANDLER_AUDIO); } - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); + base.OnAgentLoaded(agent); + } - if (isEnabled && isConvMode) + private void AddUtility(Agent agent, string functionName) + { + var (prompt, fn) = GetPromptAndFunction(functionName); + + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; } - base.OnAgentLoaded(agent); - } - - private void AddUtility(Agent agent, string utility, string functionName) - { - if (!IsEnableUtility(agent, utility)) return; - - var (prompt, fn) = GetPromptAndFunction(functionName); - if (fn != null) + if (agent.Functions == null) { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } - - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); } - } - - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - - private (string, FunctionDef?) GetPromptAndFunction(string functionName) - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); - return (prompt, loadAttachmentFn); } } + + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs index f220acdf..ac3f0ed7 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs @@ -1,10 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Abstraction.Agents; - namespace BotSharp.Plugin.AudioHandler.Hooks; public class AudioHandlerUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs index 5132ef3a..281305e8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs index 3db02f71..ba75464d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs index 1b58f455..f8f0bf51 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -1,19 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Whisper.net; -namespace BotSharp.Plugin.AudioHandler.Models -{ - public class AudioOutput - { - public List Segments { get; set; } +namespace BotSharp.Plugin.AudioHandler.Models; - public override string ToString() - { - return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; - } +public class AudioOutput +{ + public List Segments { get; set; } = new(); + + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 7afae1fc..e6ac230e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Core.Agents.Services; using Whisper.net; using Whisper.net.Ggml; @@ -7,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// /// Native Whisper provider for speech to text conversion /// -public class NativeWhisperProvider : ISpeechToText +public class NativeWhisperProvider : IAudioCompletion { - public string Provider => "native"; - private readonly IAudioProcessUtilities _audioProcessUtilities; - private static WhisperProcessor _processor; - private readonly ILogger _logger; + private static WhisperProcessor _whisperProcessor; - private string MODEL_DIR = "model"; - private string? _currentModelPath; - private Dictionary _modelPathDict = new Dictionary(); - private GgmlType? _modelType; + private readonly IServiceProvider _services; + private readonly IFileStorageService _fileStorage; + private readonly ILogger _logger; + + public string Provider => "native"; public NativeWhisperProvider( - IAudioProcessUtilities audioProcessUtilities, + BotSharpDatabaseSettings dbSettings, + IFileStorageService fileStorage, + IServiceProvider services, ILogger logger) { - _audioProcessUtilities = audioProcessUtilities; + _fileStorage = fileStorage; + _services = services; _logger = logger; } - public async Task GenerateTextFromAudioAsync(string filePath) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) { - string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - - using var stream = _audioProcessUtilities.ConvertToStream(filePath); - - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - var textResult = new List(); - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + using var stream = AudioHelper.Transform(audio, audioFileName); + await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } - _processor.Dispose(); + _whisperProcessor.Dispose(); + stream.Close(); var audioOutput = new AudioOutput { @@ -57,71 +46,69 @@ public class NativeWhisperProvider : ISpeechToText }; return audioOutput.ToString(); } - private async Task LoadWhisperModel(GgmlType modelType) + + public async Task GenerateAudioFromTextAsync(string text) + { + throw new NotImplementedException(); + } + + public void SetModelName(string model) + { + if (Enum.TryParse(model, true, out GgmlType ggmlType)) + { + LoadWhisperModel(ggmlType); + } + else + { + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + LoadWhisperModel(GgmlType.Tiny); + } + } + + private void LoadWhisperModel(GgmlType modelType) { try { - if (!Directory.Exists(MODEL_DIR)) - Directory.CreateDirectory(MODEL_DIR); - - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin") - .ToArray(); - - if (!availableModelPaths.Any()) + var modelDir = _fileStorage.BuildDirectory("models", "whisper"); + var exist = _fileStorage.ExistDirectory(modelDir); + if (!exist) { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin"); - if (modelFilePath == null) - { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - _currentModelPath = modelFilePath; - } + _fileStorage.CreateDirectory(modelDir); } - _processor = WhisperFactory - .FromPath(path: _currentModelPath) - .CreateBuilder() - .WithLanguage("auto") - .Build(); + var files = _fileStorage.GetFiles("models/whisper", "*.bin"); + var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType)); + if (string.IsNullOrEmpty(modelLoc)) + { + modelLoc = BuildModelPath(modelType); + DownloadModel(modelType, modelLoc); + } - _modelType = modelType; + var bytes = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { - throw new Exception($"Failed to load whisper model: {ex.Message}"); + var error = "Failed to load whisper model"; + _logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}"); + throw new Exception($"{error}: {ex.Message}"); } } - private async Task DownloadModel(GgmlType modelType, string modelDir) + private void DownloadModel(GgmlType modelType, string modelDir) { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType); - using var fileWriter = File.OpenWrite(modelDir); - await modelStream.CopyToAsync(fileWriter); + using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); + modelStream.Close(); } - private string SetModelPath(string rootPath, GgmlType modelType) + private string BuildModelPath(GgmlType modelType) { - string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); - return currentModelPath; + return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType)); } - public async Task SetModelName(string modelType) + private string BuildModelFile(GgmlType modelType) { - if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) - { - await LoadWhisperModel(ggmlType); - return; - } - - _logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); + return $"ggml-{modelType}.bin"; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index 4ace63db..fedf123c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -1,6 +1,5 @@ -namespace BotSharp.Plugin.AudioHandler.Settings +namespace BotSharp.Plugin.AudioHandler.Settings; + +public class AudioHandlerSettings { - public class AudioHandlerSettings - { - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index c15e6cc4..1a1188fd 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -5,6 +5,7 @@ global using System.Linq; global using System.Text.Json; global using System.Threading.Tasks; +global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; @@ -20,7 +21,7 @@ global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.AudioHandler.Enums; -global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Helpers; global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; global using BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json index b223c7ae..c0603b1a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json @@ -1,18 +1,18 @@ { - "name": "handle_audio_request", - "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", - "parameters": { - "type": "object", - "properties": { - "user_request": { - "type": "string", - "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" - }, - "is_need_summary": { - "type": "boolean", - "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" - } - }, - "required": [ "user_request" ] - } + "name": "handle_audio_request", + "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" + }, + "is_need_summary": { + "type": "boolean", + "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" + } + }, + "required": [ "user_request" ] + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index dba74d27..eba22bfa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.AzureOpenAI.Providers.Audio; using BotSharp.Plugin.AzureOpenAI.Providers.Chat; using BotSharp.Plugin.AzureOpenAI.Providers.Embedding; using BotSharp.Plugin.AzureOpenAI.Providers.Image; @@ -30,5 +31,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 00000000..5a436fef --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,90 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + + var options = new AudioTranscriptionOptions + { + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + + AudioTranscriptionFormat format; + switch (value) + { + case "json": + format = AudioTranscriptionFormat.Simple; + break; + case "srt": + format = AudioTranscriptionFormat.Srt; + break; + case "vtt": + format = AudioTranscriptionFormat.Vtt; + break; + default: + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float? GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return null; + } + + return temperature; + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 00000000..4e5fc3fa --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,102 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateAudioFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + return result.Value; + } + + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed + }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 00000000..8cc75ad8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index af82b037..678ee82b 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = value.FunctionCall.FunctionName, FunctionArgs = value.FunctionCall.FunctionArguments }; @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments }; @@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 39119050..762e870f 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly IServiceProvider _services; protected readonly ILogger _logger; - private const int DEFAULT_DIMENSION = 3072; + private const int DEFAULT_DIMENSION = 1536; protected string _model; protected int _dimension; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 43272f8e..52e1a845 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -53,7 +53,7 @@ public partial class ImageCompletionProvider var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7d608fd1..f5d8c1f9 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public partial class ImageCompletionProvider var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 88f393e0..b2e4c5fe 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public partial class ImageCompletionProvider { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 7d064154..967f3e08 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ public partial class ImageCompletionProvider : IImageCompletion case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b4f9a6e4..fa038811 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -84,11 +84,12 @@ public class HandleEmailSenderFn : IFunctionCallback { if (files.IsNullOrEmpty()) return; + var fileStorage = _services.GetRequiredService(); + foreach (var file in files) { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs index 222c27b9..beece9e2 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs index d4ae8c16..27491015 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 759f94b6..dc4ddef0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -43,7 +43,7 @@ public class EditImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } @@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback { if (image == null) return; - var files = new List() + var files = new List() { - new BotSharpFile + new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4102ff7b..9592ef80 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -50,7 +50,7 @@ public class GenerateImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } @@ -77,7 +77,7 @@ public class GenerateImageFn : IFunctionCallback { if (images.IsNullOrEmpty()) return; - var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new BotSharpFile + var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index e2b465c8..814f943e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -52,7 +52,7 @@ public class ReadPdfFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); if (screenshots.IsNullOrEmpty()) return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9e0ceaee..0465dc4e 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public class GraphDb : IGraphDb _settings = settings; } - public string Name => "Default"; + public string Provider => "Remote"; public async Task Search(string query, GraphSearchOptions options) { @@ -44,7 +44,7 @@ public class GraphDb : IGraphDb return new GraphSearchData(); } - var url = $"{_settings.BaseUrl}/query"; + var url = $"{_settings.BaseUrl}{_settings.SearchPath}"; var request = new GraphQueryRequest { Query = query, diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs index 89351398..34e0b5fd 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs @@ -3,4 +3,5 @@ namespace BotSharp.Plugin.Graph; public class GraphDbSettings { public string BaseUrl { get; set; } + public string SearchPath { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 42663445..29c81ec7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs index 09401f20..0e39bef0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs @@ -1,11 +1,11 @@ using UglyToad.PdfPig; using UglyToad.PdfPig.Content; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Plugin.KnowledgeBase.Converters; public class PigPdf2TextConverter : IPdf2TextConverter { - public string Name => "Pig"; + public string Provider => "Pig"; public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 060e2150..ffbef770 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,9 +1,13 @@ +using BotSharp.Abstraction.VectorStorage.Extensions; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { public string Name => "knowledge_retrieval"; + public string Indication => "searching my brain"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,15 +22,16 @@ public class KnowledgeRetrievalFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); - var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }); + var knowledgeService = _services.GetRequiredService(); + var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions + { + Fields = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }, + Confidence = 0.2f + }); if (!knowledges.IsNullOrEmpty()) { - var answers = knowledges.Select(x => $"Question: {x.Data[KnowledgePayloadName.Text]}\r\nAnswer: {x.Data[KnowledgePayloadName.Answer]}").ToList(); + var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList(); message.Content = string.Join("\r\n\r\n=====\r\n", answers); } else diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f709e3b8..62388f19 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -4,6 +4,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { public string Name => "memorize_knowledge"; + public string Indication => "remembering knowledge"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,23 +20,16 @@ public class MemorizeKnowledgeFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorsAsync(new List + var knowledgeService = _services.GetRequiredService(); + var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { - args.Question + Text = args.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, args.Answer } + } }); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); - await vectorDb.CreateCollection(collectionName, vector[0].Length); - - var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], - args.Question, - new Dictionary - { - { KnowledgePayloadName.Answer, args.Answer } - }); - message.Content = result ? "Saved to my brain" : "I forgot it"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs index f0538af0..88a6ebb5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs @@ -2,9 +2,9 @@ using BotSharp.Plugin.KnowledgeBase.MemVecDb; using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class VectorUtility +public static class VectorHelper { public static float[] CalEuclideanDistance(float[] vec, List records) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 213825b5..ffe4d8f1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.KnowledgeBase.Converters; using BotSharp.Plugin.KnowledgeBase.Hooks; using Microsoft.Extensions.Configuration; @@ -20,8 +21,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin return settingService.Bind("KnowledgeBase"); }); - services.AddScoped(); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); @@ -35,7 +34,7 @@ public class KnowledgeBasePlugin : IBotSharpPlugin SubMenu = new List { new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"), - new PluginMenuDef("Relations", link: "page/knowledge-base/relations") + new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships") } }); return true; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 0d4be819..17a2c757 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -8,12 +8,18 @@ public class MemoryVectorDb : IVectorDb private readonly Dictionary> _vectors = new Dictionary>(); - public string Name => "MemoryVector"; + public string Provider => "MemoryVector"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { - _collections[collectionName] = dim; + _collections[collectionName] = dimension; _vectors[collectionName] = new List(); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + return false; } public async Task> GetCollections() @@ -40,8 +46,7 @@ public class MemoryVectorDb : IVectorDb return new List(); } - var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() .Reverse() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 7eeea4f0..1437f41e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -31,6 +31,5 @@ global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; -global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; -global using BotSharp.Plugin.KnowledgeBase.Utilities; \ No newline at end of file +global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs deleted file mode 100644 index 770e1efa..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Utilities; - -public static class KnowledgeSettingUtility -{ - public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) - { - var settings = services.GetRequiredService(); - var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding; - if (found == null) - { - found = settings.Default.TextEmbedding; - } - - var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); - embedding.SetModelName(found.Model); - embedding.SetDimension(found.Dimension); - return embedding; - } -} diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 902087d7..58fb1851 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,9 +9,14 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { - public string Name => "Faiss"; + public string Provider => "Faiss"; - public Task CreateCollection(string collectionName, int dim) + public Task CreateCollection(string collectionName, int dimension) + { + throw new NotImplementedException(); + } + + public Task DeleteCollection(string collectionName) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 4b41ec46..e6d0637c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -4,8 +4,8 @@ using BotSharp.Plugin.OpenAI.Providers.Embedding; using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; -using Microsoft.Extensions.Configuration; using BotSharp.Plugin.OpenAI.Providers.Audio; +using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.OpenAI; @@ -31,7 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 00000000..9f248086 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,89 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + + var options = new AudioTranscriptionOptions + { + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + + AudioTranscriptionFormat format; + switch (value) + { + case "json": + format = AudioTranscriptionFormat.Simple; + break; + case "srt": + format = AudioTranscriptionFormat.Srt; + break; + case "vtt": + format = AudioTranscriptionFormat.Vtt; + break; + default: + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float? GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return null; + } + + return temperature; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 00000000..09acf389 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,102 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateAudioFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + return result.Value; + } + + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed + }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 00000000..3311fd43 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,21 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs deleted file mode 100644 index 2314b431..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Text; -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio; - -public class SpeechToTextProvider : ISpeechToText -{ - public string Provider => "openai"; - private readonly IServiceProvider _services; - private string? _modelName; - private AudioTranscriptionOptions? _options; - - public SpeechToTextProvider(IServiceProvider service) - { - _services = service; - } - - public async Task GenerateTextFromAudioAsync(string filePath) - { - var client = ProviderHelper - .GetClient(Provider, _modelName, _services) - .GetAudioClient(_modelName); - SetOptions(); - - var transcription = await client.TranscribeAudioAsync(filePath); - - return transcription.Value.Text; - } - - public async Task SetModelName(string modelName) - { - _modelName = modelName; - } - - public void SetOptions(AudioTranscriptionOptions? options = null) - { - if (_options == null) - { - _options = options ?? new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs deleted file mode 100644 index e559e109..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio -{ - public partial class TextToSpeechProvider : ITextToSpeech - { - public string Provider => "openai"; - private readonly IServiceProvider _services; - private string? _model; - - public TextToSpeechProvider( - IServiceProvider services) - { - _services = services; - } - - public void SetModelName(string model) - { - _model = model; - } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper - .GetClient(Provider, _model, _services) - .GetAudioClient(_model); - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 00f47149..57f7ac13 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = value.FunctionCall.FunctionName, FunctionArgs = value.FunctionCall.FunctionArguments }; @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments }; @@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index bebf6853..cb9e7d15 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -8,8 +8,8 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly IServiceProvider _services; protected readonly ILogger _logger; - private const int DEFAULT_DIMENSION = 3072; - protected string _model = "text-embedding-3-large"; + private const int DEFAULT_DIMENSION = 1536; + protected string _model = "text-embedding-3-small"; protected int _dimension = DEFAULT_DIMENSION; public virtual string Provider => "openai"; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 717002fb..47df077e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -54,7 +54,7 @@ public partial class ImageCompletionProvider var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 85c15686..2f55d917 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public partial class ImageCompletionProvider var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 8a93df83..30c8d68f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public partial class ImageCompletionProvider { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 185cf17c..0e45e789 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ public partial class ImageCompletionProvider : IImageCompletion case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 81c53f83..37ea35c4 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -29,7 +29,7 @@ public class Pdf2TextConverter : IPdf2TextConverter _paddleSharpSettings = paddleSharpSettings; } - public string Name => "Paddle"; + public string Provider => "Paddle"; public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 585dc174..c3e0a737 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -1,21 +1,13 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; -using BotSharp.Abstraction.Knowledges.Models; namespace BotSharp.Plugin.Planner.Functions; public class PrimaryStagePlanFn : IFunctionCallback { public string Name => "plan_primary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; - private object aiAssistant; + private readonly ILogger _logger; public PrimaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -25,34 +17,26 @@ public class PrimaryStagePlanFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - //debug + // Debug + var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var fn = _services.GetRequiredService(); + state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - //get knowledge from vectordb - var fn = _services.GetRequiredService(); - - var msg = new ExtractedKnowledge + // Get knowledge from vectordb + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; ; + var knowledges = await knowledgeService.SearchVectorKnowledge(task.Requirements, collectionName, new VectorSearchOptions { - Question = task.Question, - }; - var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements) - { - FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = task.Requirements - }), - KnowledgeConfidence = 0.1f, - Content = "" - }; - await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); - message.Content = retrievalMessage.Content; + Confidence = 0.1f + }); + message.Content = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); - var agentService = _services.GetRequiredService(); + // Send knowledge to AI to refine and summarize the primary planning var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - - //send knowledge to AI to refine and summarize the primary planning var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message); var plannerAgent = new Agent { @@ -62,7 +46,7 @@ public class PrimaryStagePlanFn : IFunctionCallback TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; /*await fn.InvokeFunction("plan_secondary_stage", message); @@ -99,12 +83,14 @@ public class PrimaryStagePlanFn : IFunctionCallback message.StopCompletion = true;*/ return true; } + private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.1st.plan").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.1st.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -118,13 +104,15 @@ public class PrimaryStagePlanFn : IFunctionCallback { "response_format", responseFormat } }); } + private async Task GetPlanSummaryPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "planner_prompt.two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -139,10 +127,12 @@ public class PrimaryStagePlanFn : IFunctionCallback { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); + //add "test" to wholeDialogs' last element if(plannerAgent.Name == "planner_summary") { @@ -150,6 +140,7 @@ public class PrimaryStagePlanFn : IFunctionCallback wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { //add "test" to wholeDialogs' last element in a new paragraph diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index d6e02299..211eea53 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,23 +1,13 @@ -using Azure; -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using NetTopologySuite.Index.HPRtree; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; public class SecondaryStagePlanFn : IFunctionCallback { public string Name => "plan_secondary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; public SecondaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -28,45 +18,49 @@ public class SecondaryStagePlanFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); - var msg_secondary = RoleDialogModel.From(message); - var task_primary = JsonSerializer.Deserialize(message.FunctionArgs); - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask - { - TaskDescription = task_primary.Requirements - }); - var task_secondary = JsonSerializer.Deserialize(msg_secondary.FunctionArgs); - var items = msg_secondary.Content.JsonArrayContent(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var msgSecondary = RoleDialogModel.From(message); + var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); + + msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask + { + TaskDescription = taskPrimary.Requirements + }); + + var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); + var items = msgSecondary.Content.JsonArrayContent(); - msg_secondary.KnowledgeConfidence = 0.5f; foreach (var item in items) { - if (item.NeedAdditionalInformation) + if (!item.NeedAdditionalInformation) continue; + + var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions { - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = item.Task - }); - await fn.InvokeFunction("knowledge_retrieval", msg_secondary); - message.Content += msg_secondary.Content; - } + Confidence = 0.5f + }); + message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); } + // load agent var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var secondPlanningPrompt = await GetSecondStagePlanPrompt(task_secondary, message); + var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary, message); _logger.LogInformation(secondPlanningPrompt); var plannerAgent = new Agent { - Id = "", + Id = string.Empty, Name = "test", Instruction = secondPlanningPrompt, TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; _logger.LogInformation(response.Content); return true; @@ -74,9 +68,10 @@ public class SecondaryStagePlanFn : IFunctionCallback private async Task GetSecondStagePlanPrompt(SecondaryBreakdownTask task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var planner = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = planner.Templates.First(x => x.Name == "two_stage.2nd.plan").Content; + + var planner = await agentService.GetAgent(message.CurrentAgentId); + var template = planner.Templates.FirstOrDefault(x => x.Name == "two_stage.2nd.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new SecondStagePlan { Tool = "tool name if task solution provided", @@ -91,7 +86,7 @@ public class SecondaryStagePlanFn : IFunctionCallback { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 4ec2a45d..eda46f50 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,23 +1,17 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Agents.Services; namespace BotSharp.Plugin.Planner.Functions; public class SummaryPlanFn : IFunctionCallback { public string Name => "plan_summary"; - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private object aiAssistant; - public SummaryPlanFn(IServiceProvider services, ILogger logger) + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public SummaryPlanFn( + IServiceProvider services, + ILogger logger) { _services = services; _logger = logger; @@ -27,55 +21,56 @@ public class SummaryPlanFn : IFunctionCallback { var fn = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - //debug var state = _services.GetRequiredService(); + + var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); state.SetState("max_tokens", "4096"); var task = state.GetState("requirement_detail"); - //get DDL + // Get DDL var steps = message.Content.JsonArrayContent(); - //get all the related tables - List allTables = new List(); + // Get all the related tables + var allTables = new List(); foreach (var step in steps) { allTables.AddRange(step.Tables); } message.Data = allTables.Distinct().ToList(); - //get table DDL and stores in content - var msg2 = RoleDialogModel.From(message); - await fn.InvokeFunction("get_table_definition", msg2); - message.SecondaryContent = msg2.Content; + // Get table DDL and stores in content + var msgCopy = RoleDialogModel.From(message); + await fn.InvokeFunction("get_table_definition", msgCopy); + var ddlStatements = msgCopy.Content; + + // Summarize and generate query + var summaryPlanPrompt = await GetPlanSummaryPrompt(task, message.Content, ddlStatements); + _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}"); - // summarize and generate query - var summaryPlanningPrompt = await GetPlanSummaryPrompt(task, message); - _logger.LogInformation(summaryPlanningPrompt); var plannerAgent = new Agent { Id = BuiltInAgentId.Planner, - Name = "planner_summary", - Instruction = summaryPlanningPrompt, - TemplateDict = new Dictionary(), + Name = "Planner Summary", + Instruction = summaryPlanPrompt, LlmConfig = currentAgent.LlmConfig }; - var response_summary = await GetAIResponse(plannerAgent); - message.Content = response_summary.Content; + var summary = await GetAiResponse(plannerAgent); + message.Content = summary.Content; message.StopCompletion = true; return true; } - private async Task GetPlanSummaryPrompt(string task, RoleDialogModel message) + private async Task GetPlanSummaryPrompt(string task, string knowledge, string ddlStatement) { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -84,26 +79,28 @@ public class SummaryPlanFn : IFunctionCallback return render.Render(template, new Dictionary { - { "table_structure", message.SecondaryContent }, ////check - { "task_description", task}, - { "relevant_knowledges", message.Content }, + { "table_structure", ddlStatement }, + { "task_description", task }, + { "relevant_knowledges", knowledge }, { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - //add "test" to wholeDialogs' last element - if(plannerAgent.Name == "planner_summary") + + // Add "test" to wholeDialogs' last element + if (plannerAgent.Name == "planner_summary") { - //add "test" to wholeDialogs' last element in a new paragraph + // Add "test" to wholeDialogs' last element in a new paragraph wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { - //add "test" to wholeDialogs' last element in a new paragraph + // Add "test" to wholeDialogs' last element in a new paragraph wholeDialogs.Last().Content += "\n\nYou must analyze the table description to infer the table relations."; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index be627c75..c6f819e5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -1,13 +1,6 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Planning; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Core.Routing.Planning; -using Microsoft.Extensions.Logging; -using System.Threading.Tasks; namespace BotSharp.Plugin.Planner.TwoStaging; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Using.cs b/src/Plugins/BotSharp.Plugin.Planner/Using.cs index 20880e81..1e6737df 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Using.cs @@ -3,12 +3,13 @@ global using System.Text.Json; global using System.Text.Json.Serialization; global using System.Collections.Generic; global using System.Linq; +global using System.Threading.Tasks; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; global using BotSharp.Abstraction.Plugins; -global using BotSharp.Abstraction.Planning; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; @@ -18,5 +19,18 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Templating; + +global using BotSharp.Abstraction.Knowledges; +global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.VectorStorage.Models; +global using BotSharp.Abstraction.VectorStorage.Extensions; + global using BotSharp.Plugin.Planner.Hooks; -global using BotSharp.Plugin.Planner.Enums; \ No newline at end of file +global using BotSharp.Plugin.Planner.Enums; + +global using BotSharp.Core.Infrastructures; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 0a1f6292..fb13b0c1 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -3,7 +3,7 @@ You are a Task Planner. you will breakdown user business requirements into excut Thinking process: 1. Reference to "Task Knowledge" if there is relevant knowledge; 2. Breakdown task into subtasks. - - The subtask should contains all needed parameters for subsequent steps. + - The subtask should contain all needed parameters for subsequent steps. - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true. - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. @@ -22,9 +22,7 @@ Task Knowledge: {{ k }} {% endfor %} {%- endif %} - ===== Task description: -{{ task_description }} - +{{ task_description }} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index ccec8033..d5340e2e 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -1,5 +1,5 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown task into multiple steps. -* the step should contains all needed parameters. +* The step should contains all needed parameters. * The parameters can be extracted from the original task. * You need to list all the steps in detail. Finding relationships should also be a step. * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly. diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 99d917d4..be89fec0 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -19,7 +19,7 @@ public class QdrantDb : IVectorDb _services = services; } - public string Name => "Qdrant"; + public string Provider => "Qdrant"; private QdrantClient GetClient() { @@ -35,6 +35,34 @@ public class QdrantDb : IVectorDb return _client; } + public async Task CreateCollection(string collectionName, int dim) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (exist) return false; + + // Create a new collection + await client.CreateCollectionAsync(collectionName, new VectorParams() + { + Size = (ulong)dim, + Distance = Distance.Cosine + }); + + return true; + } + + public async Task DeleteCollection(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (!exist) return false; + + await client.DeleteCollectionAsync(collectionName); + return true; + } + public async Task> GetCollections() { // List all the collections @@ -51,10 +79,34 @@ public class QdrantDb : IVectorDb return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(collectionName); + // Build query filter + Filter? queryFilter = null; + if (!filter.SearchPairs.IsNullOrEmpty()) + { + var conditions = filter.SearchPairs.Select(x => new Condition + { + Field = new FieldCondition + { + Key = x.Key, + Match = new Match { Text = x.Value }, + } + }); + + queryFilter = new Filter + { + Should = + { + conditions + } + }; + } + + var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, - offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, + filter: queryFilter, vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, @@ -93,27 +145,7 @@ public class QdrantDb : IVectorDb }); } - public async Task CreateCollection(string collectionName, int dim) - { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); - if (!exist) - { - // Create a new collection - await client.CreateCollectionAsync(collectionName, new VectorParams() - { - Size = (ulong)dim, - Distance = Distance.Cosine - }); - } - - // Get collection info - var collectionInfo = await client.GetCollectionInfoAsync(collectionName); - if (collectionInfo == null) - { - throw new Exception($"Create {collectionName} failed."); - } - } + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { @@ -160,43 +192,26 @@ public class QdrantDb : IVectorDb return results; } - var points = await client.SearchAsync(collectionName, - vector, - limit: (ulong)limit, - scoreThreshold: confidence, - vectorsSelector: new WithVectorsSelector { Enable = withVector }); - - var pickFields = fields != null; - foreach (var point in points) + var payloadSelector = new WithPayloadSelector { Enable = true }; + if (fields != null) { - var data = new Dictionary(); - if (pickFields) - { - foreach (var field in fields) - { - if (point.Payload.ContainsKey(field)) - { - data[field] = point.Payload[field].StringValue; - } - else - { - data[field] = ""; - } - } - } - else - { - data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); - } - - results.Add(new VectorCollectionData - { - Id = point.Id.Uuid, - Data = data, - Score = point.Score, - Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null - }); + payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; } + + var points = await client.SearchAsync(collectionName, + vector, + limit: (ulong)limit, + scoreThreshold: confidence, + payloadSelector: payloadSelector, + vectorsSelector: withVector); + + results = points.Select(x => new VectorCollectionData + { + Id = x.Id.Uuid, + Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Score = x.Score, + Vector = x.Vectors?.Vector?.Data?.ToArray() + }).ToList(); return results; } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index a797b47f..4912e127 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -22,16 +22,23 @@ namespace BotSharp.Plugin.SemanticKernel } - public string Name => "SemanticKernel"; + public string Provider => "SemanticKernel"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { await _memoryStore.CreateCollectionAsync(collectionName); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + await _memoryStore.DeleteCollectionAsync(collectionName); + return false; } public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { - throw new System.NotImplementedException(); + throw new NotImplementedException(); } public Task> GetCollectionData(string collectionName, IEnumerable ids, diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 6d2b4b05..266abbec 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -53,6 +53,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs new file mode 100644 index 00000000..f863adb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs @@ -0,0 +1,24 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.SqlDriver.Controllers; + +[Authorize] +[ApiController] +public class SqlDriverController : ControllerBase +{ + private readonly IServiceProvider _services; + + public SqlDriverController(IServiceProvider services) + { + _services = services; + } + + [HttpPost("/knowledge/database/import")] + public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) + { + var dbKnowledge = _services.GetRequiredService(); + return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs deleted file mode 100644 index 5c2bfa45..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs +++ /dev/null @@ -1,109 +0,0 @@ -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using MySqlConnector; -using static Dapper.SqlMapper; -using BotSharp.Abstraction.Agents.Enums; - - -namespace BotSharp.Plugin.Planner.Functions; -public class AddDatabaseKnowledgeFn : IFunctionCallback -{ - public string Name => "add_database_knowledge"; - private readonly IServiceProvider _services; - private object aiAssistant; - - public AddDatabaseKnowledgeFn(IServiceProvider services) - { - _services = services; - } - public async Task Execute(RoleDialogModel message) - { - var agentService = _services.GetRequiredService(); - var sqlDriver = _services.GetRequiredService(); - var fn = _services.GetRequiredService(); - var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); - - List allTables = new List(); - var sql = $"select table_name from information_schema.tables;"; - var result = connection.Query(sql: sql,dictionary); - foreach (var item in result) - { - allTables.Add(item.TABLE_NAME); - } - message.Data = allTables.Distinct().ToList(); - - var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var note = ""; - foreach (var item in allTables) - { - message.Data = new List { item }; - - await fn.InvokeFunction("get_table_definition", message); - var PlanningPrompt = await GetPrompt(message); - var plannerAgent = new Agent - { - Id = "", - Name = "database_knowledge", - Instruction = PlanningPrompt, - TemplateDict = new Dictionary(), - LlmConfig = currentAgent.LlmConfig - }; - var response = await GetAIResponse(plannerAgent); - try - { - var knowledge = response.Content.JsonArrayContent(); - foreach (var k in knowledge) - { - try - { - message.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = k.Question, - Answer = k.Answer - }); - await fn.InvokeFunction("memorize_knowledge", message); - message.SecondaryContent += $"Table: {item}, Question:{k.Question}, {message.Content} \r\n"; - } - catch (Exception e) - { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; - } - } - } - catch (Exception e) - { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; - } - } - return true; - } - private async Task GetAIResponse(Agent plannerAgent) - { - var conv = _services.GetRequiredService(); - var wholeDialogs = conv.GetDialogHistory(); - var completion = CompletionProvider.GetChatCompletion(_services, - provider: plannerAgent.LlmConfig.Provider, - model: plannerAgent.LlmConfig.Model); - - return await completion.GetChatCompletions(plannerAgent, wholeDialogs); - } - private async Task GetPrompt(RoleDialogModel message) - { - var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); - var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "database_knowledge").Content; - var responseFormat = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = "question", - Answer = "answer" - }); - - return render.Render(template, new Dictionary - { - { "table_structure", message.Content } - }); - } -} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 74ae8f72..14314a8f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,5 +1,4 @@ -using BotSharp.Plugin.SqlDriver.Models; -using Microsoft.EntityFrameworkCore.Metadata.Internal; +using Microsoft.Extensions.Logging; using MySqlConnector; using static Dapper.SqlMapper; @@ -9,45 +8,64 @@ public class GetTableDefinitionFn : IFunctionCallback { public string Name => "get_table_definition"; private readonly IServiceProvider _services; + private readonly ILogger _logger; - public GetTableDefinitionFn(IServiceProvider services) + public GetTableDefinitionFn( + IServiceProvider services, + ILogger logger) { _services = services; + _logger = logger; } public async Task Execute(RoleDialogModel message) { - // get agent service var agentService = _services.GetRequiredService(); - - // var args = JsonSerializer.Deserialize(message.FunctionArgs); var sqlDriver = _services.GetRequiredService(); - - //get table DDL from database var settings = _services.GetRequiredService(); + + // Get table DDL from database + var tables = message.Data as List; + if (tables.IsNullOrEmpty()) return false; + + var tableDdls = new List(); using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); + connection.Open(); - var table_ddl = ""; - foreach (var p in (List)message.Data) + foreach (var table in tables) { - dictionary["@" + "table_name"] = p; - var escapedTableName = MySqlHelper.EscapeString(p); - dictionary["table_name"] = escapedTableName; - // can you replace this with a parameterized query? - var sql = $"select * from information_schema.tables where table_name ='{dictionary["table_name"]}'"; - - var result = connection.QueryFirstOrDefault(sql: sql, dictionary); - if (result != null) + try { - sql = $"SHOW CREATE TABLE `{dictionary["table_name"]}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - table_ddl += "\r\n" + result; - } - - } - message.Content = table_ddl; + var escapedTableName = MySqlHelper.EscapeString(table); + var sql = $"select * from information_schema.tables where table_name = @tableName"; + var result = connection.QueryFirstOrDefault(sql, new + { + tableName = escapedTableName + }); + + if (result == null) continue; + + sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + tableDdls.Add(result); + } + + reader.Close(); + command.Dispose(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting ddl statement of table {table}."); + } + } + + connection.Close(); + message.Content = string.Join("\r\n\r\n", tableDdls); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs new file mode 100644 index 00000000..48ba820c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs @@ -0,0 +1,19 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class RequestBase +{ + [JsonPropertyName("provider")] + public string? Provider { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } +} + + +public class ImportDbKnowledgeRequest : RequestBase +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs new file mode 100644 index 00000000..746798c9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -0,0 +1,140 @@ +using static Dapper.SqlMapper; +using Microsoft.Extensions.Logging; +using BotSharp.Core.Infrastructures; +using MySqlConnector; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.SqlDriver.Services; + +public class DbKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public DbKnowledgeService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Import(string provider, string model, string schema) + { + var sqlDriverSettings = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var tables = new HashSet(); + using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString); + + var sql = $"select table_name from information_schema.tables where table_schema = @tableSchema"; + var results = connection.Query(sql, new + { + tableSchema = schema + }); + + foreach (var item in results) + { + if (item == null) continue; + + tables.Add(item.TABLE_NAME); + } + + foreach (var table in tables) + { + try + { + _logger.LogInformation($"Start processing table {table}\r\n"); + + var ddl = GetTableStructure(table); + if (string.IsNullOrEmpty(ddl)) continue; + + var prompt = await GetPrompt(ddl); + var response = await GetAiResponse(prompt, provider, model); + var knowledges = response.Content.JsonArrayContent(); + + if (knowledges.IsNullOrEmpty()) + { + _logger.LogInformation($"No knowledge for table {table}"); + continue; + } + + foreach (var item in knowledges) + { + await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + { + Text = item.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, item.Answer } + } + }); + + _logger.LogInformation($"Knowledge {table} is saved =>\r\nQuestion: {item.Question}\r\nAnswer: {item.Answer}\r\n"); + } + } + catch (Exception ex) + { + var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}"; + _logger.LogWarning(note); + } + } + + return true; + } + + private string GetTableStructure(string table) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); + + var ddl = string.Empty; + var escapedTableName = MySqlHelper.EscapeString(table); + var sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + ddl = reader.GetString(1); + } + + reader.Close(); + command.Dispose(); + connection.Close(); + return ddl; + } + + private async Task GetPrompt(string content) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; + + return render.Render(template, new Dictionary + { + { "table_structure", content } + }); + } + + private async Task GetAiResponse(string prompt, string provider, string model) + { + var agent = new Agent + { + Id = string.Empty, + Name = "Db knowledge", + Instruction = prompt, + }; + + var completion = CompletionProvider.GetChatCompletion(_services, provider, model); + return await completion.GetChatCompletions(agent, new List()); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index aaad1a0f..c059edfb 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -16,6 +16,7 @@ public class SqlDriverPlugin : IBotSharpPlugin }); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index 1886cfa0..c0c2d076 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -12,6 +12,7 @@ namespace BotSharp.Plugin.TencentCos.Modules private readonly string _fullBucketName; private readonly string _appId; private readonly string _region; + public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region) { _cosXml = cosXml; @@ -141,7 +142,6 @@ namespace BotSharp.Plugin.TencentCos.Modules var objects = info.contentsList; return objects.Where(o => o.size > 0).Select(o => o.key).ToList(); - } catch (CosClientException clientEx) { @@ -153,6 +153,11 @@ namespace BotSharp.Plugin.TencentCos.Modules } } + public string? GetDirFile(string dir, string key) + { + return GetDirFiles(dir).FirstOrDefault(x => x == key); + } + public List GetDirectories(string dir) { var dirs = new List(); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs index 4d803628..ec1a4161 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -2,13 +2,38 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - throw new NotImplementedException(); + try + { + var file = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}/{fileName}"; + var exist = _cosClient.BucketClient.DoesObjectExist(file); + if (exist) + { + return false; + } + + return _cosClient.BucketClient.UploadBytes(file, data.ToArray()); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + public BinaryData GetSpeechFile(string conversationId, string fileName) { - throw new NotImplementedException(); + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}"; + var key = $"{dir}/{fileName}"; + var file = _cosClient.BucketClient.GetDirFile(dir, key); + + if (string.IsNullOrWhiteSpace(file)) + { + return BinaryData.Empty; + } + + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + return BinaryData.FromBytes(bytes); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e94ab6ce..d21a4c1a 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -7,6 +7,24 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrEmpty(relativePath)) + { + return Enumerable.Empty(); + } + + try + { + return _cosClient.BucketClient.GetDirFiles(relativePath); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + public byte[] GetFileBytes(string fileStorageUrl) { try @@ -15,9 +33,9 @@ public partial class TencentCosService } catch (Exception ex) { - _logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}"); + return Array.Empty(); } - return Array.Empty(); } public bool SaveFileStreamToPath(string filePath, Stream stream) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 7e2462de..ff8dfa17 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) @@ -115,7 +115,7 @@ public partial class TencentCosService return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; @@ -254,7 +254,7 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index de222c43..a2308539 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -16,7 +16,7 @@ public partial class TencentCosService return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index ddb5e031..c48812f1 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -27,7 +27,7 @@ public partial class TencentCosService : IFileStorageService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; public TencentCosService( TencentCosSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 068fdbb6..8304e3b2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -32,7 +32,11 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/welcome")] public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { - if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; @@ -47,8 +51,10 @@ public class TwilioVoiceController : TwilioController var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); + if (!string.IsNullOrWhiteSpace(text)) { messages.Add(text); @@ -66,6 +72,7 @@ public class TwilioVoiceController : TwilioController Content = messageContent, From = request.From }; + if (!string.IsNullOrEmpty(states)) { var kvp = states.Split(':'); @@ -74,10 +81,8 @@ public class TwilioVoiceController : TwilioController callerMessage.States.Add(kvp[0], kvp[1]); } } - await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse() - .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); } else { @@ -111,12 +116,16 @@ public class TwilioVoiceController : TwilioController var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + if (request.SpeechResult != null) { await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); } + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; + if (reply == null) { var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); @@ -133,11 +142,11 @@ public class TwilioVoiceController : TwilioController } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(seg); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(seg); + var fileName = $"indication_{seqNum}_{segIndex}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); segIndex++; } @@ -162,21 +171,26 @@ public class TwilioVoiceController : TwilioController } else { - response = twilio.ReturnInstructions(new List { $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); + response = twilio.ReturnInstructions(new List + { + $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" + }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } - } + return TwiML(response); } [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] - public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); - var result = new FileContentResult(data.ToArray(), "audio/mpeg"); - result.FileDownloadName = fileName; + var fileStorage = _services.GetRequiredService(); + var data = fileStorage.GetSpeechFile(conversationId, fileName); + var result = new FileContentResult(data.ToArray(), "audio/mpeg") + { + FileDownloadName = fileName + }; return result; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs index 455e314c..dfdc66c3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -7,6 +7,7 @@ namespace BotSharp.Plugin.Twilio.Services { private readonly Channel _queue; internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() { BoundedChannelOptions options = new(100) @@ -18,7 +19,11 @@ namespace BotSharp.Plugin.Twilio.Services public async ValueTask EnqueueAsync(CallerMessage request) { - if (request == null) throw new ArgumentNullException(nameof(request)); + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); await _queue.Writer.WriteAsync(request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 3338c46d..39eac1cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; using Microsoft.Extensions.Hosting; -using System; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -58,22 +57,26 @@ namespace BotSharp.Plugin.Twilio.Services { using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); var config = sp.GetRequiredService(); + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); var states = new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) }; + foreach (var kvp in message.States) { states.Add(new MessageState(kvp.Key, kvp.Value)); } conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); var result = await conv.SendMessage(config.AgentId, inputMsg, @@ -94,14 +97,15 @@ namespace BotSharp.Plugin.Twilio.Services await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); } }, - async functionExecuted => - { } + async functionExecuted => { } ); - var textToSpeechService = CompletionProvider.GetTextToSpeech(sp, "openai", "tts-1"); - var fileService = sp.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileService.SaveSpeechFileAsync(message.ConversationId, fileName, data); + fileStorage.SaveSpeechFile(message.ConversationId, fileName, data); + reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index ee1a31e7..cecbd3c8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Utilities; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -59,6 +60,7 @@ public class TwilioService }, Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}") }; + gather.Say(message); response.Append(gather); return response; @@ -80,7 +82,8 @@ public class TwilioService Timeout = timeout > 0 ? timeout : 3, ActionOnEmptyResult = actionOnEmptyResult }; - if (speechPaths != null && speechPaths.Any()) + + if (!speechPaths.IsNullOrEmpty()) { foreach (var speechPath in speechPaths) { @@ -144,6 +147,7 @@ public class TwilioService Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), ActionOnEmptyResult = true }; + if (!string.IsNullOrEmpty(message)) { gather.Say(message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index 40924fba..f5231635 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -25,9 +25,7 @@ namespace BotSharp.Plugin.Twilio.Services { var db = _redis.GetDatabase(); var key = $"{conversationId}:Caller:{seqNum}"; - return (await db.ListRangeAsync(key)) - .Select(x => (string)x) - .ToList(); + return (await db.ListRangeAsync(key)).Select(x => (string)x).ToList(); } public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index fa971a7c..2a6d7c22 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -17,9 +17,12 @@ public class TwilioPlugin : IBotSharpPlugin var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); + services.AddScoped(); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); services.AddSingleton(); services.AddHostedService(); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c34fc791..4fcc745c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -233,8 +233,12 @@ "FileCore": { "Storage": "LocalFileStorage", - "Pdf2TextConverter": "", - "Pdf2ImageConverter": "" + "Pdf2TextConverter": { + "Provider": "" + }, + "Pdf2ImageConverter": { + "Provider": "" + } }, "TencentCos": { @@ -250,7 +254,8 @@ }, "Graph": { - "BaseUrl": "" + "BaseUrl": "", + "SearchPath": "" }, "WeChat": { @@ -262,8 +267,12 @@ }, "KnowledgeBase": { - "VectorDb": "Qdrant", - "GraphDb": "Default", + "VectorDb": { + "Provider": "Qdrant" + }, + "GraphDb": { + "Provider": "Remote" + }, "Default": { "CollectionName": "BotSharp", "TextEmbedding": { @@ -345,6 +354,7 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", + "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.TencentCos", "BotSharp.Plugin.PythonInterpreter" ]