From 96b891897ca28ad4c5eeb4ae6783b7a63cf75211 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 26 Aug 2024 17:24:07 -0500 Subject: [PATCH 01/27] add instruct api and clean code --- .../Files/IFileInstructService.cs | 6 +- .../MLTasks/ISpeechToText.cs | 6 +- .../MLTasks/Settings/LlmModelSetting.cs | 3 +- .../Instruct/FileInstructService.Audio.cs | 19 ++++ .../Instruct/FileInstructService.Image.cs | 4 +- .../Instruct/FileInstructService.Pdf.cs | 6 +- .../Services/Instruct/FileInstructService.cs | 1 + .../Controllers/InstructModeController.cs | 39 ++++++- .../Instructs/AudioCompletionViewModel.cs | 5 + .../Instructs/ImageGenerationViewModel.cs | 9 +- .../Instructs/InstructBaseViewModel.cs | 13 +++ .../Instructs/InstructMessageModel.cs | 2 - .../Instructs/PdfCompletionViewModel.cs | 10 +- .../AudioHandlerPlugin.cs | 36 +++---- .../Controllers/AudioController.cs | 15 +-- .../Enums/AudioType.cs | 39 +++---- .../Enums/UtilityName.cs | 13 +-- .../Functions/HandleAudioRequestFn.cs | 59 +++++------ .../Functions/IAudioProcessUtilities.cs | 10 -- .../AudioHelper.cs} | 94 +++++++++------- .../Helpers/IAudioHelper.cs | 6 ++ .../Hooks/AudioHandlerHook.cs | 100 ++++++++---------- .../Hooks/AudioHandlerUtilityHook.cs | 7 -- .../LlmContexts/LlmContextIn.cs | 5 - .../LlmContexts/LlmContextOut.cs | 5 - .../Models/AudioOutput.cs | 22 ++-- .../Provider/NativeWhisperProvider.cs | 72 +++++++------ .../Settings/AudioHandlerSettings.cs | 7 +- .../BotSharp.Plugin.AudioHandler/Using.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../Providers/Audio/SpeechToTextProvider.cs | 48 +++++---- .../Providers/Audio/TextToSpeechProvider.cs | 19 ++-- 32 files changed, 345 insertions(+), 339 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs rename src/Plugins/BotSharp.Plugin.AudioHandler/{Functions/AudioProcessUtilities.cs => Helpers/AudioHelper.cs} (59%) create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 78a400e1..e9ccb4ba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); Task VaryImage(string? provider, string? model, BotSharpFile image); Task EditImage(string? provider, string? model, string text, BotSharpFile image); @@ -20,6 +20,10 @@ public interface IFileInstructService Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); #endregion + #region Audio + Task ReadAudio(string? provider, string? model, BotSharpFile audio); + #endregion + #region Select file Task> SelectMessageFiles(string conversationId, SelectFileOptions options); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs index 9e0dd574..a1af443e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -1,3 +1,5 @@ +using System.IO; + namespace BotSharp.Abstraction.MLTasks; public interface ISpeechToText @@ -5,6 +7,6 @@ public interface ISpeechToText string Provider { get; } Task GenerateTextFromAudioAsync(string filePath); - // Task AudioToTextTranscript(Stream stream); - Task SetModelName(string modelType); + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName); + Task SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index cc7cfba0..008f528b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -68,5 +68,6 @@ public enum LlmModelType Text = 1, Chat = 2, Image = 3, - Embedding = 4 + Embedding = 4, + Audio = 5 } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs new file mode 100644 index 00000000..f3f40f2c --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -0,0 +1,19 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task ReadAudio(string? provider, string? model, BotSharpFile audio) + { + var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var audioBytes = await DownloadFile(audio); + using var stream = new MemoryStream(); + stream.Write(audioBytes, 0, audioBytes.Length); + stream.Position = 0; + + var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty); + stream.Close(); + return content; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index c9d35cb7..244b5ac4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -17,7 +17,7 @@ public partial class FileInstructService Files = images?.ToList() ?? new List() } }); - return message; + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2aec257b..dd504fbf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -80,9 +80,9 @@ public partial class FileInstructService var fileDir = _fileStorage.BuildDirectory(dir, guid); DeleteIfExistDirectory(fileDir, true); - var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileStorage.SaveFileBytesToPath(pdfDir, bytes); - locs.Add(pdfDir); + var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(outputDir, bytes); + locs.Add(outputDir); } } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index acd0ddaa..416d9f30 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 48fdafe1..0e60f331 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.OpenAPI.Controllers; @@ -87,8 +88,8 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); - return message.Content; + var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); + return content; } catch (Exception ex) { @@ -169,7 +170,7 @@ public class InstructModeController : ControllerBase var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); if (image == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; @@ -199,7 +200,7 @@ public class InstructModeController : ControllerBase var mask = input.Mask; if (image == null || mask == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; @@ -240,4 +241,34 @@ public class InstructModeController : ControllerBase } } #endregion + + #region Audio + [HttpPost("/instruct/audio-completion")] + public async Task AudioCompletion([FromBody] IncomingMessageModel input) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new AudioCompletionViewModel(); + + try + { + var audio = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + if (audio == null) + { + return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; + } + var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in audio completion. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs new file mode 100644 index 00000000..3c3501ae --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class AudioCompletionViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 0050de01..ea8dc076 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -2,18 +2,11 @@ using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class ImageGenerationViewModel +public class ImageGenerationViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - [JsonPropertyName("images")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable Images { get; set; } = new List(); - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } public class ImageViewModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs new file mode 100644 index 00000000..0b20fca3 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructBaseViewModel +{ + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs index 3b265e04..b736330f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.OpenAPI.ViewModels.Instructs; public class InstructMessageModel : IncomingMessageModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs index 13ed3eb9..7ac594e6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs @@ -1,13 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class PdfCompletionViewModel +public class PdfCompletionViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 304aa97b..855c0081 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,31 +1,25 @@ using BotSharp.Plugin.AudioHandler.Settings; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Abstraction.Settings; -namespace BotSharp.Plugin.AudioHandler +namespace BotSharp.Plugin.AudioHandler; + +public class AudioHandlerPlugin : IBotSharpPlugin { - public class AudioHandlerPlugin : IBotSharpPlugin + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) { - public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; - public string Name => "Audio Handler"; - public string Description => "Process audio input and transform it into text output."; - public void RegisterDI(IServiceCollection services, IConfiguration config) + services.AddScoped(provider => { - //var settings = new AudioHandlerSettings(); - //config.Bind("AudioHandler", settings); - //services.AddSingleton(x => settings); + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); - services.AddScoped(provider => - { - var settingService = provider.GetRequiredService(); - return settingService.Bind("AudioHandler"); - }); - - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - } + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index 45f4c8a4..60b2dc15 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -1,11 +1,4 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.AudioHandler.Controllers @@ -38,9 +31,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); #endif return Ok(result); @@ -58,9 +49,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); #endif return Ok(result); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs index 436b2922..7b299a76 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -1,32 +1,23 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Whisper.net.Wave; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public enum AudioType { - public enum AudioType - { - wav, - mp3, - } + wav, + mp3, +} - public static class AudioTypeExtensions +public static class AudioTypeExtensions +{ + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + public static string ToFileType(this AudioType audioType) { - public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; - public static string ToFileType(this AudioType audioType) + string type = audioType switch { - string type = audioType switch - { - AudioType.mp3 => "audio/mpeg", - AudioType.wav => "audio/wav", - _ => throw new NotImplementedException($"No support found for {audioType}") - }; - return type; - } + AudioType.mp3 => "audio/mpeg", + AudioType.wav => "audio/wav", + _ => throw new NotImplementedException($"No support found for {audioType}") + }; + return type; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs index 0deab6e9..d11bd65a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public class UtilityName { - public class UtilityName - { - public const string AudioHandler = "audio-handler"; - } + public const string AudioHandler = "audio-handler"; } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 0c2298c6..37016220 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using Microsoft.AspNetCore.StaticFiles; @@ -12,7 +11,6 @@ public class HandleAudioRequestFn : IFunctionCallback private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly BotSharpOptions _options; - private Agent? _agent; private readonly IEnumerable _audioContentType = new List { @@ -20,12 +18,10 @@ public class HandleAudioRequestFn : IFunctionCallback AudioType.wav.ToFileType(), }; - public HandleAudioRequestFn( IServiceProvider serviceProvider, ILogger logger, - BotSharpOptions options - ) + BotSharpOptions options) { _serviceProvider = serviceProvider; _logger = logger; @@ -36,20 +32,18 @@ public class HandleAudioRequestFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); var conv = _serviceProvider.GetRequiredService(); - var isNeedSummary = args?.IsNeedSummary ?? false; var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText; + var response = await GetResponeFromDialogs(dialogs); message.Content = response; return true; } - private async Task> AssembleFiles(string convId, List dialogs) + private List AssembleFiles(string convId, List dialogs) { - if (dialogs.IsNullOrEmpty()) - return new List(); + if (dialogs.IsNullOrEmpty()) return new List(); var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); @@ -60,8 +54,7 @@ public class HandleAudioRequestFn : IFunctionCallback foreach (var dialog in dialogs) { var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList(); - if (found.IsNullOrEmpty()) - continue; + if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { @@ -74,28 +67,20 @@ public class HandleAudioRequestFn : IFunctionCallback return dialogs; } - private bool ParseAudioFileType(string fileType) - { - fileType = fileType.ToLower(); - var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType); - return canParse; - } - private async Task GetResponeFromDialogs(List dialogs) { - var whisperService = await PrepareModel("native"); // openai, native + var speech2Text = await PrepareModel("native"); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); int transcribedCount = 0; + foreach (var file in dialog.Files) { - if (file == null) - continue; + if (file == null) continue; string extension = Path.GetExtension(file?.FileStorageUrl); if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) { - file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl); + file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl); transcribedCount++; } } @@ -104,23 +89,33 @@ public class HandleAudioRequestFn : IFunctionCallback { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } + var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); return string.Join("\n\r", resList); } - private async Task PrepareModel(string modelName = "native") + private async Task PrepareModel(string provider = "native") { - var whisperService = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == modelName.ToLower()); - if (whisperService == null) + var speech2Text = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == provider.ToLower()); + if (speech2Text == null) { - throw new Exception($"Can't resolve speech2text provider by {modelName}"); + throw new Exception($"Can't resolve speech2text provider by {provider}"); } - if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase)) + if (provider.IsEqualTo("openai")) { return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); } - await whisperService.SetModelName("Tiny"); - return whisperService; + + await speech2Text.SetModelName("Tiny"); + return speech2Text; + } + + private bool ParseAudioFileType(string fileType) + { + fileType = fileType.ToLower(); + var provider = new FileExtensionContentTypeProvider(); + bool canParse = Enum.TryParse(fileType, out _) || provider.TryGetContentType(fileType, out _); + return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs deleted file mode 100644 index a3c8243b..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace BotSharp.Plugin.AudioHandler.Functions -{ - public interface IAudioProcessUtilities - { - Stream ConvertMp3ToStream(string mp3FileName); - Stream ConvertWavToStream(string wavFileName); - Stream ConvertToStream(string fileName); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs similarity index 59% rename from src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs rename to src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs index a6544359..122273c8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -1,59 +1,36 @@ -using BotSharp.Plugin.AudioHandler.Enums; -using NAudio; using NAudio.Wave; using NAudio.Wave.SampleProviders; -namespace BotSharp.Plugin.AudioHandler.Functions; +namespace BotSharp.Plugin.AudioHandler.Helpers; -public class AudioProcessUtilities : IAudioProcessUtilities +public class AudioHelper : IAudioHelper { - public AudioProcessUtilities() - { - } + private readonly IServiceProvider _services; + private readonly ILogger _logger; - public Stream ConvertMp3ToStream(string mp3FileName) + public AudioHelper( + IServiceProvider services, + ILogger logger) { - var fileStream = File.OpenRead(mp3FileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; - - } - - public Stream ConvertWavToStream(string wavFileName) - { - var fileStream = File.OpenRead(wavFileName); - using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + _services = services; + _logger = logger; } public Stream ConvertToStream(string fileName) { if (string.IsNullOrEmpty(fileName)) { - throw new ArgumentNullException("fileName is Null"); + var error = "fileName is Null when converting to stream in audio processor"; + _logger.LogWarning(error); + throw new ArgumentNullException(error); } - string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); - if (!Enum.TryParse(fileExtension, out AudioType fileType)) + + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); + var error = $"File extension: '{fileExtension}' is not supported!"; + _logger.LogWarning(error); + throw new NotSupportedException(error); } var stream = fileType switch @@ -65,4 +42,39 @@ public class AudioProcessUtilities : IAudioProcessUtilities return stream; } + + + private Stream ConvertMp3ToStream(string fileName) + { + var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + } + + private Stream ConvertWavToStream(string fileName) + { + var fileStream = File.OpenRead(fileName); + using var reader = new WaveFileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs new file mode 100644 index 00000000..d096a526 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.AudioHandler.Helpers; + +public interface IAudioHelper +{ + Stream ConvertToStream(string fileName); +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index c3ff10d0..51edcefa 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,73 +1,67 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; -namespace BotSharp.Plugin.AudioHandler.Hooks +namespace BotSharp.Plugin.AudioHandler.Hooks; + +public class AudioHandlerHook : AgentHookBase, IAgentHook { - public class AudioHandlerHook : AgentHookBase, IAgentHook + private const string HANDLER_AUDIO = "handle_audio_request"; + + public override string SelfId => string.Empty; + + public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - private const string HANDLER_AUDIO = "handle_audio_request"; + } - public override string SelfId => string.Empty; + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + if (isEnabled && isConvMode) { + AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); } - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); + base.OnAgentLoaded(agent); + } - if (isEnabled && isConvMode) + private void AddUtility(Agent agent, string utility, string functionName) + { + if (!IsEnableUtility(agent, utility)) return; + + var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; } - base.OnAgentLoaded(agent); - } - - private void AddUtility(Agent agent, string utility, string functionName) - { - if (!IsEnableUtility(agent, utility)) return; - - var (prompt, fn) = GetPromptAndFunction(functionName); - if (fn != null) + if (agent.Functions == null) { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } - - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); } - } - - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - - private (string, FunctionDef?) GetPromptAndFunction(string functionName) - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); - return (prompt, loadAttachmentFn); } } + + private bool IsEnableUtility(Agent agent, string utility) + { + return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); + } + + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs index f220acdf..ac3f0ed7 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs @@ -1,10 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Abstraction.Agents; - namespace BotSharp.Plugin.AudioHandler.Hooks; public class AudioHandlerUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs index 5132ef3a..281305e8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs index 3db02f71..ba75464d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs index 1b58f455..f8f0bf51 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -1,19 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Whisper.net; -namespace BotSharp.Plugin.AudioHandler.Models -{ - public class AudioOutput - { - public List Segments { get; set; } +namespace BotSharp.Plugin.AudioHandler.Models; - public override string ToString() - { - return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; - } +public class AudioOutput +{ + public List Segments { get; set; } = new(); + + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 7afae1fc..d95f0258 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Core.Agents.Services; using Whisper.net; using Whisper.net.Ggml; @@ -9,47 +8,47 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// public class NativeWhisperProvider : ISpeechToText { + private readonly IAudioHelper _audioProcessor; + private static WhisperProcessor _whisperProcessor; + private readonly ILogger _logger; + public string Provider => "native"; - private readonly IAudioProcessUtilities _audioProcessUtilities; - private static WhisperProcessor _processor; - private readonly ILogger _logger; private string MODEL_DIR = "model"; private string? _currentModelPath; + private Dictionary _modelPathDict = new Dictionary(); private GgmlType? _modelType; public NativeWhisperProvider( - IAudioProcessUtilities audioProcessUtilities, + IAudioHelper audioProcessor, ILogger logger) { - _audioProcessUtilities = audioProcessUtilities; + _audioProcessor = audioProcessor; _logger = logger; } public async Task GenerateTextFromAudioAsync(string filePath) { string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) { throw new Exception($"Unsupported audio type: {fileExtension}"); } - using var stream = _audioProcessUtilities.ConvertToStream(filePath); - + using var stream = _audioProcessor.ConvertToStream(filePath); if (stream == null) { throw new Exception($"Failed to convert {fileExtension} to stream"); } var textResult = new List(); - - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } - _processor.Dispose(); + _whisperProcessor.Dispose(); var audioOutput = new AudioOutput { @@ -57,17 +56,35 @@ public class NativeWhisperProvider : ISpeechToText }; return audioOutput.ToString(); } + + public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + { + throw new NotImplementedException(); + } + + public async Task SetModelName(string model) + { + if (Enum.TryParse(model, true, out GgmlType ggmlType)) + { + await LoadWhisperModel(ggmlType); + return; + } + + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + await LoadWhisperModel(GgmlType.Tiny); + } + private async Task LoadWhisperModel(GgmlType modelType) { try { if (!Directory.Exists(MODEL_DIR)) + { Directory.CreateDirectory(MODEL_DIR); + } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin") - .ToArray(); - - if (!availableModelPaths.Any()) + var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray(); + if (availableModelPaths.IsNullOrEmpty()) { _currentModelPath = SetModelPath(MODEL_DIR, modelType); await DownloadModel(modelType, _currentModelPath); @@ -86,17 +103,14 @@ public class NativeWhisperProvider : ISpeechToText } } - _processor = WhisperFactory - .FromPath(path: _currentModelPath) - .CreateBuilder() - .WithLanguage("auto") - .Build(); - + _whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build(); _modelType = modelType; } catch (Exception ex) { - throw new Exception($"Failed to load whisper model: {ex.Message}"); + var error = "Failed to load whisper model"; + _logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}"); + throw new Exception($"{error}: {ex.Message}"); } } @@ -112,16 +126,4 @@ public class NativeWhisperProvider : ISpeechToText string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); return currentModelPath; } - - public async Task SetModelName(string modelType) - { - if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) - { - await LoadWhisperModel(ggmlType); - return; - } - - _logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index 4ace63db..fedf123c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -1,6 +1,5 @@ -namespace BotSharp.Plugin.AudioHandler.Settings +namespace BotSharp.Plugin.AudioHandler.Settings; + +public class AudioHandlerSettings { - public class AudioHandlerSettings - { - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index c15e6cc4..b1ba5249 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -20,7 +20,7 @@ global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.AudioHandler.Enums; -global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Helpers; global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; global using BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9e0ceaee..74552d17 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public class GraphDb : IGraphDb _settings = settings; } - public string Name => "Default"; + public string Name => "Neo4j"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs index 2314b431..e9b54af8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -1,14 +1,13 @@ -using System.Text; using OpenAI.Audio; namespace BotSharp.Plugin.OpenAI.Providers.Audio; public class SpeechToTextProvider : ISpeechToText { - public string Provider => "openai"; private readonly IServiceProvider _services; - private string? _modelName; - private AudioTranscriptionOptions? _options; + + public string Provider => "openai"; + private string? _model; public SpeechToTextProvider(IServiceProvider service) { @@ -17,30 +16,35 @@ public class SpeechToTextProvider : ISpeechToText public async Task GenerateTextFromAudioAsync(string filePath) { - var client = ProviderHelper - .GetClient(Provider, _modelName, _services) - .GetAudioClient(_modelName); - SetOptions(); - - var transcription = await client.TranscribeAudioAsync(filePath); - - return transcription.Value.Text; + var client = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareOptions(); + var result = await client.TranscribeAudioAsync(filePath, options); + return result.Value.Text; } - public async Task SetModelName(string modelName) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) { - _modelName = modelName; + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareOptions(); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; } - public void SetOptions(AudioTranscriptionOptions? options = null) + public async Task SetModelName(string model) { - if (_options == null) + _model = model; + } + + private AudioTranscriptionOptions PrepareOptions() + { + return new AudioTranscriptionOptions { - _options = options ?? new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + }; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs index e559e109..e109dfcd 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs @@ -4,8 +4,9 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio { public partial class TextToSpeechProvider : ITextToSpeech { - public string Provider => "openai"; private readonly IServiceProvider _services; + + public string Provider => "openai"; private string? _model; public TextToSpeechProvider( @@ -14,17 +15,17 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio _services = services; } + public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) + { + var client = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + } + public void SetModelName(string model) { _model = model; } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper - .GetClient(Provider, _model, _services) - .GetAudioClient(_model); - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } } } From 512d459c6b29c4d4028834b5f5839a9f1c246fdd Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 27 Aug 2024 10:42:48 -0500 Subject: [PATCH 02/27] refine instruct request model --- .../Files/Models/InputMessageFiles.cs | 7 -- .../BotSharp.Abstraction/Models/AiModel.cs | 10 --- .../Models/MessageConfig.cs | 6 +- .../Controllers/ConversationController.cs | 6 +- .../Controllers/InstructModeController.cs | 32 ++++----- .../Conversations/InputMessageFiles.cs | 7 ++ .../Instructs/InstructBaseRequest.cs | 66 +++++++++++++++++++ 7 files changed, 92 insertions(+), 42 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs deleted file mode 100644 index 29749bce..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class InputMessageFiles -{ - public List Files { get; set; } = new List(); - public BotSharpFile? Mask { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs b/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs deleted file mode 100644 index 348eb457..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BotSharp.Abstraction.Models; - -public class AiModel -{ - public string Id { get; set; } = string.Empty; - public string Model { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public int MaxLength { get; set; } - public int TokenLimit { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs index 24e4152c..9ea5917c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Models; -public class MessageConfig : InputMessageFiles +public class MessageConfig { /// /// Completion Provider @@ -15,7 +15,7 @@ public class MessageConfig : InputMessageFiles public virtual string? Model { get; set; } = null; /// - /// Model name + /// Model id /// [JsonPropertyName("model_id")] public virtual string? ModelId { get; set; } = null; @@ -34,7 +34,7 @@ public class MessageConfig : InputMessageFiles /// /// Conversation states from input /// - public List States { get; set; } = new List(); + public List States { get; set; } = new(); /// /// Agent task id diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 9b964dff..565677f7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -294,9 +294,7 @@ public class ConversationController : ControllerBase } [HttpPost("/conversation/{agentId}/{conversationId}/sse")] - public async Task SendMessageSse([FromRoute] string agentId, - [FromRoute] string conversationId, - [FromBody] NewMessageModel input) + public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) { var conv = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, input.Text) @@ -391,7 +389,7 @@ public class ConversationController : ControllerBase } [HttpPost("/agent/{agentId}/conversation/{conversationId}/upload")] - public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) + public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] InputMessageFiles input) { var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 0e60f331..b0ecd905 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.OpenAPI.Controllers; @@ -21,8 +20,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/{agentId}")] - public async Task InstructCompletion([FromRoute] string agentId, - [FromBody] InstructMessageModel input) + public async Task InstructCompletion([FromRoute] string agentId, [FromBody] InstructMessageModel input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -80,7 +78,7 @@ public class InstructModeController : ControllerBase #region Read image [HttpPost("/instruct/multi-modal")] - public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + public async Task MultiModalCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -102,7 +100,7 @@ public class InstructModeController : ControllerBase #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] IncomingMessageModel input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -128,7 +126,7 @@ public class InstructModeController : ControllerBase #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] IncomingMessageModel input) + public async Task ImageVariation([FromBody] ImageVariationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -136,14 +134,13 @@ public class InstructModeController : ControllerBase try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -158,7 +155,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] IncomingMessageModel input) + public async Task ImageEdit([FromBody] ImageEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -167,12 +164,11 @@ public class InstructModeController : ControllerBase try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -187,7 +183,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -196,7 +192,7 @@ public class InstructModeController : ControllerBase try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var image = input.File; var mask = input.Mask; if (image == null || mask == null) { @@ -219,7 +215,7 @@ public class InstructModeController : ControllerBase #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] IncomingMessageModel input) + public async Task PdfCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -244,7 +240,7 @@ public class InstructModeController : ControllerBase #region Audio [HttpPost("/instruct/audio-completion")] - public async Task AudioCompletion([FromBody] IncomingMessageModel input) + public async Task AudioCompletion([FromBody] AudioCompletionRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -253,7 +249,7 @@ public class InstructModeController : ControllerBase try { - var audio = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var audio = input.File; if (audio == null) { return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs new file mode 100644 index 00000000..5656f7ae --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class InputMessageFiles +{ + public List States { get; set; } = new(); + public List Files { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs new file mode 100644 index 00000000..1b5d551d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -0,0 +1,66 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructBaseRequest +{ + [JsonPropertyName("provider")] + public virtual string? Provider { get; set; } = null; + + [JsonPropertyName("model")] + public virtual string? Model { get; set; } = null; + + [JsonPropertyName("model_id")] + public virtual string? ModelId { get; set; } = null; + + [JsonPropertyName("states")] + public List States { get; set; } = new(); +} + +public class MultiModalRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("files")] + public List Files { get; set; } = new(); +} + +public class ImageGenerationRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; +} + +public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } +} + +public class ImageEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } +} + +public class ImageMaskEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } + + [JsonPropertyName("mask")] + public BotSharpFile Mask { get; set; } +} + +public class AudioCompletionRequest : InstructBaseRequest +{ + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } +} \ No newline at end of file From 1ace4e44aba53d698d78f221a7ea75656426eea9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 27 Aug 2024 12:12:17 -0500 Subject: [PATCH 03/27] refine file models --- .../Files/IFileInstructService.cs | 12 +++--- .../Files/IFileStorageService.cs | 6 +-- .../Files/Models/BotSharpFile.cs | 9 ++++- .../Files/Models/FileBase.cs | 28 -------------- .../Files/Models/FileInfo.cs | 38 +++++++++++++++++++ .../Files/Models/InputFileModel.cs | 16 ++++++++ .../Files/Models/InstructFileModel.cs | 18 +++++++++ .../Files/Models/MessageFileModel.cs | 2 +- .../BotSharp.Abstraction/Models/AiModel.cs | 10 +++++ .../Instruct/FileInstructService.Audio.cs | 5 ++- .../Instruct/FileInstructService.Image.cs | 22 ++++++----- .../Instruct/FileInstructService.Pdf.cs | 4 +- .../LocalFileStorageService.Conversation.cs | 2 +- .../Storage/LocalFileStorageService.User.cs | 2 +- .../Controllers/UserController.cs | 7 +++- .../Conversations/InputMessageFiles.cs | 2 +- .../Instructs/InstructBaseRequest.cs | 12 +++--- .../ViewModels/Users/UserAvatarModel.cs | 18 +++++++++ .../Functions/EditImageFn.cs | 4 +- .../Functions/GenerateImageFn.cs | 2 +- .../TencentCosService.Conversation.cs | 2 +- .../Services/TencentCosService.User.cs | 2 +- 22 files changed, 155 insertions(+), 68 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index e9ccb4ba..f35cdad5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + Task VaryImage(string? provider, string? model, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask); #endregion #region Pdf @@ -17,11 +17,11 @@ public interface IFileInstructService /// /// Pdf files /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); #endregion #region Audio - Task ReadAudio(string? provider, string? model, BotSharpFile audio); + Task ReadAudio(string? provider, string? model, InstructFileModel audio); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index a3e5a32d..6c98e21a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -16,7 +16,6 @@ public interface IFileStorageService string BuildDirectory(params string[] segments); #endregion - #region Conversation /// /// Get the message file screenshots for specific content types, e.g., pdf @@ -37,7 +36,7 @@ public interface IFileStorageService IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); - bool SaveMessageFiles(string conversationId, string messageId, string source, List files); + bool SaveMessageFiles(string conversationId, string messageId, string source, List files); /// /// Delete files under messages @@ -54,8 +53,9 @@ public interface IFileStorageService #region User string GetUserAvatar(); - bool SaveUserAvatar(BotSharpFile file); + bool SaveUserAvatar(InputFileModel file); #endregion + #region Speech Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); Task RetrieveSpeechFileAsync(string conversationId, string fileName); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 11a11e46..46cba595 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,12 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileBase +public class BotSharpFile : FileInfo { - + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index 3483921a..f952e7f9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models; public class FileBase { - /// - /// External file url - /// - [JsonPropertyName("file_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileUrl { get; set; } = string.Empty; - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileStorageUrl { get; set; } = string.Empty; - /// /// File name without extension /// @@ -29,18 +15,4 @@ public class FileBase [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; - - /// - /// File content type - /// - [JsonPropertyName("content_type")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ContentType { get; set; } = string.Empty; - - /// - /// File extension without dot - /// - [JsonPropertyName("file_extension")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs new file mode 100644 index 00000000..14c02f74 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileInfo +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs new file mode 100644 index 00000000..7eb350e3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InputFileModel : FileBase +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public new string FileName { get; set; } = string.Empty; + + /// + /// File data => format: "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public new string FileData { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs new file mode 100644 index 00000000..7eaddd5a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InstructFileModel : FileBase +{ + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; + + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 2a0128b6..59ada4f2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileBase +public class MessageFileModel : FileInfo { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs b/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs new file mode 100644 index 00000000..348eb457 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Models; + +public class AiModel +{ + public string Id { get; set; } = string.Empty; + public string Model { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public int MaxLength { get; set; } + public int TokenLimit { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index f3f40f2c..30b6dbe3 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadAudio(string? provider, string? model, BotSharpFile audio) + public async Task ReadAudio(string? provider, string? model, InstructFileModel audio) { var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); var audioBytes = await DownloadFile(audio); @@ -12,7 +12,8 @@ public partial class FileInstructService stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; - var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty); + var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; + var content = await completion.GenerateTextFromAudioAsync(stream, fileName); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 244b5ac4..cd65c16a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -14,7 +14,7 @@ public partial class FileInstructService { new RoleDialogModel(AgentRole.User, text) { - Files = images?.ToList() ?? new List() + Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() } }); return message.Content; @@ -30,7 +30,7 @@ public partial class FileInstructService return message; } - public async Task VaryImage(string? provider, string? model, BotSharpFile image) + public async Task VaryImage(string? provider, string? model, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -43,16 +43,17 @@ public partial class FileInstructService stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -65,16 +66,17 @@ public partial class FileInstructService stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask) { if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) @@ -94,10 +96,12 @@ public partial class FileInstructService maskStream.Write(maskBytes, 0, maskBytes.Length); maskStream.Position = 0; + var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; + var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); @@ -105,7 +109,7 @@ public partial class FileInstructService } #region Private methods - private async Task DownloadFile(BotSharpFile file) + private async Task DownloadFile(InstructFileModel file) { var bytes = new byte[0]; if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index dd504fbf..e7228b13 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { var content = string.Empty; @@ -50,7 +50,7 @@ public partial class FileInstructService } #region Private methods - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 8c2ed831..248806ec 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -118,7 +118,7 @@ public partial class LocalFileStorageService return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index 43ff9eed..d1e962cd 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -16,7 +16,7 @@ public partial class LocalFileStorageService return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a7d4c449..16399334 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -135,9 +135,14 @@ public class UserController : ControllerBase #region Avatar [HttpPost("/user/avatar")] - public bool UploadUserAvatar([FromBody] BotSharpFile file) + public bool UploadUserAvatar([FromBody] UserAvatarModel input) { var fileStorage = _services.GetRequiredService(); + var file = new InputFileModel + { + FileName = input.FileName, + FileData = input.FileData, + }; return fileStorage.SaveUserAvatar(file); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs index 5656f7ae..fdf088bb 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -3,5 +3,5 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations; public class InputMessageFiles { public List States { get; set; } = new(); - public List Files { get; set; } = new(); + public List Files { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 1b5d551d..45a1c8ab 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -23,7 +23,7 @@ public class MultiModalRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("files")] - public List Files { get; set; } = new(); + public List Files { get; set; } = new(); } public class ImageGenerationRequest : InstructBaseRequest @@ -35,7 +35,7 @@ public class ImageGenerationRequest : InstructBaseRequest public class ImageVariationRequest : InstructBaseRequest { [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } } public class ImageEditRequest : InstructBaseRequest @@ -44,7 +44,7 @@ public class ImageEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } } public class ImageMaskEditRequest : InstructBaseRequest @@ -53,14 +53,14 @@ public class ImageMaskEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } [JsonPropertyName("mask")] - public BotSharpFile Mask { get; set; } + public InstructFileModel Mask { get; set; } } public class AudioCompletionRequest : InstructBaseRequest { [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs new file mode 100644 index 00000000..0eab88e5 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Users; + +public class UserAvatarModel +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public string FileData { get; set; } = string.Empty; +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 759f94b6..7a5d8e72 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback { if (image == null) return; - var files = new List() + var files = new List() { - new BotSharpFile + new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4102ff7b..4c8fdf9e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -77,7 +77,7 @@ public class GenerateImageFn : IFunctionCallback { if (images.IsNullOrEmpty()) return; - var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new BotSharpFile + var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 7e2462de..4c339f12 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -115,7 +115,7 @@ public partial class TencentCosService return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index de222c43..a2308539 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -16,7 +16,7 @@ public partial class TencentCosService return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; From 35eb7dabd6e6d5ae7b570dfce2d071bae30c7364 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 11:08:12 -0500 Subject: [PATCH 04/27] refine api --- BotSharp.sln | 13 +- .../Files/IFileInstructService.cs | 2 +- .../Files/IFileStorageService.cs | 1 + .../Files/Models/BotSharpFile.cs | 2 +- .../Files/Utilities/FileUtility.cs | 21 ++ .../MLTasks/IAudioCompletion.cs | 13 ++ .../MLTasks/ISpeechToText.cs | 12 - .../MLTasks/ITextToSpeech.cs | 23 -- .../Instruct/FileInstructService.Audio.cs | 6 +- .../Storage/LocalFileStorageService.Common.cs | 16 ++ .../Infrastructures/CompletionProvider.cs | 28 +-- .../Controllers/InstructModeController.cs | 209 +++++++++++++++++- .../Instructs/AudioCompletionViewModel.cs | 5 - .../Instructs/InstructBaseRequest.cs | 5 +- .../Instructs/MultiModalViewModel.cs | 5 + .../Instructs/SpeechToTextViewModel.cs | 5 + .../AudioHandlerPlugin.cs | 3 +- .../BotSharp.Plugin.AudioHandler.csproj | 14 ++ .../Controllers/AudioController.cs | 58 ----- .../Functions/HandleAudioRequestFn.cs | 62 +++--- .../Helpers/AudioHelper.cs | 89 ++++---- .../Helpers/IAudioHelper.cs | 6 - .../Hooks/AudioHandlerHook.cs | 13 +- .../Provider/NativeWhisperProvider.cs | 103 ++++----- .../BotSharp.Plugin.AudioHandler/Using.cs | 1 + .../functions/handle_audio_request.json | 32 +-- .../Functions/HandleEmailSenderFn.cs | 3 +- .../Hooks/EmailReaderHook.cs | 7 - .../Hooks/EmailSenderHook.cs | 7 - src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../BotSharp.Plugin.KnowledgeBase.csproj | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../KnowledgeSettingHelper.cs} | 4 +- .../VectorHelper.cs} | 4 +- .../KnowledgeBasePlugin.cs | 2 +- .../MemVecDb/MemoryVectorDb.cs | 2 +- .../Services/KnowledgeService.cs | 2 +- .../BotSharp.Plugin.KnowledgeBase/Using.cs | 2 +- .../BotSharp.Plugin.OpenAI/OpenAiPlugin.cs | 5 +- .../AudioCompletionProvider.SpeechToText.cs | 53 +++++ .../AudioCompletionProvider.TextToSpeech.cs | 23 ++ .../Audio/AudioCompletionProvider.cs | 21 ++ .../Providers/Audio/SpeechToTextProvider.cs | 50 ----- .../Providers/Audio/TextToSpeechProvider.cs | 31 --- .../Services/TencentCosService.Common.cs | 22 +- .../Controllers/TwilioVoiceController.cs | 8 +- .../Services/TwilioMessageQueueService.cs | 9 +- src/WebStarter/appsettings.json | 1 + 49 files changed, 590 insertions(+), 421 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/{Utilities/KnowledgeSettingUtility.cs => Helpers/KnowledgeSettingHelper.cs} (87%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/{Utilities/VectorUtility.cs => Helpers/VectorHelper.cs} (96%) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs delete mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs delete mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs diff --git a/BotSharp.sln b/BotSharp.sln index 8dda48e8..d281450e 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -449,6 +451,14 @@ Global {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -503,6 +513,7 @@ Global {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} + {F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index f35cdad5..f8f4c605 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -21,7 +21,7 @@ public interface IFileInstructService #endregion #region Audio - Task ReadAudio(string? provider, string? model, InstructFileModel audio); + Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 6c98e21a..ebd7d61f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,6 +6,7 @@ public interface IFileStorageService { #region Common string GetDirectory(string conversationId); + IEnumerable GetFiles(string relativePath, string? searchQuery = null); byte[] GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); bool SaveFileBytesToPath(string filePath, byte[] bytes); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 46cba595..7a606b3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile : FileInfo { /// - /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// File data => format: "data:image/png;base64,aaaaaaaa" /// [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index df33906d..5c021159 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,4 +1,6 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.StaticFiles; +using System.IO; namespace BotSharp.Abstraction.Files.Utilities; @@ -26,6 +28,25 @@ public static class FileUtility return (contentType, Convert.FromBase64String(base64Str)); } + public static string BuildFileDataFromFile(string fileName, byte[] bytes) + { + var contentType = GetFileContentType(fileName); + var base64 = Convert.ToBase64String(bytes); + return $"data:{contentType};base64,{base64}"; + } + + public static string BuildFileDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var contentType = GetFileContentType(file.FileName); + var base64 = Convert.ToBase64String(stream.ToArray()); + stream.Close(); + + return $"data:{contentType};base64,{base64}"; + } + public static string GetFileContentType(string filePath) { string contentType; diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs new file mode 100644 index 00000000..d32624c0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -0,0 +1,13 @@ +using System.IO; + +namespace BotSharp.Abstraction.MLTasks; + +public interface IAudioCompletion +{ + string Provider { get; } + + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); + Task GenerateSpeechFromTextAsync(string text); + + void SetModelName(string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs deleted file mode 100644 index a1af443e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.IO; - -namespace BotSharp.Abstraction.MLTasks; - -public interface ISpeechToText -{ - string Provider { get; } - - Task GenerateTextFromAudioAsync(string filePath); - Task GenerateTextFromAudioAsync(Stream audio, string audioFileName); - Task SetModelName(string model); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs deleted file mode 100644 index 344fad0e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks -{ - public interface ITextToSpeech - { - /// - /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI - /// - string Provider { get; } - - /// - /// Set model name, one provider can consume different model or version(s) - /// - /// deployment name - void SetModelName(string model); - - Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); - } - - public interface ITextToSpeechOptions - { - - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index 30b6dbe3..e2aa844c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -4,16 +4,16 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadAudio(string? provider, string? model, InstructFileModel audio) + public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) { - var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); var audioBytes = await DownloadFile(audio); using var stream = new MemoryStream(); stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; - var content = await completion.GenerateTextFromAudioAsync(stream, fileName); + var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c49edbce..b16c2c2c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,6 +14,22 @@ public partial class LocalFileStorageService return dir; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + return Enumerable.Empty(); + } + + var path = Path.Combine(_baseDir, relativePath); + + if (!string.IsNullOrWhiteSpace(searchPattern)) + { + return Directory.GetFiles(path, searchPattern); + } + return Directory.GetFiles(path); + } + public byte[] GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 874a063e..d6746e09 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -28,6 +28,10 @@ public class CompletionProvider { return GetImageCompletion(services, provider: provider, model: model); } + else if (settings.Type == LlmModelType.Audio) + { + return GetAudioCompletion(services, provider: provider, model: model); + } else { return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig); @@ -108,7 +112,7 @@ public class CompletionProvider if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve completion provider by {provider}"); + logger.LogError($"Can't resolve text-embedding provider by {provider}"); } @@ -120,35 +124,19 @@ public class CompletionProvider return completer; } - public static ITextToSpeech GetTextToSpeech( + public static IAudioCompletion GetAudioCompletion( IServiceProvider services, string provider, string model) { - var completions = services.GetServices(); + var completions = services.GetServices(); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve text2speech provider by {provider}"); + logger.LogError($"Can't resolve audio-completion provider by {provider}"); } - completer.SetModelName(model); - return completer; - } - public static ISpeechToText GetSpeechToText( - IServiceProvider services, - string provider, - string model - ) - { - var completions = services.GetServices(); - var completer = completions.FirstOrDefault(x => x.Provider == provider); - if (completer == null) - { - var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve speech2text provider by {provider}"); - } completer.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b0ecd905..170cf2d2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -91,11 +92,40 @@ public class InstructModeController : ControllerBase } catch (Exception ex) { - var error = $"Error in analyzing files. {ex.Message}"; + var error = $"Error in reading images. {ex.Message}"; _logger.LogError(error); return error; } } + + [HttpPost("/instruct/multi-modal/upload")] + public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new MultiModalViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadImages(provider, model, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in reading image upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Generate image @@ -154,6 +184,38 @@ public class InstructModeController : ControllerBase } } + [HttpPost("/instruct/image-variation/upload")] + public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var image = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName).Substring(1), + FileData = data + }; + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(provider, model, image); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] ImageEditRequest input) { @@ -182,6 +244,38 @@ public class InstructModeController : ControllerBase } } + [HttpPost("/instruct/image-edit/upload")] + public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var image = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName).Substring(1), + FileData = data + }; + var message = await fileInstruct.EditImage(provider, model, text, image); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { @@ -211,6 +305,47 @@ public class InstructModeController : ControllerBase return imageViewModel; } } + + [HttpPost("/instruct/image-mask-edit/upload")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var imageData = FileUtility.BuildFileDataFromFile(image); + var imageFile = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(image.FileName), + FileExtension = Path.GetExtension(image.FileName).Substring(1), + FileData = imageData + }; + + var maskData = FileUtility.BuildFileDataFromFile(mask); + var maskFile = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(mask.FileName), + FileExtension = Path.GetExtension(mask.FileName).Substring(1), + FileData = maskData + }; + + var message = await fileInstruct.EditImage(provider, model, text, imageFile, maskFile); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } #endregion #region Pdf @@ -236,31 +371,91 @@ public class InstructModeController : ControllerBase return viewModel; } } + + [HttpPost("/instruct/pdf-completion/upload")] + public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new PdfCompletionViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in pdf completion upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Audio - [HttpPost("/instruct/audio-completion")] - public async Task AudioCompletion([FromBody] AudioCompletionRequest input) + [HttpPost("/instruct/speech-to-text")] + public async Task SpeechToText([FromBody] SpeechToTextRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - var viewModel = new AudioCompletionViewModel(); + var viewModel = new SpeechToTextViewModel(); try { var audio = input.File; if (audio == null) { - return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; + return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio); + var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio); viewModel.Content = content; return viewModel; } catch (Exception ex) { - var error = $"Error in audio completion. {ex.Message}"; + var error = $"Error in speech to text. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/speech-to-text/upload")] + public async Task SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] string? text = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text); + viewModel.Content = content; + stream.Close(); + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech-to-text upload. {ex.Message}"; _logger.LogError(error); viewModel.Message = error; return viewModel; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs deleted file mode 100644 index 3c3501ae..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BotSharp.OpenAPI.ViewModels.Instructs; - -public class AudioCompletionViewModel : InstructBaseViewModel -{ -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 45a1c8ab..396fff2f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -59,8 +59,11 @@ public class ImageMaskEditRequest : InstructBaseRequest public InstructFileModel Mask { get; set; } } -public class AudioCompletionRequest : InstructBaseRequest +public class SpeechToTextRequest : InstructBaseRequest { + [JsonPropertyName("text")] + public string? Text { get; set; } + [JsonPropertyName("file")] public InstructFileModel File { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs new file mode 100644 index 00000000..a7b15262 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class MultiModalViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs new file mode 100644 index 00000000..d681986d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class SpeechToTextViewModel : InstructBaseViewModel +{ +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 855c0081..2c289907 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -16,8 +16,7 @@ public class AudioHandlerPlugin : IBotSharpPlugin return settingService.Bind("AudioHandler"); }); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index 7218d40c..4ae8c7d2 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -22,5 +22,19 @@ + + + + + + + + + PreserveNewest + + + PreserveNewest + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs deleted file mode 100644 index 60b2dc15..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Diagnostics; -using BotSharp.Core.Infrastructures; - -namespace BotSharp.Plugin.AudioHandler.Controllers -{ -#if DEBUG - [AllowAnonymous] -#endif - [ApiController] - public class AudioController : ControllerBase - { - private readonly ISpeechToText _nativeWhisperProvider; - private readonly IServiceProvider _services; - - public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service) - { - _nativeWhisperProvider = nativeWhisperProvider; - _services = service; - } - - [HttpGet("audio/transcript")] - public async Task GetTextFromAudioController(string audioInputString, string modelType = "") - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - await _nativeWhisperProvider.SetModelName(modelType); - - var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - - [HttpPost("openai/audio/transcript")] - public async Task GetTextFromAudioOpenAiController(string filePath) - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1"); - var result = await client.GenerateTextFromAudioAsync(filePath); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 37016220..34ec530c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -9,6 +9,7 @@ public class HandleAudioRequestFn : IFunctionCallback public string Indication => "Handling audio request"; private readonly IServiceProvider _serviceProvider; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; @@ -19,10 +20,12 @@ public class HandleAudioRequestFn : IFunctionCallback }; public HandleAudioRequestFn( + IFileStorageService fileStorage, IServiceProvider serviceProvider, ILogger logger, BotSharpOptions options) { + _fileStorage = fileStorage; _serviceProvider = serviceProvider; _logger = logger; _options = options; @@ -43,11 +46,13 @@ public class HandleAudioRequestFn : IFunctionCallback private List AssembleFiles(string convId, List dialogs) { - if (dialogs.IsNullOrEmpty()) return new List(); + if (dialogs.IsNullOrEmpty()) + { + return new List(); + } - var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); @@ -69,53 +74,46 @@ public class HandleAudioRequestFn : IFunctionCallback private async Task GetResponeFromDialogs(List dialogs) { - var speech2Text = await PrepareModel("native"); + var audioCompletion = PrepareModel(); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); - int transcribedCount = 0; + var transcripts = new List(); foreach (var file in dialog.Files) { - if (file == null) continue; + if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; - string extension = Path.GetExtension(file?.FileStorageUrl); - if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) - { - file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl); - transcribedCount++; - } + var extension = Path.GetExtension(file.FileStorageUrl); + + var fileName = Path.GetFileName(file.FileStorageUrl); + if (!ParseAudioFileType(fileName)) continue; + + var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = new MemoryStream(bytes); + stream.Position = 0; + + var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName); + transcripts.Add(result); + stream.Close(); } - if (transcribedCount == 0) + if (transcripts.IsNullOrEmpty()) { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } - var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); - return string.Join("\n\r", resList); + return string.Join("\r\n\r\n", transcripts); } - private async Task PrepareModel(string provider = "native") + private IAudioCompletion PrepareModel() { - var speech2Text = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == provider.ToLower()); - if (speech2Text == null) - { - throw new Exception($"Can't resolve speech2text provider by {provider}"); - } - - if (provider.IsEqualTo("openai")) - { - return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); - } - - await speech2Text.SetModelName("Tiny"); - return speech2Text; + return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1"); } - private bool ParseAudioFileType(string fileType) + private bool ParseAudioFileType(string fileName) { - fileType = fileType.ToLower(); + var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(fileType, out _) || provider.TryGetContentType(fileType, out _); + bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _); return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs index 122273c8..4737a5e9 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -3,78 +3,83 @@ using NAudio.Wave.SampleProviders; namespace BotSharp.Plugin.AudioHandler.Helpers; -public class AudioHelper : IAudioHelper +public static class AudioHelper { - private readonly IServiceProvider _services; - private readonly ILogger _logger; + private const int DEFAULT_SAMPLE_RATE = 16000; - public AudioHelper( - IServiceProvider services, - ILogger logger) - { - _services = services; - _logger = logger; - } - - public Stream ConvertToStream(string fileName) + public static Stream ConvertToStream(string fileName) { if (string.IsNullOrEmpty(fileName)) { - var error = "fileName is Null when converting to stream in audio processor"; - _logger.LogWarning(error); - throw new ArgumentNullException(error); + throw new ArgumentNullException("fileName is Null when converting to stream in audio processor"); } var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - var error = $"File extension: '{fileExtension}' is not supported!"; - _logger.LogWarning(error); - throw new NotSupportedException(error); + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); } var stream = fileType switch { AudioType.mp3 => ConvertMp3ToStream(fileName), - AudioType.wav => ConvertWavToStream(fileName), - _ => throw new NotSupportedException("File extension not supported"), + _ => ConvertWavToStream(fileName) }; return stream; } - - private Stream ConvertMp3ToStream(string fileName) + public static Stream Transform(Stream stream, string fileName) { - var fileStream = File.OpenRead(fileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + Stream resultStream = new MemoryStream(); + stream.CopyTo(resultStream); + resultStream.Seek(0, SeekOrigin.Begin); + + WaveStream reader = fileType switch + { + AudioType.mp3 => new Mp3FileReader(resultStream), + _ => new WaveFileReader(resultStream) + }; + + resultStream = ChangeSampleRate(reader); + reader.Close(); + return resultStream; } - private Stream ConvertWavToStream(string fileName) + private static Stream ConvertMp3ToStream(string fileName) { - var fileStream = File.OpenRead(fileName); + using var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + return ChangeSampleRate(reader); + } + + private static Stream ConvertWavToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) + return ChangeSampleRate(reader); + } + + private static Stream ChangeSampleRate(WaveStream ws) + { + var ms = new MemoryStream(); + if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE) { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; + var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE); + WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16()); + } + else + { + ws.CopyTo(ms); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + ms.Seek(0, SeekOrigin.Begin); + return ms; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs deleted file mode 100644 index d096a526..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Plugin.AudioHandler.Helpers; - -public interface IAudioHelper -{ - Stream ConvertToStream(string fileName); -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index 51edcefa..80acb149 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; namespace BotSharp.Plugin.AudioHandler.Hooks; @@ -22,17 +21,16 @@ public class AudioHandlerHook : AgentHookBase, IAgentHook if (isEnabled && isConvMode) { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + AddUtility(agent, HANDLER_AUDIO); } base.OnAgentLoaded(agent); } - private void AddUtility(Agent agent, string utility, string functionName) + private void AddUtility(Agent agent, string functionName) { - if (!IsEnableUtility(agent, utility)) return; - var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) { if (!string.IsNullOrWhiteSpace(prompt)) @@ -51,11 +49,6 @@ public class AudioHandlerHook : AgentHookBase, IAgentHook } } - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - private (string, FunctionDef?) GetPromptAndFunction(string functionName) { var db = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index d95f0258..9bf049ac 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -6,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// /// Native Whisper provider for speech to text conversion /// -public class NativeWhisperProvider : ISpeechToText +public class NativeWhisperProvider : IAudioCompletion { - private readonly IAudioHelper _audioProcessor; private static WhisperProcessor _whisperProcessor; + + private readonly IServiceProvider _services; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; public string Provider => "native"; - private string MODEL_DIR = "model"; - private string? _currentModelPath; - - private Dictionary _modelPathDict = new Dictionary(); - private GgmlType? _modelType; - public NativeWhisperProvider( - IAudioHelper audioProcessor, + BotSharpDatabaseSettings dbSettings, + IFileStorageService fileStorage, + IServiceProvider services, ILogger logger) { - _audioProcessor = audioProcessor; + _fileStorage = fileStorage; + _services = services; _logger = logger; } - public async Task GenerateTextFromAudioAsync(string filePath) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) { - string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - - using var stream = _audioProcessor.ConvertToStream(filePath); - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - var textResult = new List(); + + using var stream = AudioHelper.Transform(audio, audioFileName); await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } _whisperProcessor.Dispose(); + stream.Close(); var audioOutput = new AudioOutput { @@ -57,54 +47,45 @@ public class NativeWhisperProvider : ISpeechToText return audioOutput.ToString(); } - public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + public async Task GenerateSpeechFromTextAsync(string text) { throw new NotImplementedException(); } - public async Task SetModelName(string model) + public void SetModelName(string model) { if (Enum.TryParse(model, true, out GgmlType ggmlType)) { - await LoadWhisperModel(ggmlType); - return; + LoadWhisperModel(ggmlType); + } + else + { + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + LoadWhisperModel(GgmlType.Tiny); } - - _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); } - private async Task LoadWhisperModel(GgmlType modelType) + private void LoadWhisperModel(GgmlType modelType) { try { - if (!Directory.Exists(MODEL_DIR)) + var modelDir = _fileStorage.BuildDirectory("models", "whisper"); + var exist = _fileStorage.ExistDirectory(modelDir); + if (!exist) { - Directory.CreateDirectory(MODEL_DIR); + _fileStorage.CreateDirectory(modelDir); } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray(); - if (availableModelPaths.IsNullOrEmpty()) + var files = _fileStorage.GetFiles("models/whisper", "*.bin"); + var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType)); + if (string.IsNullOrEmpty(modelLoc)) { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin"); - if (modelFilePath == null) - { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - _currentModelPath = modelFilePath; - } + modelLoc = BuildModelPath(modelType); + DownloadModel(modelType, modelLoc); } - _whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build(); - _modelType = modelType; + var bytes = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { @@ -114,16 +95,20 @@ public class NativeWhisperProvider : ISpeechToText } } - private async Task DownloadModel(GgmlType modelType, string modelDir) + private void DownloadModel(GgmlType modelType, string modelDir) { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType); - using var fileWriter = File.OpenWrite(modelDir); - await modelStream.CopyToAsync(fileWriter); + using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); + modelStream.Close(); } - private string SetModelPath(string rootPath, GgmlType modelType) + private string BuildModelPath(GgmlType modelType) { - string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); - return currentModelPath; + return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType)); + } + + private string BuildModelFile(GgmlType modelType) + { + return $"ggml-{modelType}.bin"; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index b1ba5249..1a1188fd 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -5,6 +5,7 @@ global using System.Linq; global using System.Text.Json; global using System.Threading.Tasks; +global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json index b223c7ae..c0603b1a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json @@ -1,18 +1,18 @@ { - "name": "handle_audio_request", - "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", - "parameters": { - "type": "object", - "properties": { - "user_request": { - "type": "string", - "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" - }, - "is_need_summary": { - "type": "boolean", - "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" - } - }, - "required": [ "user_request" ] - } + "name": "handle_audio_request", + "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" + }, + "is_need_summary": { + "type": "boolean", + "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" + } + }, + "required": [ "user_request" ] + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b4f9a6e4..fa038811 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -84,11 +84,12 @@ public class HandleEmailSenderFn : IFunctionCallback { if (files.IsNullOrEmpty()) return; + var fileStorage = _services.GetRequiredService(); + foreach (var file in files) { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs index 222c27b9..beece9e2 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs index d4ae8c16..27491015 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 74552d17..55a11ec9 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public class GraphDb : IGraphDb _settings = settings; } - public string Name => "Neo4j"; + public string Name => "Remote"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 42663445..29c81ec7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 060e2150..7a7db8ae 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -18,7 +18,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); + var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f709e3b8..f77954d8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -18,7 +18,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); + var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorsAsync(new List { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs similarity index 87% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index 770e1efa..b1b280eb 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class KnowledgeSettingUtility +public static class KnowledgeSettingHelper { public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs index f0538af0..88a6ebb5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs @@ -2,9 +2,9 @@ using BotSharp.Plugin.KnowledgeBase.MemVecDb; using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class VectorUtility +public static class VectorHelper { public static float[] CalEuclideanDistance(float[] vec, List records) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 213825b5..4350e33c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -35,7 +35,7 @@ public class KnowledgeBasePlugin : IBotSharpPlugin SubMenu = new List { new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"), - new PluginMenuDef("Relations", link: "page/knowledge-base/relations") + new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships") } }); return true; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 0d4be819..aa5c0f4e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -40,7 +40,7 @@ public class MemoryVectorDb : IVectorDb return new List(); } - var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index eec49891..4e40fea4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -33,6 +33,6 @@ public partial class KnowledgeService : IKnowledgeService private ITextEmbedding GetTextEmbedding(string collection) { - return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection); + return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 7eeea4f0..a78afefd 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -33,4 +33,4 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; -global using BotSharp.Plugin.KnowledgeBase.Utilities; \ No newline at end of file +global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 4b41ec46..e6d0637c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -4,8 +4,8 @@ using BotSharp.Plugin.OpenAI.Providers.Embedding; using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; -using Microsoft.Extensions.Configuration; using BotSharp.Plugin.OpenAI.Providers.Audio; +using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.OpenAI; @@ -31,7 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 00000000..cf4837a1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,53 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var options = new AudioTranscriptionOptions + { + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format) + { + var value = !string.IsNullOrEmpty(format) ? format : "verbose"; + + AudioTranscriptionFormat retFormat; + switch (value) + { + case "json": + retFormat = AudioTranscriptionFormat.Simple; + break; + case "srt": + retFormat = AudioTranscriptionFormat.Srt; + break; + case "vtt": + retFormat = AudioTranscriptionFormat.Vtt; + break; + default: + retFormat = AudioTranscriptionFormat.Verbose; + break; + } + + return retFormat; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 00000000..a1ad2a1f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,23 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateSpeechFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + return result.Value; + } + + private SpeechGenerationOptions PrepareGenerationOptions() + { + return new SpeechGenerationOptions + { + + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 00000000..3311fd43 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,21 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs deleted file mode 100644 index e9b54af8..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ /dev/null @@ -1,50 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio; - -public class SpeechToTextProvider : ISpeechToText -{ - private readonly IServiceProvider _services; - - public string Provider => "openai"; - private string? _model; - - public SpeechToTextProvider(IServiceProvider service) - { - _services = service; - } - - public async Task GenerateTextFromAudioAsync(string filePath) - { - var client = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - var options = PrepareOptions(); - var result = await client.TranscribeAudioAsync(filePath, options); - return result.Value.Text; - } - - public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) - { - var audioClient = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - var options = PrepareOptions(); - var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); - return result.Value.Text; - } - - public async Task SetModelName(string model) - { - _model = model; - } - - private AudioTranscriptionOptions PrepareOptions() - { - return new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs deleted file mode 100644 index e109dfcd..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ /dev/null @@ -1,31 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio -{ - public partial class TextToSpeechProvider : ITextToSpeech - { - private readonly IServiceProvider _services; - - public string Provider => "openai"; - private string? _model; - - public TextToSpeechProvider( - IServiceProvider services) - { - _services = services; - } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } - - public void SetModelName(string model) - { - _model = model; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e94ab6ce..d21a4c1a 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -7,6 +7,24 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrEmpty(relativePath)) + { + return Enumerable.Empty(); + } + + try + { + return _cosClient.BucketClient.GetDirFiles(relativePath); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + public byte[] GetFileBytes(string fileStorageUrl) { try @@ -15,9 +33,9 @@ public partial class TencentCosService } catch (Exception ex) { - _logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}"); + return Array.Empty(); } - return Array.Empty(); } public bool SaveFileStreamToPath(string filePath, Stream stream) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 9521e8ff..eb7d775c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -112,11 +112,11 @@ public class TwilioVoiceController : TwilioController } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var fileStorage = _services.GetRequiredService(); + var data = await completion.GenerateSpeechFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; } response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 19e0ce68..e11284bd 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; using Microsoft.Extensions.Hosting; -using System; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -97,11 +96,11 @@ namespace BotSharp.Plugin.Twilio.Services async functionExecuted => { } ); - var textToSpeechService = CompletionProvider.GetTextToSpeech(sp, "openai", "tts-1"); - var fileService = sp.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateSpeechFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileService.SaveSpeechFileAsync(message.ConversationId, fileName, data); + await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c34fc791..5b2203fb 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -345,6 +345,7 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", + "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.TencentCos", "BotSharp.Plugin.PythonInterpreter" ] From 4b739a04d967b63ce02b2fd19f605b8899901b85 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 11:41:42 -0500 Subject: [PATCH 05/27] refine api --- .../Files/Utilities/FileUtility.cs | 4 +- .../Controllers/InstructModeController.cs | 68 +++++++++++-------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index 5c021159..bba60b10 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -47,11 +47,11 @@ public static class FileUtility return $"data:{contentType};base64,{base64}"; } - public static string GetFileContentType(string filePath) + public static string GetFileContentType(string fileName) { string contentType; var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) + if (!provider.TryGetContentType(fileName, out contentType)) { contentType = string.Empty; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 170cf2d2..b8aedadc 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -173,6 +173,7 @@ public class InstructModeController : ControllerBase var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; } catch (Exception ex) @@ -194,17 +195,20 @@ public class InstructModeController : ControllerBase try { - var data = FileUtility.BuildFileDataFromFile(file); - var image = new InstructFileModel + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageVariation(new Agent() { - FileName = Path.GetFileNameWithoutExtension(file.FileName), - FileExtension = Path.GetExtension(file.FileName).Substring(1), - FileData = data - }; - var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(provider, model, image); + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + return imageViewModel; } catch (Exception ex) @@ -255,16 +259,20 @@ public class InstructModeController : ControllerBase try { - var data = FileUtility.BuildFileDataFromFile(file); - var image = new InstructFileModel + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() { - FileName = Path.GetFileNameWithoutExtension(file.FileName), - FileExtension = Path.GetExtension(file.FileName).Substring(1), - FileData = data - }; - var message = await fileInstruct.EditImage(provider, model, text, image); + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), stream, file.FileName); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + return imageViewModel; } catch (Exception ex) @@ -317,25 +325,25 @@ public class InstructModeController : ControllerBase try { - var imageData = FileUtility.BuildFileDataFromFile(image); - var imageFile = new InstructFileModel - { - FileName = Path.GetFileNameWithoutExtension(image.FileName), - FileExtension = Path.GetExtension(image.FileName).Substring(1), - FileData = imageData - }; + using var imageStream = new MemoryStream(); + image.CopyTo(imageStream); + imageStream.Position = 0; - var maskData = FileUtility.BuildFileDataFromFile(mask); - var maskFile = new InstructFileModel - { - FileName = Path.GetFileNameWithoutExtension(mask.FileName), - FileExtension = Path.GetExtension(mask.FileName).Substring(1), - FileData = maskData - }; + using var maskStream = new MemoryStream(); + mask.CopyTo(maskStream); + maskStream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName); - var message = await fileInstruct.EditImage(provider, model, text, imageFile, maskFile); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageStream.Close(); + maskStream.Close(); + return imageViewModel; } catch (Exception ex) From a4be2e867184cfb23160515ab19517353a409060 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 12:06:23 -0500 Subject: [PATCH 06/27] add speech2text options --- .../Image/ImageCompletionProvider.Edit.cs | 2 +- .../ImageCompletionProvider.Generation.cs | 2 +- .../ImageCompletionProvider.Variation.cs | 2 +- .../Functions/EditImageFn.cs | 2 +- .../Functions/GenerateImageFn.cs | 2 +- .../AudioCompletionProvider.SpeechToText.cs | 56 +++++++++++++++---- .../Image/ImageCompletionProvider.Edit.cs | 2 +- .../ImageCompletionProvider.Generation.cs | 2 +- .../ImageCompletionProvider.Variation.cs | 2 +- 9 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 43272f8e..52e1a845 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -53,7 +53,7 @@ public partial class ImageCompletionProvider var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7d608fd1..f5d8c1f9 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public partial class ImageCompletionProvider var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 88f393e0..b2e4c5fe 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public partial class ImageCompletionProvider { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 7a5d8e72..dc4ddef0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -43,7 +43,7 @@ public class EditImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4c8fdf9e..9592ef80 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -50,7 +50,7 @@ public class GenerateImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index cf4837a1..b31c3035 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -17,37 +17,73 @@ public partial class AudioCompletionProvider private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) { var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + var options = new AudioTranscriptionOptions { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, Prompt = text }; return options; } - private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format) + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) { - var value = !string.IsNullOrEmpty(format) ? format : "verbose"; + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; - AudioTranscriptionFormat retFormat; + AudioTranscriptionFormat format; switch (value) { case "json": - retFormat = AudioTranscriptionFormat.Simple; + format = AudioTranscriptionFormat.Simple; break; case "srt": - retFormat = AudioTranscriptionFormat.Srt; + format = AudioTranscriptionFormat.Srt; break; case "vtt": - retFormat = AudioTranscriptionFormat.Vtt; + format = AudioTranscriptionFormat.Vtt; break; default: - retFormat = AudioTranscriptionFormat.Verbose; + format = AudioTranscriptionFormat.Verbose; break; } - return retFormat; + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return 0.0f; + } + + return temperature; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 717002fb..47df077e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -54,7 +54,7 @@ public partial class ImageCompletionProvider var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 85c15686..2f55d917 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public partial class ImageCompletionProvider var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 8a93df83..30c8d68f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public partial class ImageCompletionProvider { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions From 5cb87e8c8fa297bf43b772d0418ca67b4d8555d0 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 13:43:37 -0500 Subject: [PATCH 07/27] add text to speech api --- .../Files/Models/BotSharpFile.cs | 2 +- .../{FileInfo.cs => FileInformation.cs} | 2 +- .../Files/Models/MessageFileModel.cs | 2 +- .../MLTasks/IAudioCompletion.cs | 2 +- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 3 +- .../Controllers/InstructModeController.cs | 14 +++ .../Instructs/InstructBaseRequest.cs | 6 ++ .../Provider/NativeWhisperProvider.cs | 2 +- .../AudioCompletionProvider.SpeechToText.cs | 4 +- .../AudioCompletionProvider.TextToSpeech.cs | 89 +++++++++++++++++-- .../Controllers/TwilioVoiceController.cs | 2 +- .../Services/TwilioMessageQueueService.cs | 2 +- 12 files changed, 115 insertions(+), 15 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Files/Models/{FileInfo.cs => FileInformation.cs} (97%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 7a606b3d..bc670626 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileInfo +public class BotSharpFile : FileInformation { /// /// File data => format: "data:image/png;base64,aaaaaaaa" diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs similarity index 97% rename from src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs index 14c02f74..fe767e16 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class FileInfo +public class FileInformation { /// /// External file url diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 59ada4f2..da2ddec6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileInfo +public class MessageFileModel : FileInformation { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs index d32624c0..54ed2d7d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -7,7 +7,7 @@ public interface IAudioCompletion string Provider { get; } Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); - Task GenerateSpeechFromTextAsync(string text); + Task GenerateAudioFromTextAsync(string text); void SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 24002225..9094107d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -19,6 +19,7 @@ + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b8aedadc..bc02e846 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -469,5 +469,19 @@ public class InstructModeController : ControllerBase return viewModel; } } + + [HttpPost("/instruct/text-to-speech")] + public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1"); + var binaryData = await completion.GenerateAudioFromTextAsync(input.Text); + var stream = binaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 396fff2f..39a6439d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -66,4 +66,10 @@ public class SpeechToTextRequest : InstructBaseRequest [JsonPropertyName("file")] public InstructFileModel File { get; set; } +} + +public class TextToSpeechRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 9bf049ac..e6ac230e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -47,7 +47,7 @@ public class NativeWhisperProvider : IAudioCompletion return audioOutput.ToString(); } - public async Task GenerateSpeechFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index b31c3035..9f248086 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -77,11 +77,11 @@ public partial class AudioCompletionProvider return granularity; } - private float GetTemperature(string input) + private float? GetTemperature(string input) { if (!float.TryParse(input, out var temperature)) { - return 0.0f; + return null; } return temperature; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index a1ad2a1f..09acf389 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -4,20 +4,99 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateSpeechFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text) { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); return result.Value; } - private SpeechGenerationOptions PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() { - return new SpeechGenerationOptions - { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index e4bb9e44..571a5101 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -115,7 +115,7 @@ public class TwilioVoiceController : TwilioController { var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var fileStorage = _services.GetRequiredService(); - var data = await completion.GenerateSpeechFromTextAsync(indication); + var data = await completion.GenerateAudioFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index e11284bd..6b558aad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -98,7 +98,7 @@ namespace BotSharp.Plugin.Twilio.Services ); var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); var fileStorage = sp.GetRequiredService(); - var data = await completion.GenerateSpeechFromTextAsync(reply.Content); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); reply.SpeechFileName = fileName; From 4a870a397229a47a1f16ff04c766b63e003bf70f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 13:47:09 -0500 Subject: [PATCH 08/27] add azure audio completion --- .../AzureOpenAiPlugin.cs | 2 + .../AudioCompletionProvider.SpeechToText.cs | 90 ++++++++++++++++ .../AudioCompletionProvider.TextToSpeech.cs | 102 ++++++++++++++++++ .../Audio/AudioCompletionProvider.cs | 19 ++++ 4 files changed, 213 insertions(+) create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index dba74d27..eba22bfa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.AzureOpenAI.Providers.Audio; using BotSharp.Plugin.AzureOpenAI.Providers.Chat; using BotSharp.Plugin.AzureOpenAI.Providers.Embedding; using BotSharp.Plugin.AzureOpenAI.Providers.Image; @@ -30,5 +31,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 00000000..5a436fef --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,90 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + + var options = new AudioTranscriptionOptions + { + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + + AudioTranscriptionFormat format; + switch (value) + { + case "json": + format = AudioTranscriptionFormat.Simple; + break; + case "srt": + format = AudioTranscriptionFormat.Srt; + break; + case "vtt": + format = AudioTranscriptionFormat.Vtt; + break; + default: + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float? GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return null; + } + + return temperature; + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 00000000..4e5fc3fa --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,102 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateAudioFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + return result.Value; + } + + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed + }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 00000000..8cc75ad8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} From 8674ac9830f7af8540be94d505d3fb8f038f3601 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 15:34:41 -0500 Subject: [PATCH 09/27] refine speech storage --- .../Files/IFileStorageService.cs | 6 ++-- .../Storage/LocalFileStorageService.Audio.cs | 36 ++++++++++++------- .../LocalFileStorageService.Conversation.cs | 2 +- .../Storage/LocalFileStorageService.cs | 6 ---- .../Functions/ReadPdfFn.cs | 2 +- .../Modules/BucketClient.cs | 26 ++++++++++++++ .../Services/TencentCosService.Audio.cs | 33 ++++++++++++++--- .../TencentCosService.Conversation.cs | 2 +- .../Services/TencentCosService.cs | 2 +- .../Controllers/TwilioVoiceController.cs | 26 ++++++++++---- .../Services/TwilioMessageQueue.cs | 7 +++- .../Services/TwilioMessageQueueService.cs | 11 ++++-- .../Services/TwilioService.cs | 3 ++ .../Services/TwilioSessionManager.cs | 4 +-- .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 3 ++ 15 files changed, 125 insertions(+), 44 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index ebd7d61f..373466c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -24,7 +24,7 @@ public interface IFileStorageService /// /// /// - Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -58,7 +58,7 @@ public interface IFileStorageService #endregion #region Speech - Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); - Task RetrieveSpeechFileAsync(string conversationId, string fileName); + bool SaveSpeechFile(string conversationId, string fileName, BinaryData data); + BinaryData GetSpeechFile(string conversationId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index e90c7812..1aaba575 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -1,28 +1,38 @@ using System.IO; -namespace BotSharp.Core.Files.Services +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService { - public partial class LocalFileStorageService + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + try { var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } - var filePath = Path.Combine(dir, fileName); - if (File.Exists(filePath)) return; - using var file = File.Create(filePath); - using var input = data.ToStream(); - await input.CopyToAsync(file); - } - public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + var filePath = Path.Combine(dir, fileName); + if (File.Exists(filePath)) return false; + + using var fs = File.Create(filePath); + using var ds = data.ToStream(); + ds.CopyTo(fs); + return true; + } + catch (Exception ex) { - var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); - using var file = new FileStream(path, FileMode.Open, FileAccess.Read); - return await BinaryData.FromStreamAsync(file); + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; } } + + public BinaryData GetSpeechFile(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return BinaryData.FromStream(file); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 248806ec..c40e2d59 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 94ee2beb..750803c4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _audioTypes = new List - { - "mp3", - "wav" - }; - private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index e2b465c8..814f943e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -52,7 +52,7 @@ public class ReadPdfFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); if (screenshots.IsNullOrEmpty()) return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index 1886cfa0..fe6f0b7b 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -12,6 +12,7 @@ namespace BotSharp.Plugin.TencentCos.Modules private readonly string _fullBucketName; private readonly string _appId; private readonly string _region; + public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region) { _cosXml = cosXml; @@ -141,7 +142,32 @@ namespace BotSharp.Plugin.TencentCos.Modules var objects = info.contentsList; return objects.Where(o => o.size > 0).Select(o => o.key).ToList(); + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + public string? GetDirFile(string dir, string key) + { + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + + return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key; } catch (CosClientException clientEx) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs index 4d803628..ec1a4161 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -2,13 +2,38 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - throw new NotImplementedException(); + try + { + var file = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}/{fileName}"; + var exist = _cosClient.BucketClient.DoesObjectExist(file); + if (exist) + { + return false; + } + + return _cosClient.BucketClient.UploadBytes(file, data.ToArray()); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + public BinaryData GetSpeechFile(string conversationId, string fileName) { - throw new NotImplementedException(); + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}"; + var key = $"{dir}/{fileName}"; + var file = _cosClient.BucketClient.GetDirFile(dir, key); + + if (string.IsNullOrWhiteSpace(file)) + { + return BinaryData.Empty; + } + + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + return BinaryData.FromBytes(bytes); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 4c339f12..9badec03 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index ddb5e031..c48812f1 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -27,7 +27,7 @@ public partial class TencentCosService : IFileStorageService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; public TencentCosService( TencentCosSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 571a5101..cf49141b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -31,7 +31,11 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/welcome")] public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { - if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; @@ -46,13 +50,16 @@ public class TwilioVoiceController : TwilioController var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); + if (!string.IsNullOrWhiteSpace(text)) { messages.Add(text); await sessionManager.StageCallerMessageAsync(conversationId, seqNum, text); } + VoiceResponse response; if (messages.Count == 0 && seqNum == 0) { @@ -64,6 +71,7 @@ public class TwilioVoiceController : TwilioController { messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum - 1); } + var messageContent = string.Join("\r\n", messages); var callerMessage = new CallerMessage() { @@ -72,6 +80,7 @@ public class TwilioVoiceController : TwilioController Content = messageContent, From = request.From }; + if (!string.IsNullOrEmpty(states)) { var kvp = states.Split(':'); @@ -80,11 +89,12 @@ public class TwilioVoiceController : TwilioController callerMessage.States.Add(kvp[0], kvp[1]); } } - await messageQueue.EnqueueAsync(callerMessage); + await messageQueue.EnqueueAsync(callerMessage); int audioIndex = Random.Shared.Next(1, 5); response = twilio.ReturnInstructions($"twilio/hold-on-{audioIndex}.mp3", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1); } + return TwiML(response); } @@ -95,10 +105,12 @@ public class TwilioVoiceController : TwilioController var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + if (request.SpeechResult != null) { await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); } + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; if (reply == null) @@ -117,7 +129,7 @@ public class TwilioVoiceController : TwilioController var fileStorage = _services.GetRequiredService(); var data = await completion.GenerateAudioFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; - await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; } response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); @@ -137,17 +149,17 @@ public class TwilioVoiceController : TwilioController { response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } - } + return TwiML(response); } [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] - public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); + var fileStorage = _services.GetRequiredService(); + var data = fileStorage.GetSpeechFile(conversationId, fileName); var result = new FileContentResult(data.ToArray(), "audio/mpeg"); result.FileDownloadName = fileName; return result; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs index 455e314c..dfdc66c3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -7,6 +7,7 @@ namespace BotSharp.Plugin.Twilio.Services { private readonly Channel _queue; internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() { BoundedChannelOptions options = new(100) @@ -18,7 +19,11 @@ namespace BotSharp.Plugin.Twilio.Services public async ValueTask EnqueueAsync(CallerMessage request) { - if (request == null) throw new ArgumentNullException(nameof(request)); + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); await _queue.Writer.WriteAsync(request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 6b558aad..825bdb3b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -57,22 +57,26 @@ namespace BotSharp.Plugin.Twilio.Services { using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); var config = sp.GetRequiredService(); + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); var states = new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) }; + foreach (var kvp in message.States) { states.Add(new MessageState(kvp.Key, kvp.Value)); } conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); var result = await conv.SendMessage(config.AgentId, inputMsg, @@ -93,14 +97,15 @@ namespace BotSharp.Plugin.Twilio.Services await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); } }, - async functionExecuted => - { } + async functionExecuted => { } ); + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); var fileStorage = sp.GetRequiredService(); var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); + fileStorage.SaveSpeechFile(message.ConversationId, fileName, data); + reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 309d627c..e6f930bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -59,6 +59,7 @@ public class TwilioService }, Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}") }; + gather.Say(message); response.Append(gather); return response; @@ -80,6 +81,7 @@ public class TwilioService Timeout = timeout > 0 ? timeout : 3, ActionOnEmptyResult = actionOnEmptyResult }; + if (!string.IsNullOrEmpty(speechPath)) { gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); @@ -119,6 +121,7 @@ public class TwilioService Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), ActionOnEmptyResult = true }; + if (!string.IsNullOrEmpty(message)) { gather.Say(message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index eae0b238..117fc3bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -25,9 +25,7 @@ namespace BotSharp.Plugin.Twilio.Services { var db = _redis.GetDatabase(); var key = $"{conversationId}:Caller:{seqNum}"; - return (await db.ListRangeAsync(key)) - .Select(x => (string)x) - .ToList(); + return (await db.ListRangeAsync(key)).Select(x => (string)x).ToList(); } public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index fa971a7c..2a6d7c22 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -17,9 +17,12 @@ public class TwilioPlugin : IBotSharpPlugin var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); + services.AddScoped(); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); services.AddSingleton(); services.AddHostedService(); From f4e20ac64c6ba7628c718f8e657667d9f339be23 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 17:45:33 -0500 Subject: [PATCH 10/27] add search filter --- .../BotSharp.Abstraction/Models/KeyValue.cs | 10 +++ .../VectorStorage/Models/VectorFilter.cs | 5 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 81 ++++++++++--------- .../SemanticKernelMemoryStoreProvider.cs | 2 +- 4 files changed, 59 insertions(+), 39 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs new file mode 100644 index 00000000..f981ba6e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Models; + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index 91abcc25..85b9dec2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -4,4 +4,7 @@ public class VectorFilter : StringIdPagination { [JsonPropertyName("with_vector")] public bool WithVector { get; set; } -} + + [JsonPropertyName("search_pairs")] + public IEnumerable? SearchPairs { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 99d917d4..496a8fc2 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -51,10 +51,34 @@ public class QdrantDb : IVectorDb return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(collectionName); + // Build query filter + Filter? queryFilter = null; + if (!filter.SearchPairs.IsNullOrEmpty()) + { + var conditions = filter.SearchPairs.Select(x => new Condition + { + Field = new FieldCondition + { + Key = x.Key, + Match = new Match { Text = x.Value }, + } + }); + + queryFilter = new Filter + { + Should = + { + conditions + } + }; + } + + var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, - offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, + filter: queryFilter, vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, @@ -160,43 +184,26 @@ public class QdrantDb : IVectorDb return results; } - var points = await client.SearchAsync(collectionName, - vector, - limit: (ulong)limit, - scoreThreshold: confidence, - vectorsSelector: new WithVectorsSelector { Enable = withVector }); - - var pickFields = fields != null; - foreach (var point in points) + var payloadSelector = new WithPayloadSelector { Enable = true }; + if (fields != null) { - var data = new Dictionary(); - if (pickFields) - { - foreach (var field in fields) - { - if (point.Payload.ContainsKey(field)) - { - data[field] = point.Payload[field].StringValue; - } - else - { - data[field] = ""; - } - } - } - else - { - data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); - } - - results.Add(new VectorCollectionData - { - Id = point.Id.Uuid, - Data = data, - Score = point.Score, - Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null - }); + payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; } + + var points = await client.SearchAsync(collectionName, + vector, + limit: (ulong)limit, + scoreThreshold: confidence, + payloadSelector: payloadSelector, + vectorsSelector: withVector); + + results = points.Select(x => new VectorCollectionData + { + Id = x.Id.Uuid, + Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Score = x.Score, + Vector = x.Vectors?.Vector?.Data?.ToArray() + }).ToList(); return results; } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index a797b47f..89510f4d 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -31,7 +31,7 @@ namespace BotSharp.Plugin.SemanticKernel public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { - throw new System.NotImplementedException(); + throw new NotImplementedException(); } public Task> GetCollectionData(string collectionName, IEnumerable ids, From 3fd7f749cac45ca8517e33bd245896dc803b988b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 14:30:47 -0500 Subject: [PATCH 11/27] add create / delete vector collection --- .../Knowledges/IKnowledgeService.cs | 7 +++ .../VectorStorage/IVectorDb.cs | 3 +- .../Controllers/KnowledgeBaseController.cs | 19 +++++++ .../MemVecDb/MemoryVectorDb.cs | 10 +++- .../Services/KnowledgeService.Create.cs | 19 +++++++ .../Services/KnowledgeService.Delete.cs | 19 +++++++ .../Services/KnowledgeService.cs | 1 + .../Providers/FaissDb.cs | 7 ++- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 50 +++++++++++-------- .../SemanticKernelMemoryStoreProvider.cs | 9 +++- .../Modules/BucketClient.cs | 23 +-------- 11 files changed, 119 insertions(+), 48 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 20825791..1021ea1c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + #region Vector + Task CreateVectorCollection(string collectionName, int dimension); + Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); @@ -12,6 +15,10 @@ public interface IKnowledgeService Task DeleteVectorCollectionData(string collectionName, string id); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); + #endregion + + #region Graph Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 53b921a1..c169988b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -9,7 +9,8 @@ public interface IVectorDb Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dim); + Task CreateCollection(string collectionName, int dimension); + Task DeleteCollection(string collectionName); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, Guid id); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index deb21378..4a045e4b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -18,12 +18,25 @@ public class KnowledgeBaseController : ControllerBase _services = services; } + #region Vector [HttpGet("knowledge/vector/collections")] public async Task> GetVectorCollections() { return await _knowledgeService.GetVectorCollections(); } + [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")] + public async Task CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension) + { + return await _knowledgeService.CreateVectorCollection(collection, dimension); + } + + [HttpDelete("knowledge/vector/{collection}/delete-collection")] + public async Task GetVectorCollections([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollection(collection); + } + [HttpPost("/knowledge/vector/{collection}/search")] public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { @@ -109,7 +122,10 @@ public class KnowledgeBaseController : ControllerBase System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } + #endregion + + #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) { @@ -124,7 +140,9 @@ public class KnowledgeBaseController : ControllerBase Result = result.Result }; } + #endregion + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) { @@ -148,4 +166,5 @@ public class KnowledgeBaseController : ControllerBase GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null }; } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index aa5c0f4e..2f4e2db6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -10,10 +10,16 @@ public class MemoryVectorDb : IVectorDb public string Name => "MemoryVector"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { - _collections[collectionName] = dim; + _collections[collectionName] = dimension; _vectors[collectionName] = new List(); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + return false; } public async Task> GetCollections() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index 9b185867..1df460a0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -25,6 +25,25 @@ public partial class KnowledgeService } } + public async Task CreateVectorCollection(string collectionName, int dimension) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.CreateCollection(collectionName, dimension); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index e78990a7..b0973a99 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,6 +2,25 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { + public async Task DeleteVectorCollection(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollection(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task DeleteVectorCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 4e40fea4..aec32dc5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService : IKnowledgeService diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 902087d7..30150d4c 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -11,7 +11,12 @@ public class FaissDb : IVectorDb { public string Name => "Faiss"; - public Task CreateCollection(string collectionName, int dim) + public Task CreateCollection(string collectionName, int dimension) + { + throw new NotImplementedException(); + } + + public Task DeleteCollection(string collectionName) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 496a8fc2..927f3616 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -35,6 +35,34 @@ public class QdrantDb : IVectorDb return _client; } + public async Task CreateCollection(string collectionName, int dim) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (exist) return false; + + // Create a new collection + await client.CreateCollectionAsync(collectionName, new VectorParams() + { + Size = (ulong)dim, + Distance = Distance.Cosine + }); + + return true; + } + + public async Task DeleteCollection(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (!exist) return false; + + await client.DeleteCollectionAsync(collectionName); + return true; + } + public async Task> GetCollections() { // List all the collections @@ -117,27 +145,7 @@ public class QdrantDb : IVectorDb }); } - public async Task CreateCollection(string collectionName, int dim) - { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); - if (!exist) - { - // Create a new collection - await client.CreateCollectionAsync(collectionName, new VectorParams() - { - Size = (ulong)dim, - Distance = Distance.Cosine - }); - } - - // Get collection info - var collectionInfo = await client.GetCollectionInfoAsync(collectionName); - if (collectionInfo == null) - { - throw new Exception($"Create {collectionName} failed."); - } - } + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 89510f4d..11df098c 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -24,9 +24,16 @@ namespace BotSharp.Plugin.SemanticKernel public string Name => "SemanticKernel"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { await _memoryStore.CreateCollectionAsync(collectionName); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + await _memoryStore.DeleteCollectionAsync(collectionName); + return false; } public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index fe6f0b7b..c0c2d076 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -155,28 +155,7 @@ namespace BotSharp.Plugin.TencentCos.Modules public string? GetDirFile(string dir, string key) { - try - { - var request = new GetBucketRequest(_fullBucketName); - request.SetPrefix($"{dir.TrimEnd('/')}/"); - request.SetDelimiter("/"); - - var result = _cosXml.GetBucket(request); - - var info = result.listBucket; - - var objects = info.contentsList; - - return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key; - } - catch (CosClientException clientEx) - { - throw new Exception(clientEx.Message); - } - catch (CosServerException serverEx) - { - throw new Exception(serverEx.Message); - } + return GetDirFiles(dir).FirstOrDefault(x => x == key); } public List GetDirectories(string dir) From 09137ff8e7b30e9e5ec60961c35c217ad7df2be5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 14:57:58 -0500 Subject: [PATCH 12/27] get knowledge base text embedding dimension from llm --- .../Embedding/TextEmbeddingProvider.cs | 2 +- .../Helpers/KnowledgeSettingHelper.cs | 18 +++++++++++++++++- .../Embedding/TextEmbeddingProvider.cs | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 39119050..762e870f 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly IServiceProvider _services; protected readonly ILogger _logger; - private const int DEFAULT_DIMENSION = 3072; + private const int DEFAULT_DIMENSION = 1536; protected string _model; protected int _dimension; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index b1b280eb..3b05bdba 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Settings; + namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class KnowledgeSettingHelper @@ -12,8 +14,22 @@ public static class KnowledgeSettingHelper } var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); + var dimension = found.Dimension; + + if (found.Dimension <= 0) + { + dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model); + } + embedding.SetModelName(found.Model); - embedding.SetDimension(found.Dimension); + embedding.SetDimension(dimension); return embedding; } + + private static int GetLlmTextEmbeddingDimension(IServiceProvider services, string provider, string model) + { + var settings = services.GetRequiredService(); + var found = settings.GetSetting(provider, model); + return found?.Dimension ?? 0; + } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index bebf6853..e27d4fcb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly IServiceProvider _services; protected readonly ILogger _logger; - private const int DEFAULT_DIMENSION = 3072; + private const int DEFAULT_DIMENSION = 1536; protected string _model = "text-embedding-3-large"; protected int _dimension = DEFAULT_DIMENSION; From c3f0bdea14fae5690bba06c6bf5fcfcbbf742bb6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 15:00:28 -0500 Subject: [PATCH 13/27] add comments --- .../Conversations/Models/RoleDialogModel.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 3d8a25e9..f93ef96f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -75,6 +75,9 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? RichContent { get; set; } + /// + /// Rich content for secondary language + /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? SecondaryRichContent { get; set; } @@ -86,6 +89,9 @@ public class RoleDialogModel : ITrackableMessage public FunctionCallFromLlm Instruction { get; set; } + /// + /// Files to be used in conversation + /// public List Files { get; set; } = new List(); /// @@ -94,7 +100,14 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); + + /// + /// Knowledge confidence + /// + [JsonPropertyName("knowledge_confidence")] public float KnowledgeConfidence { get; set; } = 0.5f; + + private RoleDialogModel() { } From f74237e8bb5ee73363ccc5b9b71b343decd07516 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 11:29:21 -0500 Subject: [PATCH 14/27] minor change --- .../Controllers/TwilioVoiceController.cs | 21 ++++++++++++------- .../Services/TwilioService.cs | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 9d0368f6..912dd122 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -115,6 +115,7 @@ public class TwilioVoiceController : TwilioController var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); if (request.SpeechResult != null) { @@ -123,6 +124,7 @@ public class TwilioVoiceController : TwilioController var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; + if (reply == null) { var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); @@ -139,11 +141,11 @@ public class TwilioVoiceController : TwilioController } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(seg); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(seg); + var fileName = $"indication_{seqNum}_{segIndex}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); segIndex++; } @@ -168,7 +170,10 @@ public class TwilioVoiceController : TwilioController } else { - response = twilio.ReturnInstructions(new List { $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); + response = twilio.ReturnInstructions(new List + { + $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" + }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } } @@ -181,8 +186,10 @@ public class TwilioVoiceController : TwilioController { var fileStorage = _services.GetRequiredService(); var data = fileStorage.GetSpeechFile(conversationId, fileName); - var result = new FileContentResult(data.ToArray(), "audio/mpeg"); - result.FileDownloadName = fileName; + var result = new FileContentResult(data.ToArray(), "audio/mpeg") + { + FileDownloadName = fileName + }; return result; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 611007fa..c43782ab 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Utilities; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -81,7 +82,8 @@ public class TwilioService Timeout = timeout > 0 ? timeout : 2, ActionOnEmptyResult = actionOnEmptyResult }; - if (speechPaths != null && speechPaths.Any()) + + if (!speechPaths.IsNullOrEmpty()) { foreach (var speechPath in speechPaths) { From 6aece1d73c7f781a4e1b69a29cd5989b93dcde04 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 13:09:28 -0500 Subject: [PATCH 15/27] clean comments --- .../BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 2f4e2db6..3a695e93 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -47,7 +47,6 @@ public class MemoryVectorDb : IVectorDb } var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() .Reverse() From 4102f6fa1f83d7c9dc40a6885f8ea2f5fb983eb6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 16:12:20 -0500 Subject: [PATCH 16/27] refine planner --- .../Routing/RoutingService.GetPlanner.cs | 1 - .../templates/database_knowledge.liquid | 8 +- .../Controllers/LlmProviderController.cs | 2 +- .../Embedding/TextEmbeddingProvider.cs | 2 +- .../Functions/PrimaryStagePlanFn.cs | 33 +++++--- .../Functions/SecondaryStagePlanFn.cs | 47 +++++------ .../Functions/SummaryPlanFn.cs | 14 ++-- .../templates/two_stage.1st.plan.liquid | 6 +- .../templates/two_stage.2nd.plan.liquid | 2 +- .../Functions/AddDatabaseKnowledge.cs | 79 +++++++++++-------- .../Functions/GetTableDefinitionFn.cs | 32 +++----- 11 files changed, 123 insertions(+), 103 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs index 72549e51..0fcaa5a1 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Core.Routing.Planning; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid index 0332af76..4c9f51c3 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid @@ -1,8 +1,10 @@ -You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name. -the summarized question/answer should: +You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname, data type and comments, generate the related knowledge for DBA and BA. When users ask the question, they don't know the table name. + +The summarized question/answer should: 1. help user to identify the location of tables to find further information 2. identify the table structure and data relationship based on the task description -3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer +3. summarize all the tables to table relationship information based on the FOREIGN KEY, and include both tables in the answer + Go through all the columns and generate multiple question & answer pairs. The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n. diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 5571d9fa..02282b85 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -25,6 +25,6 @@ public class LlmProviderController : ControllerBase public IEnumerable GetLlmProviderModels([FromRoute] string provider) { var list = _llmProvider.GetProviderModels(provider); - return list.Where(x => !x.ImageGeneration); + return list.Where(x => x.Type == LlmModelType.Chat); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index e27d4fcb..cb9e7d15 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -9,7 +9,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly ILogger _logger; private const int DEFAULT_DIMENSION = 1536; - protected string _model = "text-embedding-3-large"; + protected string _model = "text-embedding-3-small"; protected int _dimension = DEFAULT_DIMENSION; public virtual string Provider => "openai"; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index a12cafed..2842397b 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -13,9 +13,9 @@ namespace BotSharp.Plugin.Planner.Functions; public class PrimaryStagePlanFn : IFunctionCallback { public string Name => "plan_primary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; - private object aiAssistant; + private readonly ILogger _logger; public PrimaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -25,31 +25,33 @@ public class PrimaryStagePlanFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - //debug + // Debug var state = _services.GetRequiredService(); state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - //get knowledge from vectordb + // Get knowledge from vectordb var fn = _services.GetRequiredService(); var msg = new ExtractedKnowledge { Question = task.Question, }; + var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements) { FunctionArgs = JsonSerializer.Serialize(msg), KnowledgeConfidence = 0.1f, - Content = "" + Content = string.Empty }; + await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); message.Content = retrievalMessage.Content; var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - //send knowledge to AI to refine and summarize the primary planning + // Send knowledge to AI to refine and summarize the primary planning var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message); var plannerAgent = new Agent { @@ -59,7 +61,7 @@ public class PrimaryStagePlanFn : IFunctionCallback TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; /*await fn.InvokeFunction("plan_secondary_stage", message); @@ -96,12 +98,14 @@ public class PrimaryStagePlanFn : IFunctionCallback message.StopCompletion = true;*/ return true; } + private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.1st.plan").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.1st.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -115,13 +119,15 @@ public class PrimaryStagePlanFn : IFunctionCallback { "response_format", responseFormat } }); } + private async Task GetPlanSummaryPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "planner_prompt.two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -136,10 +142,12 @@ public class PrimaryStagePlanFn : IFunctionCallback { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); + //add "test" to wholeDialogs' last element if(plannerAgent.Name == "planner_summary") { @@ -147,6 +155,7 @@ public class PrimaryStagePlanFn : IFunctionCallback wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { //add "test" to wholeDialogs' last element in a new paragraph diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index d6e02299..2723d070 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,13 +1,10 @@ -using Azure; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions; using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Templating; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using NetTopologySuite.Index.HPRtree; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -16,8 +13,9 @@ namespace BotSharp.Plugin.Planner.Functions; public class SecondaryStagePlanFn : IFunctionCallback { public string Name => "plan_secondary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; public SecondaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -28,45 +26,49 @@ public class SecondaryStagePlanFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); - var msg_secondary = RoleDialogModel.From(message); - var task_primary = JsonSerializer.Deserialize(message.FunctionArgs); - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask - { - TaskDescription = task_primary.Requirements - }); - var task_secondary = JsonSerializer.Deserialize(msg_secondary.FunctionArgs); - var items = msg_secondary.Content.JsonArrayContent(); - msg_secondary.KnowledgeConfidence = 0.5f; + var msgSecondary = RoleDialogModel.From(message); + var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); + + msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask + { + TaskDescription = taskPrimary.Requirements + }); + + var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); + var items = msgSecondary.Content.JsonArrayContent(); + + msgSecondary.KnowledgeConfidence = 0.5f; foreach (var item in items) { if (item.NeedAdditionalInformation) { - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge + msgSecondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge { Question = item.Task }); - await fn.InvokeFunction("knowledge_retrieval", msg_secondary); - message.Content += msg_secondary.Content; + await fn.InvokeFunction("knowledge_retrieval", msgSecondary); + message.Content += msgSecondary.Content; } } + // load agent var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var secondPlanningPrompt = await GetSecondStagePlanPrompt(task_secondary, message); + var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary, message); _logger.LogInformation(secondPlanningPrompt); var plannerAgent = new Agent { - Id = "", + Id = string.Empty, Name = "test", Instruction = secondPlanningPrompt, TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; _logger.LogInformation(response.Content); return true; @@ -74,9 +76,10 @@ public class SecondaryStagePlanFn : IFunctionCallback private async Task GetSecondStagePlanPrompt(SecondaryBreakdownTask task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var planner = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = planner.Templates.First(x => x.Name == "two_stage.2nd.plan").Content; + + var planner = await agentService.GetAgent(message.CurrentAgentId); + var template = planner.Templates.FirstOrDefault(x => x.Name == "two_stage.2nd.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new SecondStagePlan { Tool = "tool name if task solution provided", @@ -91,7 +94,7 @@ public class SecondaryStagePlanFn : IFunctionCallback { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 098edd53..5fd9fe94 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -32,6 +32,7 @@ public class SummaryPlanFn : IFunctionCallback // summarize and generate query var summaryPlanningPrompt = await GetPlanSummaryPrompt(task, message); _logger.LogInformation(summaryPlanningPrompt); + var plannerAgent = new Agent { Id = BuiltInAgentId.Planner, @@ -39,7 +40,7 @@ public class SummaryPlanFn : IFunctionCallback Instruction = summaryPlanningPrompt, TemplateDict = new Dictionary() }; - var response_summary = await GetAIResponse(plannerAgent); + var response_summary = await GetAiResponse(plannerAgent); message.Content = response_summary.Content; message.StopCompletion = true; @@ -51,9 +52,10 @@ public class SummaryPlanFn : IFunctionCallback { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -68,17 +70,19 @@ public class SummaryPlanFn : IFunctionCallback { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); + //add "test" to wholeDialogs' last element - if(plannerAgent.Name == "planner_summary") + if (plannerAgent.Name == "planner_summary") { //add "test" to wholeDialogs' last element in a new paragraph wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { //add "test" to wholeDialogs' last element in a new paragraph diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 0a1f6292..fb13b0c1 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -3,7 +3,7 @@ You are a Task Planner. you will breakdown user business requirements into excut Thinking process: 1. Reference to "Task Knowledge" if there is relevant knowledge; 2. Breakdown task into subtasks. - - The subtask should contains all needed parameters for subsequent steps. + - The subtask should contain all needed parameters for subsequent steps. - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true. - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. @@ -22,9 +22,7 @@ Task Knowledge: {{ k }} {% endfor %} {%- endif %} - ===== Task description: -{{ task_description }} - +{{ task_description }} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index ccec8033..d5340e2e 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -1,5 +1,5 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown task into multiple steps. -* the step should contains all needed parameters. +* The step should contains all needed parameters. * The parameters can be extracted from the original task. * You need to list all the steps in detail. Finding relationships should also be a step. * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly. diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs index 5c2bfa45..5a50e355 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs @@ -3,58 +3,69 @@ using BotSharp.Core.Infrastructures; using MySqlConnector; using static Dapper.SqlMapper; using BotSharp.Abstraction.Agents.Enums; - +using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; + public class AddDatabaseKnowledgeFn : IFunctionCallback { public string Name => "add_database_knowledge"; - private readonly IServiceProvider _services; - private object aiAssistant; - public AddDatabaseKnowledgeFn(IServiceProvider services) + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public AddDatabaseKnowledgeFn( + IServiceProvider services, + ILogger logger) { _services = services; + _logger = logger; } + public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); var sqlDriver = _services.GetRequiredService(); var fn = _services.GetRequiredService(); var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); - List allTables = new List(); + + var allTables = new HashSet(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + var sql = $"select table_name from information_schema.tables;"; - var result = connection.Query(sql: sql,dictionary); - foreach (var item in result) + var results = connection.Query(sql, new Dictionary()); + + foreach (var item in results) { + if (item == null) continue; + allTables.Add(item.TABLE_NAME); } - message.Data = allTables.Distinct().ToList(); + message.Data = allTables.ToList(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var note = ""; - foreach (var item in allTables) + var errorNote = string.Empty; + + foreach (var table in allTables) { - message.Data = new List { item }; + message.Data = new List { table }; await fn.InvokeFunction("get_table_definition", message); - var PlanningPrompt = await GetPrompt(message); + var planningPrompt = await GetPrompt(message); var plannerAgent = new Agent { - Id = "", - Name = "database_knowledge", - Instruction = PlanningPrompt, - TemplateDict = new Dictionary(), + Id = string.Empty, + Name = "Database Knowledge", + Instruction = planningPrompt, LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + try { - var knowledge = response.Content.JsonArrayContent(); - foreach (var k in knowledge) + var response = await GetAiResponse(plannerAgent); + var knowledges = response.Content.JsonArrayContent(); + foreach (var k in knowledges) { try { @@ -64,42 +75,44 @@ public class AddDatabaseKnowledgeFn : IFunctionCallback Answer = k.Answer }); await fn.InvokeFunction("memorize_knowledge", message); - message.SecondaryContent += $"Table: {item}, Question:{k.Question}, {message.Content} \r\n"; + message.SecondaryContent += $"Table: {table}, Question: {k.Question}, {message.Content}\r\n"; } catch (Exception e) { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; + var note = $"Error processing table {table}: {e.Message}\r\n{e.InnerException}"; + errorNote += note; + _logger.LogWarning(note); } } } catch (Exception e) { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; + errorNote += $"Error processing table {table}: {e.Message}\r\n{e.InnerException}\r\n"; + _logger.LogWarning(errorNote); } } return true; } - private async Task GetAIResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var completion = CompletionProvider.GetChatCompletion(_services, + + var completion = CompletionProvider.GetChatCompletion(_services, provider: plannerAgent.LlmConfig.Provider, model: plannerAgent.LlmConfig.Model); return await completion.GetChatCompletions(plannerAgent, wholeDialogs); } + private async Task GetPrompt(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "database_knowledge").Content; - var responseFormat = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = "question", - Answer = "answer" - }); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; return render.Render(template, new Dictionary { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 74ae8f72..47f64a1f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.SqlDriver.Models; -using Microsoft.EntityFrameworkCore.Metadata.Internal; using MySqlConnector; using static Dapper.SqlMapper; @@ -17,37 +15,31 @@ public class GetTableDefinitionFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - // get agent service var agentService = _services.GetRequiredService(); - - // var args = JsonSerializer.Deserialize(message.FunctionArgs); var sqlDriver = _services.GetRequiredService(); - - //get table DDL from database var settings = _services.GetRequiredService(); + + // Get table DDL from database using var connection = new MySqlConnection(settings.MySqlConnectionString); var dictionary = new Dictionary(); + var tableDdls = new List(); - var table_ddl = ""; foreach (var p in (List)message.Data) { - dictionary["@" + "table_name"] = p; var escapedTableName = MySqlHelper.EscapeString(p); + dictionary["@" + "table_name"] = p; dictionary["table_name"] = escapedTableName; - // can you replace this with a parameterized query? - var sql = $"select * from information_schema.tables where table_name ='{dictionary["table_name"]}'"; + var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'"; var result = connection.QueryFirstOrDefault(sql: sql, dictionary); - if (result != null) - { - sql = $"SHOW CREATE TABLE `{dictionary["table_name"]}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - table_ddl += "\r\n" + result; - } - - } - message.Content = table_ddl; + if (result == null) continue; + sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + result = connection.QueryFirstOrDefault(sql: sql, dictionary); + tableDdls.Add(result); + } + + message.Content = string.Join("\r\n", tableDdls); return true; } } From a528a3f6b9242c4b7223d42844a32e857721e749 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 16:59:37 -0500 Subject: [PATCH 17/27] upgrade apphost --- src/BotSharp.AppHost/BotSharp.AppHost.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 6e8a2ae3..2a2705db 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -10,7 +10,7 @@ - + From f38fdf7497e441034c72006f51f68feced7f239b Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Fri, 30 Aug 2024 19:31:18 -0500 Subject: [PATCH 18/27] update settings --- .../Files/Converters/IPdf2ImageConverter.cs | 2 +- .../Files/FileCoreSettings.cs | 9 +++++++-- .../BotSharp.Abstraction/Graph/IGraphDb.cs | 2 +- .../Knowledges/IPdf2TextConverter.cs | 2 +- .../Knowledges/Settings/KnowledgeBaseSettings.cs | 12 ++++++++---- .../VectorStorage/IVectorDb.cs | 2 +- .../Services/Instruct/FileInstructService.Pdf.cs | 2 +- .../LocalFileStorageService.Conversation.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../Helpers/KnowledgeSettingHelper.cs | 2 -- .../MemVecDb/MemoryVectorDb.cs | 2 +- .../Services/KnowledgeService.cs | 4 ++-- .../Services/PigPdf2TextConverter.cs | 2 +- .../BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- .../Providers/Pdf2TextConverter.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 2 +- .../SemanticKernelMemoryStoreProvider.cs | 2 +- .../Services/TencentCosService.Conversation.cs | 2 +- src/WebStarter/appsettings.json | 16 ++++++++++++---- 22 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 87df6137..24b28402 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { - public string Name { get; } + public string Provider { get; } /// /// Convert pdf pages to images, and return a list of image file paths diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index 10ccd1a0..dc4ac40b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public string Pdf2TextConverter { get; set; } - public string Pdf2ImageConverter { get; set; } + public SettingBase Pdf2TextConverter { get; set; } + public SettingBase Pdf2ImageConverter { get; set; } } + +public class SettingBase +{ + public string Provider { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs index 64c5388b..b15503e2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph; public interface IGraphDb { - public string Name { get; } + public string Provider { get; } Task Search(string query, GraphSearchOptions options); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index b8f2d47b..d6edad54 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - public string Name { get; } + public string Provider { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 963a2b3b..217dafb9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { - public string VectorDb { get; set; } - public string GraphDb { get; set; } + public SettingBase VectorDb { get; set; } + public SettingBase GraphDb { get; set; } public DefaultKnowledgeBaseSetting Default { get; set; } public List Collections { get; set; } = new(); @@ -23,9 +23,13 @@ public class VectorCollectionSetting public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; } } -public class KnowledgeTextEmbeddingSetting +public class KnowledgeTextEmbeddingSetting : SettingBase { - public string Provider { get; set; } public string Model { get; set; } public int Dimension { get; set; } +} + +public class SettingBase +{ + public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index c169988b..cedacb46 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - string Name { get; } + string Provider { get; } Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index e7228b13..29dfdaf9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -98,7 +98,7 @@ public partial class FileInstructService { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index c40e2d59..5638179c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -276,7 +276,7 @@ public partial class LocalFileStorageService private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 4a045e4b..260dc166 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -104,7 +104,7 @@ public class KnowledgeBaseController : ControllerBase public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); var filePath = Path.GetTempFileName(); using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 55a11ec9..9aaf68eb 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public class GraphDb : IGraphDb _settings = settings; } - public string Name => "Remote"; + public string Provider => "Remote"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 7a7db8ae..d6d719d7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -21,7 +21,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }); if (!knowledges.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f77954d8..ca545c65 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -25,7 +25,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback args.Question }); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); await vectorDb.CreateCollection(collectionName, vector[0].Length); var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index 3b05bdba..48033c91 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Settings; - namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class KnowledgeSettingHelper diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 3a695e93..17a2c757 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -8,7 +8,7 @@ public class MemoryVectorDb : IVectorDb private readonly Dictionary> _vectors = new Dictionary>(); - public string Name => "MemoryVector"; + public string Provider => "MemoryVector"; public async Task CreateCollection(string collectionName, int dimension) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index aec32dc5..bf597395 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -22,13 +22,13 @@ public partial class KnowledgeService : IKnowledgeService private IVectorDb GetVectorDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); return db; } private IGraphDb GetGraphDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.GraphDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.GraphDb.Provider); return db; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 09401f20..c334505e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { - public string Name => "Pig"; + public string Provider => "Pig"; public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 30150d4c..58fb1851 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,7 +9,7 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { - public string Name => "Faiss"; + public string Provider => "Faiss"; public Task CreateCollection(string collectionName, int dimension) { diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 81c53f83..37ea35c4 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -29,7 +29,7 @@ public class Pdf2TextConverter : IPdf2TextConverter _paddleSharpSettings = paddleSharpSettings; } - public string Name => "Paddle"; + public string Provider => "Paddle"; public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 927f3616..be89fec0 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -19,7 +19,7 @@ public class QdrantDb : IVectorDb _services = services; } - public string Name => "Qdrant"; + public string Provider => "Qdrant"; private QdrantClient GetClient() { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 11df098c..4912e127 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -22,7 +22,7 @@ namespace BotSharp.Plugin.SemanticKernel } - public string Name => "SemanticKernel"; + public string Provider => "SemanticKernel"; public async Task CreateCollection(string collectionName, int dimension) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 9badec03..ff8dfa17 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -254,7 +254,7 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 5b2203fb..f470fe25 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -233,8 +233,12 @@ "FileCore": { "Storage": "LocalFileStorage", - "Pdf2TextConverter": "", - "Pdf2ImageConverter": "" + "Pdf2TextConverter": { + "Provider": "" + }, + "Pdf2ImageConverter": { + "Provider": "" + } }, "TencentCos": { @@ -262,8 +266,12 @@ }, "KnowledgeBase": { - "VectorDb": "Qdrant", - "GraphDb": "Default", + "VectorDb": { + "Provider": "Qdrant" + }, + "GraphDb": { + "Provider": "Remote" + }, "Default": { "CollectionName": "BotSharp", "TextEmbedding": { From e660957c469fdd6c8af2cfcc2c861f7867a6a1a1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 20:10:00 -0500 Subject: [PATCH 19/27] update settings --- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs | 1 + src/WebStarter/appsettings.json | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9aaf68eb..0465dc4e 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -44,7 +44,7 @@ public class GraphDb : IGraphDb return new GraphSearchData(); } - var url = $"{_settings.BaseUrl}/query"; + var url = $"{_settings.BaseUrl}{_settings.SearchPath}"; var request = new GraphQueryRequest { Query = query, diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs index 89351398..34e0b5fd 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs @@ -3,4 +3,5 @@ namespace BotSharp.Plugin.Graph; public class GraphDbSettings { public string BaseUrl { get; set; } + public string SearchPath { get; set; } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index f470fe25..4fcc745c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -254,7 +254,8 @@ }, "Graph": { - "BaseUrl": "" + "BaseUrl": "", + "SearchPath": "" }, "WeChat": { From 495ac9949c0494b45f7342adfb795f11cc1c2938 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sat, 31 Aug 2024 00:57:49 -0500 Subject: [PATCH 20/27] relocate knowledge service --- .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Helpers/KnowledgeSettingHelper.cs | 5 ++++- .../Knowledges/KnowledgeCorePlugin.cs | 19 +++++++++++++++++++ .../Services/KnowledgeService.Create.cs | 9 +++++++-- .../Services/KnowledgeService.Delete.cs | 2 +- .../Services/KnowledgeService.Get.cs | 4 +++- .../Services/KnowledgeService.Update.cs | 4 +++- .../Knowledges}/Services/KnowledgeService.cs | 10 ++++++---- .../Functions/KnowledgeRetrievalFn.cs | 2 ++ .../Functions/MemorizeKnowledgeFn.cs | 2 ++ .../KnowledgeBasePlugin.cs | 1 - 11 files changed, 48 insertions(+), 12 deletions(-) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Helpers/KnowledgeSettingHelper.cs (89%) create mode 100644 src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Create.cs (88%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Delete.cs (95%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Get.cs (96%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Update.cs (91%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.cs (80%) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ce4c82d2..ddd39d0c 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs index 48033c91..a278d7bf 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs @@ -1,4 +1,7 @@ -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Knowledges.Helpers; public static class KnowledgeSettingHelper { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs new file mode 100644 index 00000000..6118647b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs @@ -0,0 +1,19 @@ +using BotSharp.Core.Knowledges.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Knowledges; + +public class KnowledgeCorePlugin : IBotSharpPlugin +{ + public string Id => "a5ebc8f9-d089-44d0-bf00-4eac97a050bc"; + + public string Name => "Knowledge Core"; + + public string Description => "Provides core knowledge services."; + + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 1df460a0..2927d952 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,11 +1,16 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + + var textChopper = _services.GetRequiredService(); + var lines = textChopper.Chop(knowledge.Content, new ChunkOption { Size = 1024, Conjunction = 32, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs index b0973a99..404b4e2c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs index ee36780e..69093659 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs index 0fa1f5cd..b9652f61 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs similarity index 80% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index bf597395..2c0853d0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,22 +1,24 @@ +using BotSharp.Abstraction.Graph; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.VectorStorage; +using BotSharp.Core.Knowledges.Helpers; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly ITextChopper _textChopper; private readonly ILogger _logger; public KnowledgeService( IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper, ILogger logger) { _services = services; _settings = settings; - _textChopper = textChopper; _logger = logger; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index d6d719d7..400b1792 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Core.Knowledges.Helpers; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index ca545c65..08b55d1c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Core.Knowledges.Helpers; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 4350e33c..dd864a73 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -21,7 +21,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin }); services.AddScoped(); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); From 48be12a92085b40c4b5253c99ccab05a6db98301 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sat, 31 Aug 2024 01:05:53 -0500 Subject: [PATCH 21/27] relocate text chopper --- .../BotSharp.Abstraction/Knowledges/ITextChopper.cs | 11 ----------- .../Knowledges/Helpers}/TextChopperService.cs | 11 ++++++----- .../Knowledges/Services/KnowledgeService.Create.cs | 5 ++--- .../KnowledgeBasePlugin.cs | 1 - 4 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs rename src/{Plugins/BotSharp.Plugin.KnowledgeBase/Services => Infrastructure/BotSharp.Core/Knowledges/Helpers}/TextChopperService.cs (78%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs deleted file mode 100644 index 7d06a8a4..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs +++ /dev/null @@ -1,11 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; - -namespace BotSharp.Abstraction.Knowledges; - -/// -/// Chop large content into chunks -/// -public interface ITextChopper -{ - List Chop(string content, ChunkOption option); -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs similarity index 78% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs index 88c77641..68b58539 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs @@ -1,17 +1,18 @@ +using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Helpers; -public class TextChopperService : ITextChopper +public static class TextChopper { - public List Chop(string content, ChunkOption option) + public static List Chop(string content, ChunkOption option) { content = Regex.Replace(content, @"\.{2,}", " "); content = Regex.Replace(content, @"_{2,}", " "); return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option); } - private List ChopByWord(string content, ChunkOption option) + private static List ChopByWord(string content, ChunkOption option) { var chunks = new List(); @@ -34,7 +35,7 @@ public class TextChopperService : ITextChopper return chunks; } - private List ChopByChar(string content, ChunkOption option) + private static List ChopByChar(string content, ChunkOption option) { var chunks = new List(); var currentPos = 0; diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 2927d952..d8b82fb5 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Core.Knowledges.Helpers; namespace BotSharp.Core.Knowledges.Services; @@ -8,9 +9,7 @@ public partial class KnowledgeService public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; - - var textChopper = _services.GetRequiredService(); - var lines = textChopper.Chop(knowledge.Content, new ChunkOption + var lines = TextChopper.Chop(knowledge.Content, new ChunkOption { Size = 1024, Conjunction = 32, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index dd864a73..9a955a5b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -20,7 +20,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin return settingService.Bind("KnowledgeBase"); }); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); From 2ea8ddfe194cf77fb27b0a5bbedfbfb4a334524a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 1 Sep 2024 00:15:13 -0500 Subject: [PATCH 22/27] relocate --- .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Knowledges/KnowledgeCorePlugin.cs | 19 ------------------- .../PigPdf2TextConverter.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 -- .../Functions/MemorizeKnowledgeFn.cs | 2 -- .../Helpers/KnowledgeSettingHelper.cs | 5 +---- .../Helpers/TextChopper.cs} | 3 +-- .../KnowledgeBasePlugin.cs | 2 ++ .../Services/KnowledgeService.Create.cs | 6 +----- .../Services/KnowledgeService.Delete.cs | 2 +- .../Services/KnowledgeService.Get.cs | 4 +--- .../Services/KnowledgeService.Update.cs | 4 +--- .../Services/KnowledgeService.cs | 8 +------- 13 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/{Services => Converters}/PigPdf2TextConverter.cs (95%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Helpers/KnowledgeSettingHelper.cs (89%) rename src/{Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs => Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs} (94%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Create.cs (92%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Delete.cs (95%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Get.cs (96%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Update.cs (91%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.cs (80%) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ddd39d0c..ce4c82d2 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs deleted file mode 100644 index 6118647b..00000000 --- a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs +++ /dev/null @@ -1,19 +0,0 @@ -using BotSharp.Core.Knowledges.Services; -using Microsoft.Extensions.Configuration; - -namespace BotSharp.Core.Knowledges; - -public class KnowledgeCorePlugin : IBotSharpPlugin -{ - public string Id => "a5ebc8f9-d089-44d0-bf00-4eac97a050bc"; - - public string Name => "Knowledge Core"; - - public string Description => "Provides core knowledge services."; - - - public void RegisterDI(IServiceCollection services, IConfiguration config) - { - services.AddScoped(); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs index c334505e..0e39bef0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs @@ -1,7 +1,7 @@ using UglyToad.PdfPig; using UglyToad.PdfPig.Content; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Plugin.KnowledgeBase.Converters; public class PigPdf2TextConverter : IPdf2TextConverter { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 400b1792..d6d719d7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Knowledges.Helpers; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 08b55d1c..ca545c65 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Knowledges.Helpers; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs similarity index 89% rename from src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index a278d7bf..48033c91 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,7 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Settings; -using BotSharp.Abstraction.MLTasks; - -namespace BotSharp.Core.Knowledges.Helpers; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class KnowledgeSettingHelper { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs similarity index 94% rename from src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs index 68b58539..9e9b5f9d 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs @@ -1,7 +1,6 @@ -using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Core.Knowledges.Helpers; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class TextChopper { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 9a955a5b..75d3284b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.KnowledgeBase.Converters; using BotSharp.Plugin.KnowledgeBase.Hooks; using Microsoft.Extensions.Configuration; @@ -20,6 +21,7 @@ public class KnowledgeBasePlugin : IBotSharpPlugin return settingService.Bind("KnowledgeBase"); }); + services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs similarity index 92% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index d8b82fb5..2c4004fc 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -1,8 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Core.Knowledges.Helpers; - -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs similarity index 95% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index 404b4e2c..b0973a99 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 69093659..ee36780e 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs index b9652f61..0fa1f5cd 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.VectorStorage.Models; - -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs similarity index 80% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 2c0853d0..c668e07e 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Graph; -using BotSharp.Abstraction.Knowledges.Settings; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.VectorStorage; -using BotSharp.Core.Knowledges.Helpers; - -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService : IKnowledgeService { From 638a9d6a387b260bc9218304d16f652a088c0eec Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 2 Sep 2024 18:54:40 -0500 Subject: [PATCH 23/27] refine image options --- .../Providers/Image/ImageCompletionProvider.cs | 12 ------------ .../Providers/Image/ImageCompletionProvider.cs | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 7d064154..967f3e08 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ public partial class ImageCompletionProvider : IImageCompletion case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 185cf17c..0e45e789 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ public partial class ImageCompletionProvider : IImageCompletion case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ public partial class ImageCompletionProvider : IImageCompletion GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; From 949a2cce9f0cc7c4243ce0a7a20bd9ba6bd360f3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 3 Sep 2024 12:52:33 -0500 Subject: [PATCH 24/27] refine knowledge searching --- .../Conversations/Models/RoleDialogModel.cs | 1 - .../Helpers/VectorStorageHelper.cs | 29 ++++++++++++++++ .../Models/VectorSearchResult.cs | 2 ++ .../Functions/KnowledgeRetrievalFn.cs | 17 ++++++---- .../Functions/MemorizeKnowledgeFn.cs | 23 +++++-------- .../Functions/PrimaryStagePlanFn.cs | 34 ++++++------------- .../Functions/SecondaryStagePlanFn.cs | 26 +++++--------- .../Functions/SummaryPlanFn.cs | 6 ---- .../TwoStaging/TwoStageTaskPlanner.cs | 7 ---- src/Plugins/BotSharp.Plugin.Planner/Using.cs | 18 ++++++++-- 10 files changed, 86 insertions(+), 77 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index f93ef96f..84f194c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Conversations.Models; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs new file mode 100644 index 00000000..f1f0ebde --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.VectorStorage.Helpers; + +public static class VectorStorageHelper +{ + public static string ToQuestionAnswer(this VectorSearchResult data) + { + if (data?.Data == null) return string.Empty; + + return $"Question: {data.Data[KnowledgePayloadName.Text]}\r\nAnswer: {data.Data[KnowledgePayloadName.Answer]}"; + } + + public static string ToPayloadPair(this VectorSearchResult data, IList payloads) + { + if (data?.Data == null || payloads.IsNullOrEmpty()) return string.Empty; + + var results = data.Data.Where(x => payloads.Contains(x.Key)) + .OrderBy(x => payloads.IndexOf(x.Key)) + .Select(x => + { + return $"{x.Key}: {x.Value}"; + }) + .ToList(); + + return string.Join("\r\n", results.Where(x => !string.IsNullOrWhiteSpace(x))); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs index ce39edbf..f2c582ce 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorSearchResult : VectorCollectionData diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index d6d719d7..a9337419 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,9 +1,13 @@ +using BotSharp.Abstraction.VectorStorage.Helpers; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { public string Name => "knowledge_retrieval"; + public string Indication => "searching my brain"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,15 +22,16 @@ public class KnowledgeRetrievalFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); - var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }); + var knowledgeService = _services.GetRequiredService(); + var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions + { + Fields = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }, + Confidence = 0.2f + }); if (!knowledges.IsNullOrEmpty()) { - var answers = knowledges.Select(x => $"Question: {x.Data[KnowledgePayloadName.Text]}\r\nAnswer: {x.Data[KnowledgePayloadName.Answer]}").ToList(); + var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList(); message.Content = string.Join("\r\n\r\n=====\r\n", answers); } else diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index ca545c65..62388f19 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -4,6 +4,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { public string Name => "memorize_knowledge"; + public string Indication => "remembering knowledge"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,23 +20,16 @@ public class MemorizeKnowledgeFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorsAsync(new List + var knowledgeService = _services.GetRequiredService(); + var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { - args.Question + Text = args.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, args.Answer } + } }); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); - await vectorDb.CreateCollection(collectionName, vector[0].Length); - - var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], - args.Question, - new Dictionary - { - { KnowledgePayloadName.Answer, args.Answer } - }); - message.Content = result ? "Saved to my brain" : "I forgot it"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 2842397b..01d2bfdf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -1,12 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; -using BotSharp.Abstraction.Knowledges.Models; namespace BotSharp.Plugin.Planner.Functions; @@ -27,26 +19,20 @@ public class PrimaryStagePlanFn : IFunctionCallback { // Debug var state = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var fn = _services.GetRequiredService(); + state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - // Get knowledge from vectordb - var fn = _services.GetRequiredService(); - - var msg = new ExtractedKnowledge + // Get knowledge from vectordb + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; ; + var knowledges = await knowledgeService.SearchVectorKnowledge(task.Question, collectionName, new VectorSearchOptions { - Question = task.Question, - }; - - var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements) - { - FunctionArgs = JsonSerializer.Serialize(msg), - KnowledgeConfidence = 0.1f, - Content = string.Empty - }; - - await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); - message.Content = retrievalMessage.Content; + Confidence = 0.1f + }); + message.Content = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index 2723d070..211eea53 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,12 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; @@ -26,6 +18,9 @@ public class SecondaryStagePlanFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; var msgSecondary = RoleDialogModel.From(message); var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); @@ -38,18 +33,15 @@ public class SecondaryStagePlanFn : IFunctionCallback var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); var items = msgSecondary.Content.JsonArrayContent(); - msgSecondary.KnowledgeConfidence = 0.5f; foreach (var item in items) { - if (item.NeedAdditionalInformation) + if (!item.NeedAdditionalInformation) continue; + + var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions { - msgSecondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = item.Task - }); - await fn.InvokeFunction("knowledge_retrieval", msgSecondary); - message.Content += msgSecondary.Content; - } + Confidence = 0.5f + }); + message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); } // load agent diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 5fd9fe94..0f9da4aa 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index be627c75..c6f819e5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -1,13 +1,6 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Planning; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Core.Routing.Planning; -using Microsoft.Extensions.Logging; -using System.Threading.Tasks; namespace BotSharp.Plugin.Planner.TwoStaging; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Using.cs b/src/Plugins/BotSharp.Plugin.Planner/Using.cs index 20880e81..fa59ee23 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Using.cs @@ -3,12 +3,13 @@ global using System.Text.Json; global using System.Text.Json.Serialization; global using System.Collections.Generic; global using System.Linq; +global using System.Threading.Tasks; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; global using BotSharp.Abstraction.Plugins; -global using BotSharp.Abstraction.Planning; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; @@ -18,5 +19,18 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Templating; + +global using BotSharp.Abstraction.Knowledges; +global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.VectorStorage.Models; +global using BotSharp.Abstraction.VectorStorage.Helpers; + global using BotSharp.Plugin.Planner.Hooks; -global using BotSharp.Plugin.Planner.Enums; \ No newline at end of file +global using BotSharp.Plugin.Planner.Enums; + +global using BotSharp.Core.Infrastructures; \ No newline at end of file From e21a44ec3d9b6fcd5105634bc07e982dfd8a9e72 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 3 Sep 2024 15:41:12 -0500 Subject: [PATCH 25/27] refine db knowledge --- .../BotSharp.Logger/Hooks/VerboseLogHook.cs | 11 +- .../Controllers/KnowledgeBaseController.cs | 1 + .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../BotSharp.Plugin.SqlDriver.csproj | 3 +- .../Controllers/SqlDriverController.cs | 24 +++ .../Functions/GetTableDefinitionFn.cs | 31 ++-- .../Models/RequestBase.cs | 19 +++ .../Services/DbKnowledgeService.cs | 140 ++++++++++++++++++ .../SqlDriverPlugin.cs | 1 + 10 files changed, 221 insertions(+), 21 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index cf6985e9..afb8dd01 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -23,10 +23,13 @@ public class VerboseLogHook : IContentGeneratingHook { if (!_convSettings.ShowVerboseLog) return; - var dialog = conversations.Last(); - var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; - _logger.LogInformation(log); - + var dialog = conversations.LastOrDefault(); + if (dialog != null) + { + var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; + _logger.LogInformation(log); + } + await Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 260dc166..827c2d7c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -142,6 +142,7 @@ public class KnowledgeBaseController : ControllerBase } #endregion + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index af82b037..678ee82b 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = value.FunctionCall.FunctionName, FunctionArgs = value.FunctionCall.FunctionArguments }; @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments }; @@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 00f47149..57f7ac13 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = value.FunctionCall.FunctionName, FunctionArgs = value.FunctionCall.FunctionArguments }; @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments }; @@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 6d2b4b05..266abbec 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -53,6 +53,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs new file mode 100644 index 00000000..f863adb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs @@ -0,0 +1,24 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.SqlDriver.Controllers; + +[Authorize] +[ApiController] +public class SqlDriverController : ControllerBase +{ + private readonly IServiceProvider _services; + + public SqlDriverController(IServiceProvider services) + { + _services = services; + } + + [HttpPost("/knowledge/database/import")] + public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) + { + var dbKnowledge = _services.GetRequiredService(); + return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 47f64a1f..6fcee0bd 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -20,25 +20,36 @@ public class GetTableDefinitionFn : IFunctionCallback var settings = _services.GetRequiredService(); // Get table DDL from database - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); var tableDdls = new List(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); - foreach (var p in (List)message.Data) + foreach (var table in (List)message.Data) { - var escapedTableName = MySqlHelper.EscapeString(p); - dictionary["@" + "table_name"] = p; - dictionary["table_name"] = escapedTableName; + var escapedTableName = MySqlHelper.EscapeString(table); + + var sql = $"select * from information_schema.tables where table_name = @tableName"; + var result = connection.QueryFirstOrDefault(sql, new + { + tableName = escapedTableName + }); - var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'"; - var result = connection.QueryFirstOrDefault(sql: sql, dictionary); if (result == null) continue; sql = $"SHOW CREATE TABLE `{escapedTableName}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - tableDdls.Add(result); + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + tableDdls.Add(result); + } + + reader.Close(); + command.Dispose(); } + connection.Close(); message.Content = string.Join("\r\n", tableDdls); return true; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs new file mode 100644 index 00000000..48ba820c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs @@ -0,0 +1,19 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class RequestBase +{ + [JsonPropertyName("provider")] + public string? Provider { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } +} + + +public class ImportDbKnowledgeRequest : RequestBase +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs new file mode 100644 index 00000000..7807f87f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -0,0 +1,140 @@ +using static Dapper.SqlMapper; +using Microsoft.Extensions.Logging; +using BotSharp.Core.Infrastructures; +using MySqlConnector; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.SqlDriver.Services; + +public class DbKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public DbKnowledgeService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Import(string provider, string model, string schema) + { + var sqlDriverSettings = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var tables = new HashSet(); + using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString); + + var sql = $"select table_name from information_schema.tables where table_schema = @tableSchema"; + var results = connection.Query(sql, new + { + tableSchema = schema + }); + + foreach (var item in results) + { + if (item == null) continue; + + tables.Add(item.TABLE_NAME); + } + + foreach (var table in tables) + { + try + { + _logger.LogInformation($"Start processing table {table}\r\n"); + + var ddl = GetTableStructure(table); + if (string.IsNullOrEmpty(ddl)) continue; + + var prompt = await GetPrompt(ddl); + var response = await GetAiResponse(prompt, provider, model); + var knowledges = response.Content.JsonArrayContent(); + + if (knowledges.IsNullOrEmpty()) + { + _logger.LogInformation($"No knowledge for table {table}"); + continue; + } + + foreach (var item in knowledges) + { + await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + { + Text = item.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, item.Answer } + } + }); + + _logger.LogInformation($"Knowledge {table} is saved =>\r\nQuestion: {item.Question}\r\nAnswer: {item.Answer}\r\n"); + } + } + catch (Exception ex) + { + var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}"; + _logger.LogWarning(note); + } + } + + return true; + } + + private string GetTableStructure(string table) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); + + var result = string.Empty; + var escapedTableName = MySqlHelper.EscapeString(table); + var sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + } + + reader.Close(); + command.Dispose(); + connection.Close(); + return result; + } + + private async Task GetPrompt(string content) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; + + return render.Render(template, new Dictionary + { + { "table_structure", content } + }); + } + + private async Task GetAiResponse(string prompt, string provider, string model) + { + var agent = new Agent + { + Id = string.Empty, + Name = "Db knowledge", + Instruction = prompt, + }; + + var completion = CompletionProvider.GetChatCompletion(_services, provider, model); + return await completion.GetChatCompletions(agent, new List()); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index aaad1a0f..c059edfb 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -16,6 +16,7 @@ public class SqlDriverPlugin : IBotSharpPlugin }); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); From 32c99dee5f6218bb05064a942464c12fae9de451 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 3 Sep 2024 22:13:25 -0500 Subject: [PATCH 26/27] relocate --- .../BotSharp.Abstraction/BotSharp.Abstraction.csproj | 2 +- .../VectorStorageExtension.cs} | 4 ++-- src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj | 2 +- .../BotSharp.Core/Conversations/ConversationPlugin.cs | 3 +++ .../Knowledges}/Helpers/KnowledgeSettingHelper.cs | 5 ++++- .../BotSharp.Core/Knowledges}/Helpers/TextChopper.cs | 3 ++- .../Knowledges}/Services/KnowledgeService.Create.cs | 6 +++++- .../Knowledges}/Services/KnowledgeService.Delete.cs | 2 +- .../Knowledges}/Services/KnowledgeService.Get.cs | 4 +++- .../Knowledges}/Services/KnowledgeService.Update.cs | 4 +++- .../Knowledges}/Services/KnowledgeService.cs | 8 +++++++- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs | 1 - src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs | 1 - src/Plugins/BotSharp.Plugin.Planner/Using.cs | 2 +- .../Controllers/TwilioVoiceController.cs | 3 +-- 16 files changed, 35 insertions(+), 17 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/VectorStorage/{Helpers/VectorStorageHelper.cs => Extensions/VectorStorageExtension.cs} (90%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Helpers/KnowledgeSettingHelper.cs (89%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Helpers/TextChopper.cs (94%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Create.cs (92%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Delete.cs (95%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Get.cs (96%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Update.cs (91%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.cs (80%) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 94c9e27c..697f94a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs similarity index 90% rename from src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs rename to src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs index f1f0ebde..ec519f04 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs @@ -1,9 +1,9 @@ using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Abstraction.VectorStorage.Helpers; +namespace BotSharp.Abstraction.VectorStorage.Extensions; -public static class VectorStorageHelper +public static class VectorStorageExtension { public static string ToQuestionAnswer(this VectorSearchResult data) { diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 05d485b7..ff10a517 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index f06e2ba2..05653d9c 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -6,6 +6,7 @@ using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Settings; using BotSharp.Abstraction.Templating; using BotSharp.Core.Instructs; +using BotSharp.Core.Knowledges.Services; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; using BotSharp.Core.Templating; @@ -54,6 +55,8 @@ public class ConversationPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } public bool AttachMenu(List menu) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs index 48033c91..a278d7bf 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs @@ -1,4 +1,7 @@ -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Knowledges.Helpers; public static class KnowledgeSettingHelper { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs similarity index 94% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs index 9e9b5f9d..68b58539 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs @@ -1,6 +1,7 @@ +using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +namespace BotSharp.Core.Knowledges.Helpers; public static class TextChopper { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 2c4004fc..d8b82fb5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,4 +1,8 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs index b0973a99..404b4e2c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs index ee36780e..69093659 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs index 0fa1f5cd..b9652f61 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs similarity index 80% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index c668e07e..2c0853d0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,4 +1,10 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Graph; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.VectorStorage; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService : IKnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index a9337419..ffbef770 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.VectorStorage.Helpers; +using BotSharp.Abstraction.VectorStorage.Extensions; namespace BotSharp.Plugin.KnowledgeBase.Functions; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 75d3284b..ffe4d8f1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -21,7 +21,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin return settingService.Bind("KnowledgeBase"); }); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index a78afefd..1437f41e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -31,6 +31,5 @@ global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; -global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/Using.cs b/src/Plugins/BotSharp.Plugin.Planner/Using.cs index fa59ee23..1e6737df 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Using.cs @@ -28,7 +28,7 @@ global using BotSharp.Abstraction.Knowledges; global using BotSharp.Abstraction.Knowledges.Settings; global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage.Models; -global using BotSharp.Abstraction.VectorStorage.Helpers; +global using BotSharp.Abstraction.VectorStorage.Extensions; global using BotSharp.Plugin.Planner.Hooks; global using BotSharp.Plugin.Planner.Enums; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 23a7b469..8304e3b2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -82,8 +82,7 @@ public class TwilioVoiceController : TwilioController } } - response = new VoiceResponse() - .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); } else { From b244e07e1d0055a35a3da82628ffca9f687866c3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 4 Sep 2024 10:16:12 -0500 Subject: [PATCH 27/27] remove knowledge confidence --- .../Conversations/Models/RoleDialogModel.cs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 2c43df1c..8484c5fb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -100,12 +100,6 @@ public class RoleDialogModel : ITrackableMessage [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); - /// - /// Knowledge confidence - /// - [JsonPropertyName("knowledge_confidence")] - public float KnowledgeConfidence { get; set; } = 0.5f; - private RoleDialogModel() { @@ -146,8 +140,7 @@ public class RoleDialogModel : ITrackableMessage Payload = source.Payload, StopCompletion = source.StopCompletion, Instruction = source.Instruction, - Data = source.Data, - KnowledgeConfidence = source.KnowledgeConfidence + Data = source.Data }; } }