SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #616 from iceljc/features/refine-audio-handler

Browse source 
Features/refine audio handler
This commit is contained in:
iceljc 2024-09-04 11:16:08 -05:00 committed by GitHub
parent 0e156c319a c2b870b655
commit 997bfc344f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
156 changed files with 2223 additions and 1244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
BotSharp.sln
View file

@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -449,6 +451,14 @@ Global
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -503,6 +513,7 @@ Global
{05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5}
{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C}
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}
{F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

2
src/BotSharp.AppHost/BotSharp.AppHost.csproj
View file

@ -10,7 +10,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Aspire.Hosting.AppHost" Version="8.0.0" />
<PackageReference Include="Aspire.Hosting.AppHost" Version="8.2.0" />
<PackageReference Include="Aspire.Hosting.NodeJs" Version="8.0.1" />
</ItemGroup>

2
src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

13
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent;
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Abstraction.Conversations.Models;
@ -75,6 +74,9 @@ public class RoleDialogModel : ITrackableMessage
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public RichContent<IRichMessage>? RichContent { get; set; }
/// <summary>
/// Rich content for secondary language
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public RichContent<IRichMessage>? SecondaryRichContent { get; set; }
@ -86,6 +88,9 @@ public class RoleDialogModel : ITrackableMessage
public FunctionCallFromLlm Instruction { get; set; }
/// <summary>
/// Files to be used in conversation
/// </summary>
public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
/// <summary>
@ -94,7 +99,8 @@ public class RoleDialogModel : ITrackableMessage
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonPropertyName("generated_images")]
public List<ImageGeneration> GeneratedImages { get; set; } = new List<ImageGeneration>();
public float KnowledgeConfidence { get; set; } = 0.5f;
private RoleDialogModel()
{
}
@ -134,8 +140,7 @@ public class RoleDialogModel : ITrackableMessage
Payload = source.Payload,
StopCompletion = source.StopCompletion,
Instruction = source.Instruction,
Data = source.Data,
KnowledgeConfidence = source.KnowledgeConfidence
Data = source.Data
};
}
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs
View file

@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters;
public interface IPdf2ImageConverter
{
public string Name { get; }
public string Provider { get; }
/// <summary>
/// Convert pdf pages to images, and return a list of image file paths

9
src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs
View file

@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files;
public class FileCoreSettings
{
public string Storage { get; set; } = FileStorageEnum.LocalFileStorage;
public string Pdf2TextConverter { get; set; }
public string Pdf2ImageConverter { get; set; }
public SettingBase Pdf2TextConverter { get; set; }
public SettingBase Pdf2ImageConverter { get; set; }
}
public class SettingBase
{
public string Provider { get; set; }
}

14
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs
View file

@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files;
public interface IFileInstructService
{
#region Image
Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images);
Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images);
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask);
Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask);
#endregion
#region Pdf
@ -17,7 +17,11 @@ public interface IFileInstructService
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files);
#endregion
#region Audio
Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null);
#endregion
#region Select file

13
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -6,6 +6,7 @@ public interface IFileStorageService
{
#region Common
string GetDirectory(string conversationId);
IEnumerable<string> GetFiles(string relativePath, string? searchQuery = null);
byte[] GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
@ -16,7 +17,6 @@ public interface IFileStorageService
string BuildDirectory(params string[] segments);
#endregion
#region Conversation
/// <summary>
/// Get the message file screenshots for specific content types, e.g., pdf
@ -24,7 +24,7 @@ public interface IFileStorageService
/// <param name="conversationId"></param>
/// <param name="messageIds"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds);
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds);
/// <summary>
/// Get the files that have been uploaded in the chat. No screenshot images are included.
@ -37,7 +37,7 @@ public interface IFileStorageService
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files);
/// <summary>
/// Delete files under messages
@ -54,10 +54,11 @@ public interface IFileStorageService
#region User
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
bool SaveUserAvatar(InputFileModel file);
#endregion
#region Speech
Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data);
Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName);
bool SaveSpeechFile(string conversationId, string fileName, BinaryData data);
BinaryData GetSpeechFile(string conversationId, string fileName);
#endregion
}

9
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -1,7 +1,12 @@
namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile : FileBase
public class BotSharpFile : FileInformation
{
/// <summary>
/// File data => format: "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
}

28
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs
View file

@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models;
public class FileBase
{
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileStorageUrl { get; set; } = string.Empty;
/// <summary>
/// File name without extension
/// </summary>
@ -29,18 +15,4 @@ public class FileBase
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
/// <summary>
/// File content type
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; } = string.Empty;
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
}

38
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs Normal file
View file

@ -0,0 +1,38 @@
namespace BotSharp.Abstraction.Files.Models;
public class FileInformation
{
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileStorageUrl { get; set; } = string.Empty;
/// <summary>
/// File content type
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; } = string.Empty;
/// <summary>
/// File name without extension
/// </summary>
[JsonPropertyName("file_name")]
public string FileName { get; set; } = string.Empty;
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
}

16
src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs Normal file
View file

@ -0,0 +1,16 @@
namespace BotSharp.Abstraction.Files.Models;
public class InputFileModel : FileBase
{
/// <summary>
/// File name with extension
/// </summary>
[JsonPropertyName("file_name")]
public new string FileName { get; set; } = string.Empty;
/// <summary>
/// File data => format: "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
public new string FileData { get; set; } = string.Empty;
}

7
src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs
View file

@ -1,7 +0,0 @@
namespace BotSharp.Abstraction.Files.Models;
public class InputMessageFiles
{
public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
public BotSharpFile? Mask { get; set; }
}

18
src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs Normal file
View file

@ -0,0 +1,18 @@
namespace BotSharp.Abstraction.Files.Models;
public class InstructFileModel : FileBase
{
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Files.Models;
public class MessageFileModel : FileBase
public class MessageFileModel : FileInformation
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }

25
src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs
View file

@ -1,4 +1,6 @@
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Abstraction.Files.Utilities;
@ -26,11 +28,30 @@ public static class FileUtility
return (contentType, Convert.FromBase64String(base64Str));
}
public static string GetFileContentType(string filePath)
public static string BuildFileDataFromFile(string fileName, byte[] bytes)
{
var contentType = GetFileContentType(fileName);
var base64 = Convert.ToBase64String(bytes);
return $"data:{contentType};base64,{base64}";
}
public static string BuildFileDataFromFile(IFormFile file)
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var contentType = GetFileContentType(file.FileName);
var base64 = Convert.ToBase64String(stream.ToArray());
stream.Close();
return $"data:{contentType};base64,{base64}";
}
public static string GetFileContentType(string fileName)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
if (!provider.TryGetContentType(fileName, out contentType))
{
contentType = string.Empty;
}

2
src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph;
public interface IGraphDb
{
public string Name { get; }
public string Provider { get; }
Task<GraphSearchData> Search(string query, GraphSearchOptions options);
}

7
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Knowledges;
public interface IKnowledgeService
{
#region Vector
Task<bool> CreateVectorCollection(string collectionName, int dimension);
Task<bool> DeleteVectorCollection(string collectionName);
Task<IEnumerable<string>> GetVectorCollections();
Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model);
@ -12,6 +15,10 @@ public interface IKnowledgeService
Task<bool> DeleteVectorCollectionData(string collectionName, string id);
Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
#endregion
#region Graph
Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options);
Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions);
#endregion
}

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs
View file

@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges
{
public interface IPdf2TextConverter
{
public string Name { get; }
public string Provider { get; }
Task<string> ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum);
}
}

11
src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs
View file

@ -1,11 +0,0 @@
using BotSharp.Abstraction.Knowledges.Models;
namespace BotSharp.Abstraction.Knowledges;
/// <summary>
/// Chop large content into chunks
/// </summary>
public interface ITextChopper
{
List<string> Chop(string content, ChunkOption option);
}

12
src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs
View file

@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Knowledges.Settings;
public class KnowledgeBaseSettings
{
public string VectorDb { get; set; }
public string GraphDb { get; set; }
public SettingBase VectorDb { get; set; }
public SettingBase GraphDb { get; set; }
public DefaultKnowledgeBaseSetting Default { get; set; }
public List<VectorCollectionSetting> Collections { get; set; } = new();
@ -23,9 +23,13 @@ public class VectorCollectionSetting
public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
}
public class KnowledgeTextEmbeddingSetting
public class KnowledgeTextEmbeddingSetting : SettingBase
{
public string Provider { get; set; }
public string Model { get; set; }
public int Dimension { get; set; }
}
public class SettingBase
{
public string Provider { get; set; }
}

13
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs Normal file
View file

@ -0,0 +1,13 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
public interface IAudioCompletion
{
string Provider { get; }
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null);
Task<BinaryData> GenerateAudioFromTextAsync(string text);
void SetModelName(string model);
}

10
src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs
View file

@ -1,10 +0,0 @@
namespace BotSharp.Abstraction.MLTasks;
public interface ISpeechToText
{
string Provider { get; }
Task<string> GenerateTextFromAudioAsync(string filePath);
// Task<string> AudioToTextTranscript(Stream stream);
Task SetModelName(string modelType);
}

23
src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs
View file

@ -1,23 +0,0 @@
namespace BotSharp.Abstraction.MLTasks
{
public interface ITextToSpeech
{
/// <summary>
/// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
/// </summary>
string Provider { get; }
/// <summary>
/// Set model name, one provider can consume different model or version(s)
/// </summary>
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null);
}
public interface ITextToSpeechOptions
{
}
}

3
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -68,5 +68,6 @@ public enum LlmModelType
Text = 1,
Chat = 2,
Image = 3,
Embedding = 4
Embedding = 4,
Audio = 5
}

10
src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Models;
public class KeyValue
{
[JsonPropertyName("key")]
public string Key { get; set; }
[JsonPropertyName("value")]
public string Value { get; set; }
}

6
src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Models;
public class MessageConfig : InputMessageFiles
public class MessageConfig
{
/// <summary>
/// Completion Provider
@ -15,7 +15,7 @@ public class MessageConfig : InputMessageFiles
public virtual string? Model { get; set; } = null;
/// <summary>
/// Model name
/// Model id
/// </summary>
[JsonPropertyName("model_id")]
public virtual string? ModelId { get; set; } = null;
@ -34,7 +34,7 @@ public class MessageConfig : InputMessageFiles
/// <summary>
/// Conversation states from input
/// </summary>
public List<MessageState> States { get; set; } = new List<MessageState>();
public List<MessageState> States { get; set; } = new();
/// <summary>
/// Agent task id

29
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs Normal file
View file

@ -0,0 +1,29 @@
using BotSharp.Abstraction.Knowledges.Enums;
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Abstraction.VectorStorage.Extensions;
public static class VectorStorageExtension
{
public static string ToQuestionAnswer(this VectorSearchResult data)
{
if (data?.Data == null) return string.Empty;
return $"Question: {data.Data[KnowledgePayloadName.Text]}\r\nAnswer: {data.Data[KnowledgePayloadName.Answer]}";
}
public static string ToPayloadPair(this VectorSearchResult data, IList<string> payloads)
{
if (data?.Data == null || payloads.IsNullOrEmpty()) return string.Empty;
var results = data.Data.Where(x => payloads.Contains(x.Key))
.OrderBy(x => payloads.IndexOf(x.Key))
.Select(x =>
{
return $"{x.Key}: {x.Value}";
})
.ToList();
return string.Join("\r\n", results.Where(x => !string.IsNullOrWhiteSpace(x)));
}
}

5
src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
View file

@ -4,12 +4,13 @@ namespace BotSharp.Abstraction.VectorStorage;
public interface IVectorDb
{
string Name { get; }
string Provider { get; }
Task<IEnumerable<string>> GetCollections();
Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
Task CreateCollection(string collectionName, int dim);
Task<bool> CreateCollection(string collectionName, int dimension);
Task<bool> DeleteCollection(string collectionName);
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
Task<bool> DeleteCollectionData(string collectionName, Guid id);

5
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs
View file

@ -4,4 +4,7 @@ public class VectorFilter : StringIdPagination
{
[JsonPropertyName("with_vector")]
public bool WithVector { get; set; }
}
[JsonPropertyName("search_pairs")]
public IEnumerable<KeyValue>? SearchPairs { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Knowledges.Enums;
namespace BotSharp.Abstraction.VectorStorage.Models;
public class VectorSearchResult : VectorCollectionData

2
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

3
src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs
View file

@ -6,6 +6,7 @@ using BotSharp.Abstraction.Routing.Planning;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Templating;
using BotSharp.Core.Instructs;
using BotSharp.Core.Knowledges.Services;
using BotSharp.Core.Messaging;
using BotSharp.Core.Routing.Planning;
using BotSharp.Core.Templating;
@ -54,6 +55,8 @@ public class ConversationPlugin : IBotSharpPlugin
services.AddScoped<IExecutor, InstructExecutor>();
services.AddScoped<IInstructService, InstructService>();
services.AddScoped<ITokenStatistics, TokenStatistics>();
services.AddScoped<IKnowledgeService, KnowledgeService>();
}
public bool AttachMenu(List<PluginMenuDef> menu)

20
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs Normal file
View file

@ -0,0 +1,20 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null)
{
var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
stream.Position = 0;
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text);
stream.Close();
return content;
}
}

24
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images)
public async Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images)
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
@ -14,10 +14,10 @@ public partial class FileInstructService
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images?.ToList() ?? new List<BotSharpFile>()
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List<BotSharpFile>()
}
});
return message;
return message.Content;
}
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)
@ -30,7 +30,7 @@ public partial class FileInstructService
return message;
}
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image)
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
@ -43,16 +43,17 @@ public partial class FileInstructService
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image)
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
@ -65,16 +66,17 @@ public partial class FileInstructService
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, text), stream, fileName);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask)
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask)
{
if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) ||
(string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData)))
@ -94,10 +96,12 @@ public partial class FileInstructService
maskStream.Write(maskBytes, 0, maskBytes.Length);
maskStream.Position = 0;
var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}";
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName);
imageStream.Close();
maskStream.Close();
@ -105,7 +109,7 @@ public partial class FileInstructService
}
#region Private methods
private async Task<byte[]> DownloadFile(BotSharpFile file)
private async Task<byte[]> DownloadFile(InstructFileModel file)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))

12
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files)
{
var content = string.Empty;
@ -50,7 +50,7 @@ public partial class FileInstructService
}
#region Private methods
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{
@ -80,9 +80,9 @@ public partial class FileInstructService
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, true);
var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(pdfDir, bytes);
locs.Add(pdfDir);
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(outputDir, bytes);
locs.Add(outputDir);
}
}
catch (Exception ex)
@ -98,7 +98,7 @@ public partial class FileInstructService
{
var images = new List<string>();
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter);
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
if (converter == null || files.IsNullOrEmpty())
{
return images;

1
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
View file

@ -1,3 +1,4 @@
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService : IFileInstructService

36
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs
View file

@ -1,28 +1,38 @@
using System.IO;
namespace BotSharp.Core.Files.Services
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public partial class LocalFileStorageService
public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data)
{
public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)
try
{
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var filePath = Path.Combine(dir, fileName);
if (File.Exists(filePath)) return;
using var file = File.Create(filePath);
using var input = data.ToStream();
await input.CopyToAsync(file);
}
public async Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName)
var filePath = Path.Combine(dir, fileName);
if (File.Exists(filePath)) return false;
using var fs = File.Create(filePath);
using var ds = data.ToStream();
ds.CopyTo(fs);
return true;
}
catch (Exception ex)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
return await BinaryData.FromStreamAsync(file);
_logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public BinaryData GetSpeechFile(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
return BinaryData.FromStream(file);
}
}

16
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs
View file

@ -14,6 +14,22 @@ public partial class LocalFileStorageService
return dir;
}
public IEnumerable<string> GetFiles(string relativePath, string? searchPattern = null)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return Enumerable.Empty<string>();
}
var path = Path.Combine(_baseDir, relativePath);
if (!string.IsNullOrWhiteSpace(searchPattern))
{
return Directory.GetFiles(path, searchPattern);
}
return Directory.GetFiles(path);
}
public byte[] GetFileBytes(string fileStorageUrl)
{
using var stream = File.OpenRead(fileStorageUrl);

6
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
@ -118,7 +118,7 @@ public partial class LocalFileStorageService
return foundMsgs;
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files)
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files)
{
if (files.IsNullOrEmpty()) return false;
@ -276,7 +276,7 @@ public partial class LocalFileStorageService
private IPdf2ImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter);
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

2
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs
View file

@ -16,7 +16,7 @@ public partial class LocalFileStorageService
return found;
}
public bool SaveUserAvatar(BotSharpFile file)
public bool SaveUserAvatar(InputFileModel file)
{
if (file == null || string.IsNullOrEmpty(file.FileData)) return false;

6
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService
private readonly ILogger<LocalFileStorageService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _audioTypes = new List<string>
{
"mp3",
"wav"
};
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
private const string USER_FILE_FOLDER = "user";

28
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -28,6 +28,10 @@ public class CompletionProvider
{
return GetImageCompletion(services, provider: provider, model: model);
}
else if (settings.Type == LlmModelType.Audio)
{
return GetAudioCompletion(services, provider: provider, model: model);
}
else
{
return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig);
@ -108,7 +112,7 @@ public class CompletionProvider
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve completion provider by {provider}");
logger.LogError($"Can't resolve text-embedding provider by {provider}");
}
@ -120,35 +124,19 @@ public class CompletionProvider
return completer;
}
public static ITextToSpeech GetTextToSpeech(
public static IAudioCompletion GetAudioCompletion(
IServiceProvider services,
string provider,
string model)
{
var completions = services.GetServices<ITextToSpeech>();
var completions = services.GetServices<IAudioCompletion>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve text2speech provider by {provider}");
logger.LogError($"Can't resolve audio-completion provider by {provider}");
}
completer.SetModelName(model);
return completer;
}
public static ISpeechToText GetSpeechToText(
IServiceProvider services,
string provider,
string model
)
{
var completions = services.GetServices<ISpeechToText>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve speech2text provider by {provider}");
}
completer.SetModelName(model);
return completer;
}

36
src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs Normal file
View file

@ -0,0 +1,36 @@
using BotSharp.Abstraction.Knowledges.Settings;
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Core.Knowledges.Helpers;
public static class KnowledgeSettingHelper
{
public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName)
{
var settings = services.GetRequiredService<KnowledgeBaseSettings>();
var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding;
if (found == null)
{
found = settings.Default.TextEmbedding;
}
var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == found.Provider);
var dimension = found.Dimension;
if (found.Dimension <= 0)
{
dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model);
}
embedding.SetModelName(found.Model);
embedding.SetDimension(dimension);
return embedding;
}
private static int GetLlmTextEmbeddingDimension(IServiceProvider services, string provider, string model)
{
var settings = services.GetRequiredService<ILlmProviderService>();
var found = settings.GetSetting(provider, model);
return found?.Dimension ?? 0;
}
}

11
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs → src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs
View file

@ -1,17 +1,18 @@
using BotSharp.Abstraction.Knowledges.Models;
using System.Text.RegularExpressions;
namespace BotSharp.Plugin.KnowledgeBase.Services;
namespace BotSharp.Core.Knowledges.Helpers;
public class TextChopperService : ITextChopper
public static class TextChopper
{
public List<string> Chop(string content, ChunkOption option)
public static List<string> Chop(string content, ChunkOption option)
{
content = Regex.Replace(content, @"\.{2,}", " ");
content = Regex.Replace(content, @"_{2,}", " ");
return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option);
}
private List<string> ChopByWord(string content, ChunkOption option)
private static List<string> ChopByWord(string content, ChunkOption option)
{
var chunks = new List<string>();
@ -34,7 +35,7 @@ public class TextChopperService : ITextChopper
return chunks;
}
private List<string> ChopByChar(string content, ChunkOption option)
private static List<string> ChopByChar(string content, ChunkOption option)
{
var chunks = new List<string>();
var currentPos = 0;

27
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs → src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs
View file

@ -1,11 +1,15 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.VectorStorage.Models;
using BotSharp.Core.Knowledges.Helpers;
namespace BotSharp.Core.Knowledges.Services;
public partial class KnowledgeService
{
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
{
var index = 0;
var lines = _textChopper.Chop(knowledge.Content, new ChunkOption
var lines = TextChopper.Chop(knowledge.Content, new ChunkOption
{
Size = 1024,
Conjunction = 32,
@ -25,6 +29,25 @@ public partial class KnowledgeService
}
}
public async Task<bool> CreateVectorCollection(string collectionName, int dimension)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
var db = GetVectorDb();
return await db.CreateCollection(collectionName, dimension);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
{
try

21
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs → src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs
View file

@ -1,7 +1,26 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
namespace BotSharp.Core.Knowledges.Services;
public partial class KnowledgeService
{
public async Task<bool> DeleteVectorCollection(string collectionName)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
var db = GetVectorDb();
return await db.DeleteCollection(collectionName);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<bool> DeleteVectorCollectionData(string collectionName, string id)
{
try

4
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs → src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Abstraction.Graph.Models;
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Plugin.KnowledgeBase.Services;
namespace BotSharp.Core.Knowledges.Services;
public partial class KnowledgeService
{

4
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs → src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs
View file

@ -1,4 +1,6 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Core.Knowledges.Services;
public partial class KnowledgeService
{

17
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs → src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs
View file

@ -1,38 +1,41 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
using BotSharp.Abstraction.Graph;
using BotSharp.Abstraction.Knowledges.Settings;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.VectorStorage;
using BotSharp.Core.Knowledges.Helpers;
namespace BotSharp.Core.Knowledges.Services;
public partial class KnowledgeService : IKnowledgeService
{
private readonly IServiceProvider _services;
private readonly KnowledgeBaseSettings _settings;
private readonly ITextChopper _textChopper;
private readonly ILogger<KnowledgeService> _logger;
public KnowledgeService(
IServiceProvider services,
KnowledgeBaseSettings settings,
ITextChopper textChopper,
ILogger<KnowledgeService> logger)
{
_services = services;
_settings = settings;
_textChopper = textChopper;
_logger = logger;
}
private IVectorDb GetVectorDb()
{
var db = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
var db = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider);
return db;
}
private IGraphDb GetGraphDb()
{
var db = _services.GetServices<IGraphDb>().FirstOrDefault(x => x.Name == _settings.GraphDb);
var db = _services.GetServices<IGraphDb>().FirstOrDefault(x => x.Provider == _settings.GraphDb.Provider);
return db;
}
private ITextEmbedding GetTextEmbedding(string collection)
{
return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection);
return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection);
}
}

1
src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Routing.Enums;
using BotSharp.Abstraction.Routing.Planning;
using BotSharp.Core.Routing.Planning;

8
src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid
View file

@ -1,8 +1,10 @@
You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name.
the summarized question/answer should:
You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname, data type and comments, generate the related knowledge for DBA and BA. When users ask the question, they don't know the table name.
The summarized question/answer should:
1. help user to identify the location of tables to find further information
2. identify the table structure and data relationship based on the task description
3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer
3. summarize all the tables to table relationship information based on the FOREIGN KEY, and include both tables in the answer
Go through all the columns and generate multiple question & answer pairs.
The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n.

11
src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs
View file

@ -23,10 +23,13 @@ public class VerboseLogHook : IContentGeneratingHook
{
if (!_convSettings.ShowVerboseLog) return;
var dialog = conversations.Last();
var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>";
_logger.LogInformation(log);
var dialog = conversations.LastOrDefault();
if (dialog != null)
{
var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>";
_logger.LogInformation(log);
}
await Task.CompletedTask;
}

3
src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
@ -19,6 +19,7 @@
<ItemGroup>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageReference Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">

6
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -294,9 +294,7 @@ public class ConversationController : ControllerBase
}
[HttpPost("/conversation/{agentId}/{conversationId}/sse")]
public async Task SendMessageSse([FromRoute] string agentId,
[FromRoute] string conversationId,
[FromBody] NewMessageModel input)
public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input)
{
var conv = _services.GetRequiredService<IConversationService>();
var inputMsg = new RoleDialogModel(AgentRole.User, input.Text)
@ -391,7 +389,7 @@ public class ConversationController : ControllerBase
}
[HttpPost("/agent/{agentId}/conversation/{conversationId}/upload")]
public async Task<string> UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input)
public async Task<string> UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] InputMessageFiles input)
{
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(conversationId, input.States);

284
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
@ -20,8 +21,7 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/{agentId}")]
public async Task<InstructResult> InstructCompletion([FromRoute] string agentId,
[FromBody] InstructMessageModel input)
public async Task<InstructResult> InstructCompletion([FromRoute] string agentId, [FromBody] InstructMessageModel input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -79,7 +79,7 @@ public class InstructModeController : ControllerBase
#region Read image
[HttpPost("/instruct/multi-modal")]
public async Task<string> MultiModalCompletion([FromBody] IncomingMessageModel input)
public async Task<string> MultiModalCompletion([FromBody] MultiModalRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -87,21 +87,50 @@ public class InstructModeController : ControllerBase
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files);
return message.Content;
var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files);
return content;
}
catch (Exception ex)
{
var error = $"Error in analyzing files. {ex.Message}";
var error = $"Error in reading images. {ex.Message}";
_logger.LogError(error);
return error;
}
}
[HttpPost("/instruct/multi-modal/upload")]
public async Task<MultiModalViewModel> MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new MultiModalViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var files = new List<InstructFileModel>
{
new InstructFileModel { FileData = data }
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadImages(provider, model, text, files);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in reading image upload. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
#endregion
#region Generate image
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -127,7 +156,7 @@ public class InstructModeController : ControllerBase
#region Edit image
[HttpPost("/instruct/image-variation")]
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] IncomingMessageModel input)
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] ImageVariationRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -135,16 +164,16 @@ public class InstructModeController : ControllerBase
try
{
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (image == null)
if (input.File == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
}
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.VaryImage(input.Provider, input.Model, image);
var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
@ -156,8 +185,43 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-variation/upload")]
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
stream.Close();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image variation upload. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-edit")]
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] IncomingMessageModel input)
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditRequest input)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -166,12 +230,11 @@ public class InstructModeController : ControllerBase
try
{
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (image == null)
if (input.File == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" };
}
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image);
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -185,8 +248,44 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-edit/upload")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, file.FileName);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
stream.Close();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image edit upload. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-mask-edit")]
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] IncomingMessageModel input)
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditRequest input)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -195,11 +294,11 @@ public class InstructModeController : ControllerBase
try
{
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
var image = input.File;
var mask = input.Mask;
if (image == null || mask == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" };
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" };
}
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask);
imageViewModel.Content = message.Content;
@ -214,11 +313,52 @@ public class InstructModeController : ControllerBase
return imageViewModel;
}
}
[HttpPost("/instruct/image-mask-edit/upload")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
using var imageStream = new MemoryStream();
image.CopyTo(imageStream);
imageStream.Position = 0;
using var maskStream = new MemoryStream();
mask.CopyTo(maskStream);
maskStream.Position = 0;
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
imageStream.Close();
maskStream.Close();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image mask edit upload. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
#endregion
#region Pdf
[HttpPost("/instruct/pdf-completion")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] IncomingMessageModel input)
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] MultiModalRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -239,5 +379,109 @@ public class InstructModeController : ControllerBase
return viewModel;
}
}
[HttpPost("/instruct/pdf-completion/upload")]
public async Task<PdfCompletionViewModel> PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List<MessageState>? states = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new PdfCompletionViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var files = new List<InstructFileModel>
{
new InstructFileModel { FileData = data }
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in pdf completion upload. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
#endregion
#region Audio
[HttpPost("/instruct/speech-to-text")]
public async Task<SpeechToTextViewModel> SpeechToText([FromBody] SpeechToTextRequest input)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new SpeechToTextViewModel();
try
{
var audio = input.File;
if (audio == null)
{
return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" };
}
var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in speech to text. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
[HttpPost("/instruct/speech-to-text/upload")]
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] string? text = null, [FromForm] List<MessageState>? states = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new SpeechToTextViewModel();
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text);
viewModel.Content = content;
stream.Close();
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in speech-to-text upload. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
[HttpPost("/instruct/text-to-speech")]
public async Task<IActionResult> TextToSpeech([FromBody] TextToSpeechRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1");
var binaryData = await completion.GenerateAudioFromTextAsync(input.Text);
var stream = binaryData.ToStream();
stream.Position = 0;
return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" };
}
#endregion
}

22
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -18,12 +18,25 @@ public class KnowledgeBaseController : ControllerBase
_services = services;
}
#region Vector
[HttpGet("knowledge/vector/collections")]
public async Task<IEnumerable<string>> GetVectorCollections()
{
return await _knowledgeService.GetVectorCollections();
}
[HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")]
public async Task<bool> CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension)
{
return await _knowledgeService.CreateVectorCollection(collection, dimension);
}
[HttpDelete("knowledge/vector/{collection}/delete-collection")]
public async Task<bool> GetVectorCollections([FromRoute] string collection)
{
return await _knowledgeService.DeleteVectorCollection(collection);
}
[HttpPost("/knowledge/vector/{collection}/search")]
public async Task<IEnumerable<VectorKnowledgeViewModel>> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request)
{
@ -91,7 +104,7 @@ public class KnowledgeBaseController : ControllerBase
public async Task<IActionResult> UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum)
{
var setttings = _services.GetRequiredService<FileCoreSettings>();
var textConverter = _services.GetServices<IPdf2TextConverter>().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter);
var textConverter = _services.GetServices<IPdf2TextConverter>().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider);
var filePath = Path.GetTempFileName();
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None))
@ -109,7 +122,10 @@ public class KnowledgeBaseController : ControllerBase
System.IO.File.Delete(filePath);
return Ok(new { count = 1, file.Length });
}
#endregion
#region Graph
[HttpPost("/knowledge/graph/search")]
public async Task<GraphKnowledgeViewModel> SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request)
{
@ -124,7 +140,10 @@ public class KnowledgeBaseController : ControllerBase
Result = result.Result
};
}
#endregion
#region Knowledge
[HttpPost("/knowledge/search")]
public async Task<KnowledgeSearchViewModel> SearchKnowledge([FromBody] SearchKnowledgeRequest request)
{
@ -148,4 +167,5 @@ public class KnowledgeBaseController : ControllerBase
GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null
};
}
#endregion
}

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs
View file

@ -25,6 +25,6 @@ public class LlmProviderController : ControllerBase
public IEnumerable<LlmModelSetting> GetLlmProviderModels([FromRoute] string provider)
{
var list = _llmProvider.GetProviderModels(provider);
return list.Where(x => !x.ImageGeneration);
return list.Where(x => x.Type == LlmModelType.Chat);
}
}

7
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -135,9 +135,14 @@ public class UserController : ControllerBase
#region Avatar
[HttpPost("/user/avatar")]
public bool UploadUserAvatar([FromBody] BotSharpFile file)
public bool UploadUserAvatar([FromBody] UserAvatarModel input)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var file = new InputFileModel
{
FileName = input.FileName,
FileData = input.FileData,
};
return fileStorage.SaveUserAvatar(file);
}

7
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.OpenAPI.ViewModels.Conversations;
public class InputMessageFiles
{
public List<MessageState> States { get; set; } = new();
public List<InputFileModel> Files { get; set; } = new();
}

9
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs
View file

@ -2,18 +2,11 @@ using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class ImageGenerationViewModel
public class ImageGenerationViewModel : InstructBaseViewModel
{
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("images")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<ImageViewModel> Images { get; set; } = new List<ImageViewModel>();
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}
public class ImageViewModel

75
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs Normal file
View file

@ -0,0 +1,75 @@
using System.Text.Json.Serialization;
namespace BotSharp.Abstraction.Instructs.Models;
public class InstructBaseRequest
{
[JsonPropertyName("provider")]
public virtual string? Provider { get; set; } = null;
[JsonPropertyName("model")]
public virtual string? Model { get; set; } = null;
[JsonPropertyName("model_id")]
public virtual string? ModelId { get; set; } = null;
[JsonPropertyName("states")]
public List<MessageState> States { get; set; } = new();
}
public class MultiModalRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
[JsonPropertyName("files")]
public List<InstructFileModel> Files { get; set; } = new();
}
public class ImageGenerationRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
}
public class ImageVariationRequest : InstructBaseRequest
{
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}
public class ImageEditRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}
public class ImageMaskEditRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
[JsonPropertyName("mask")]
public InstructFileModel Mask { get; set; }
}
public class SpeechToTextRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string? Text { get; set; }
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}
public class TextToSpeechRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; }
}

13
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs Normal file
View file

@ -0,0 +1,13 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class InstructBaseViewModel
{
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Conversations.Models;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class InstructMessageModel : IncomingMessageModel

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class MultiModalViewModel : InstructBaseViewModel
{
}

10
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs
View file

@ -1,13 +1,5 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class PdfCompletionViewModel
public class PdfCompletionViewModel : InstructBaseViewModel
{
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class SpeechToTextViewModel : InstructBaseViewModel
{
}

18
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs Normal file
View file

@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Users;
public class UserAvatarModel
{
/// <summary>
/// File name with extension
/// </summary>
[JsonPropertyName("file_name")]
public string FileName { get; set; } = string.Empty;
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
public string FileData { get; set; } = string.Empty;
}

35
src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs
View file

@ -1,31 +1,24 @@
using BotSharp.Plugin.AudioHandler.Settings;
using BotSharp.Plugin.AudioHandler.Provider;
using BotSharp.Abstraction.Settings;
namespace BotSharp.Plugin.AudioHandler
namespace BotSharp.Plugin.AudioHandler;
public class AudioHandlerPlugin : IBotSharpPlugin
{
public class AudioHandlerPlugin : IBotSharpPlugin
public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8";
public string Name => "Audio Handler";
public string Description => "Process audio input and transform it into text output.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8";
public string Name => "Audio Handler";
public string Description => "Process audio input and transform it into text output.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
services.AddScoped(provider =>
{
//var settings = new AudioHandlerSettings();
//config.Bind("AudioHandler", settings);
//services.AddSingleton(x => settings);
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddScoped<ISpeechToText, NativeWhisperProvider>();
services.AddScoped<IAudioProcessUtilities, AudioProcessUtilities>();
services.AddScoped<IAgentHook, AudioHandlerHook>();
services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
}
services.AddScoped<IAudioCompletion, NativeWhisperProvider>();
services.AddScoped<IAgentHook, AudioHandlerHook>();
services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
}
}

14
src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj
View file

@ -22,5 +22,19 @@
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_audio_request.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_audio_request.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_audio_request.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_audio_request.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>

69
src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs
View file

@ -1,69 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.AudioHandler.Models;
using BotSharp.Plugin.AudioHandler.Provider;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.AudioHandler.Controllers
{
#if DEBUG
[AllowAnonymous]
#endif
[ApiController]
public class AudioController : ControllerBase
{
private readonly ISpeechToText _nativeWhisperProvider;
private readonly IServiceProvider _services;
public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service)
{
_nativeWhisperProvider = nativeWhisperProvider;
_services = service;
}
[HttpGet("audio/transcript")]
public async Task<IActionResult> GetTextFromAudioController(string audioInputString, string modelType = "")
{
#if DEBUG
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
await _nativeWhisperProvider.SetModelName(modelType);
var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);
}
[HttpPost("openai/audio/transcript")]
public async Task<IActionResult> GetTextFromAudioOpenAiController(string filePath)
{
#if DEBUG
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1");
var result = await client.GenerateTextFromAudioAsync(filePath);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);
}
}
}

39
src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs
View file

@ -1,32 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Whisper.net.Wave;
namespace BotSharp.Plugin.AudioHandler.Enums;
namespace BotSharp.Plugin.AudioHandler.Enums
public enum AudioType
{
public enum AudioType
{
wav,
mp3,
}
wav,
mp3,
}
public static class AudioTypeExtensions
public static class AudioTypeExtensions
{
public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
public static string ToFileType(this AudioType audioType)
{
public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
public static string ToFileType(this AudioType audioType)
string type = audioType switch
{
string type = audioType switch
{
AudioType.mp3 => "audio/mpeg",
AudioType.wav => "audio/wav",
_ => throw new NotImplementedException($"No support found for {audioType}")
};
return type;
}
AudioType.mp3 => "audio/mpeg",
AudioType.wav => "audio/wav",
_ => throw new NotImplementedException($"No support found for {audioType}")
};
return type;
}
}

13
src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs
View file

@ -1,13 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.Enums;
namespace BotSharp.Plugin.AudioHandler.Enums
public class UtilityName
{
public class UtilityName
{
public const string AudioHandler = "audio-handler";
}
public const string AudioHandler = "audio-handler";
}

68
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs
View file

@ -1,68 +0,0 @@
using BotSharp.Plugin.AudioHandler.Enums;
using NAudio;
using NAudio.Wave;
using NAudio.Wave.SampleProviders;
namespace BotSharp.Plugin.AudioHandler.Functions;
public class AudioProcessUtilities : IAudioProcessUtilities
{
public AudioProcessUtilities()
{
}
public Stream ConvertMp3ToStream(string mp3FileName)
{
var fileStream = File.OpenRead(mp3FileName);
using var reader = new Mp3FileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
}
public Stream ConvertWavToStream(string wavFileName)
{
var fileStream = File.OpenRead(wavFileName);
using var reader = new WaveFileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
}
public Stream ConvertToStream(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException("fileName is Null");
}
string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse<AudioType>(fileExtension, out AudioType fileType))
{
throw new NotSupportedException($"File extension: '{fileExtension}' not supported");
}
var stream = fileType switch
{
AudioType.mp3 => ConvertMp3ToStream(fileName),
AudioType.wav => ConvertWavToStream(fileName),
_ => throw new NotSupportedException("File extension not supported"),
};
return stream;
}
}

85
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.StaticFiles;
@ -10,9 +9,9 @@ public class HandleAudioRequestFn : IFunctionCallback
public string Indication => "Handling audio request";
private readonly IServiceProvider _serviceProvider;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
private Agent? _agent;
private readonly IEnumerable<string> _audioContentType = new List<string>
{
@ -20,13 +19,13 @@ public class HandleAudioRequestFn : IFunctionCallback
AudioType.wav.ToFileType(),
};
public HandleAudioRequestFn(
IFileStorageService fileStorage,
IServiceProvider serviceProvider,
ILogger<HandleAudioRequestFn> logger,
BotSharpOptions options
)
BotSharpOptions options)
{
_fileStorage = fileStorage;
_serviceProvider = serviceProvider;
_logger = logger;
_options = options;
@ -36,32 +35,31 @@ public class HandleAudioRequestFn : IFunctionCallback
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var isNeedSummary = args?.IsNeedSummary ?? false;
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText;
var response = await GetResponeFromDialogs(dialogs);
message.Content = response;
return true;
}
private async Task<List<RoleDialogModel>> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var fileService = _serviceProvider.GetRequiredService<IFileStorageService>();
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList();
foreach (var dialog in dialogs)
{
var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty())
continue;
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
{
@ -74,53 +72,48 @@ public class HandleAudioRequestFn : IFunctionCallback
return dialogs;
}
private bool ParseAudioFileType(string fileType)
{
fileType = fileType.ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType);
return canParse;
}
private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var whisperService = await PrepareModel("native"); // openai, native
var audioCompletion = PrepareModel();
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
int transcribedCount = 0;
var transcripts = new List<string>();
foreach (var file in dialog.Files)
{
if (file == null)
continue;
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
string extension = Path.GetExtension(file?.FileStorageUrl);
if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl))
{
file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl);
transcribedCount++;
}
var extension = Path.GetExtension(file.FileStorageUrl);
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!ParseAudioFileType(fileName)) continue;
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = new MemoryStream(bytes);
stream.Position = 0;
var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
}
if (transcribedCount == 0)
if (transcripts.IsNullOrEmpty())
{
throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
}
var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList();
return string.Join("\n\r", resList);
return string.Join("\r\n\r\n", transcripts);
}
private async Task<ISpeechToText> PrepareModel(string modelName = "native")
private IAudioCompletion PrepareModel()
{
var whisperService = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == modelName.ToLower());
if (whisperService == null)
{
throw new Exception($"Can't resolve speech2text provider by {modelName}");
}
return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1");
}
if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase))
{
return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1");
}
await whisperService.SetModelName("Tiny");
return whisperService;
private bool ParseAudioFileType(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(extension, out _) || provider.TryGetContentType(fileName, out _);
return canParse;
}
}

10
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs
View file

@ -1,10 +0,0 @@
namespace BotSharp.Plugin.AudioHandler.Functions
{
public interface IAudioProcessUtilities
{
Stream ConvertMp3ToStream(string mp3FileName);
Stream ConvertWavToStream(string wavFileName);
Stream ConvertToStream(string fileName);
}
}

85
src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs Normal file
View file

@ -0,0 +1,85 @@
using NAudio.Wave;
using NAudio.Wave.SampleProviders;
namespace BotSharp.Plugin.AudioHandler.Helpers;
public static class AudioHelper
{
private const int DEFAULT_SAMPLE_RATE = 16000;
public static Stream ConvertToStream(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException("fileName is Null when converting to stream in audio processor");
}
var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse(fileExtension, out AudioType fileType))
{
throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!");
}
var stream = fileType switch
{
AudioType.mp3 => ConvertMp3ToStream(fileName),
_ => ConvertWavToStream(fileName)
};
return stream;
}
public static Stream Transform(Stream stream, string fileName)
{
var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse(fileExtension, out AudioType fileType))
{
throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!");
}
Stream resultStream = new MemoryStream();
stream.CopyTo(resultStream);
resultStream.Seek(0, SeekOrigin.Begin);
WaveStream reader = fileType switch
{
AudioType.mp3 => new Mp3FileReader(resultStream),
_ => new WaveFileReader(resultStream)
};
resultStream = ChangeSampleRate(reader);
reader.Close();
return resultStream;
}
private static Stream ConvertMp3ToStream(string fileName)
{
using var fileStream = File.OpenRead(fileName);
using var reader = new Mp3FileReader(fileStream);
return ChangeSampleRate(reader);
}
private static Stream ConvertWavToStream(string fileName)
{
using var fileStream = File.OpenRead(fileName);
using var reader = new WaveFileReader(fileStream);
return ChangeSampleRate(reader);
}
private static Stream ChangeSampleRate(WaveStream ws)
{
var ms = new MemoryStream();
if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE)
{
var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE);
WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16());
}
else
{
ws.CopyTo(ms);
}
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
}

95
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs
View file

@ -1,73 +1,60 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
namespace BotSharp.Plugin.AudioHandler.Hooks
namespace BotSharp.Plugin.AudioHandler.Hooks;
public class AudioHandlerHook : AgentHookBase, IAgentHook
{
public class AudioHandlerHook : AgentHookBase, IAgentHook
private const string HANDLER_AUDIO = "handle_audio_request";
public override string SelfId => string.Empty;
public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
{
private const string HANDLER_AUDIO = "handle_audio_request";
}
public override string SelfId => string.Empty;
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
if (isEnabled && isConvMode)
{
AddUtility(agent, HANDLER_AUDIO);
}
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
base.OnAgentLoaded(agent);
}
if (isEnabled && isConvMode)
private void AddUtility(Agent agent, string functionName)
{
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
base.OnAgentLoaded(agent);
}
private void AddUtility(Agent agent, string utility, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
if (agent.Functions == null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
if (agent.Functions == null)
{
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}

7
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs
View file

@ -1,10 +1,3 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Agents;
namespace BotSharp.Plugin.AudioHandler.Hooks;
public class AudioHandlerUtilityHook : IAgentUtilityHook

5
src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs
View file

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.LlmContexts;

5
src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs
View file

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.LlmContexts;

22
src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs
View file

@ -1,19 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Whisper.net;
namespace BotSharp.Plugin.AudioHandler.Models
{
public class AudioOutput
{
public List<SegmentData> Segments { get; set; }
namespace BotSharp.Plugin.AudioHandler.Models;
public override string ToString()
{
return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty;
}
public class AudioOutput
{
public List<SegmentData> Segments { get; set; } = new();
public override string ToString()
{
return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty;
}
}

135
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Core.Agents.Services;
using Whisper.net;
using Whisper.net.Ggml;
@ -7,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
/// <summary>
/// Native Whisper provider for speech to text conversion
/// </summary>
public class NativeWhisperProvider : ISpeechToText
public class NativeWhisperProvider : IAudioCompletion
{
public string Provider => "native";
private readonly IAudioProcessUtilities _audioProcessUtilities;
private static WhisperProcessor _processor;
private readonly ILogger _logger;
private static WhisperProcessor _whisperProcessor;
private string MODEL_DIR = "model";
private string? _currentModelPath;
private Dictionary<GgmlType, string> _modelPathDict = new Dictionary<GgmlType, string>();
private GgmlType? _modelType;
private readonly IServiceProvider _services;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<NativeWhisperProvider> _logger;
public string Provider => "native";
public NativeWhisperProvider(
IAudioProcessUtilities audioProcessUtilities,
BotSharpDatabaseSettings dbSettings,
IFileStorageService fileStorage,
IServiceProvider services,
ILogger<NativeWhisperProvider> logger)
{
_audioProcessUtilities = audioProcessUtilities;
_fileStorage = fileStorage;
_services = services;
_logger = logger;
}
public async Task<string> GenerateTextFromAudioAsync(string filePath)
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
{
string fileExtension = Path.GetExtension(filePath);
if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
using var stream = _audioProcessUtilities.ConvertToStream(filePath);
if (stream == null)
{
throw new Exception($"Failed to convert {fileExtension} to stream");
}
var textResult = new List<SegmentData>();
await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false))
using var stream = AudioHelper.Transform(audio, audioFileName);
await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false))
{
textResult.Add(result);
}
_processor.Dispose();
_whisperProcessor.Dispose();
stream.Close();
var audioOutput = new AudioOutput
{
@ -57,71 +46,69 @@ public class NativeWhisperProvider : ISpeechToText
};
return audioOutput.ToString();
}
private async Task LoadWhisperModel(GgmlType modelType)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
{
throw new NotImplementedException();
}
public void SetModelName(string model)
{
if (Enum.TryParse(model, true, out GgmlType ggmlType))
{
LoadWhisperModel(ggmlType);
}
else
{
_logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!");
LoadWhisperModel(GgmlType.Tiny);
}
}
private void LoadWhisperModel(GgmlType modelType)
{
try
{
if (!Directory.Exists(MODEL_DIR))
Directory.CreateDirectory(MODEL_DIR);
var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin")
.ToArray();
if (!availableModelPaths.Any())
var modelDir = _fileStorage.BuildDirectory("models", "whisper");
var exist = _fileStorage.ExistDirectory(modelDir);
if (!exist)
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin");
if (modelFilePath == null)
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
_currentModelPath = modelFilePath;
}
_fileStorage.CreateDirectory(modelDir);
}
_processor = WhisperFactory
.FromPath(path: _currentModelPath)
.CreateBuilder()
.WithLanguage("auto")
.Build();
var files = _fileStorage.GetFiles("models/whisper", "*.bin");
var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType));
if (string.IsNullOrEmpty(modelLoc))
{
modelLoc = BuildModelPath(modelType);
DownloadModel(modelType, modelLoc);
}
_modelType = modelType;
var bytes = _fileStorage.GetFileBytes(modelLoc);
_whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build();
}
catch (Exception ex)
{
throw new Exception($"Failed to load whisper model: {ex.Message}");
var error = "Failed to load whisper model";
_logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}");
throw new Exception($"{error}: {ex.Message}");
}
}
private async Task DownloadModel(GgmlType modelType, string modelDir)
private void DownloadModel(GgmlType modelType, string modelDir)
{
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType);
using var fileWriter = File.OpenWrite(modelDir);
await modelStream.CopyToAsync(fileWriter);
using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult();
_fileStorage.SaveFileStreamToPath(modelDir, modelStream);
modelStream.Close();
}
private string SetModelPath(string rootPath, GgmlType modelType)
private string BuildModelPath(GgmlType modelType)
{
string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin");
return currentModelPath;
return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType));
}
public async Task SetModelName(string modelType)
private string BuildModelFile(GgmlType modelType)
{
if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
{
await LoadWhisperModel(ggmlType);
return;
}
_logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!");
await LoadWhisperModel(GgmlType.Tiny);
return $"ggml-{modelType}.bin";
}
}

7
src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
View file

@ -1,6 +1,5 @@
namespace BotSharp.Plugin.AudioHandler.Settings
namespace BotSharp.Plugin.AudioHandler.Settings;
public class AudioHandlerSettings
{
public class AudioHandlerSettings
{
}
}

3
src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
View file

@ -5,6 +5,7 @@ global using System.Linq;
global using System.Text.Json;
global using System.Threading.Tasks;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;
@ -20,7 +21,7 @@ global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.AudioHandler.Enums;
global using BotSharp.Plugin.AudioHandler.Functions;
global using BotSharp.Plugin.AudioHandler.Helpers;
global using BotSharp.Plugin.AudioHandler.Hooks;
global using BotSharp.Plugin.AudioHandler.Models;
global using BotSharp.Plugin.AudioHandler.LlmContexts;

32
src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json
View file

@ -1,18 +1,18 @@
{
"name": "handle_audio_request",
"description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
},
"is_need_summary": {
"type": "boolean",
"description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
}
},
"required": [ "user_request" ]
}
"name": "handle_audio_request",
"description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
},
"is_need_summary": {
"type": "boolean",
"description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
}
},
"required": [ "user_request" ]
}
}

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.AzureOpenAI.Providers.Audio;
using BotSharp.Plugin.AzureOpenAI.Providers.Chat;
using BotSharp.Plugin.AzureOpenAI.Providers.Embedding;
using BotSharp.Plugin.AzureOpenAI.Providers.Image;
@ -30,5 +31,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
}
}

90
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs Normal file
View file

@ -0,0 +1,90 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareTranscriptionOptions(text);
var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options);
return result.Value.Text;
}
private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
{
var state = _services.GetRequiredService<IConversationStateService>();
var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format"));
var granularity = GetGranularity(state.GetState("audio_granularity"));
var temperature = GetTemperature(state.GetState("audio_temperature"));
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
Granularities = granularity,
Temperature = temperature,
Prompt = text
};
return options;
}
private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "verbose";
AudioTranscriptionFormat format;
switch (value)
{
case "json":
format = AudioTranscriptionFormat.Simple;
break;
case "srt":
format = AudioTranscriptionFormat.Srt;
break;
case "vtt":
format = AudioTranscriptionFormat.Vtt;
break;
default:
format = AudioTranscriptionFormat.Verbose;
break;
}
return format;
}
private AudioTimestampGranularities GetGranularity(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "default";
AudioTimestampGranularities granularity;
switch (value)
{
case "word":
granularity = AudioTimestampGranularities.Word;
break;
case "segment":
granularity = AudioTimestampGranularities.Segment;
break;
default:
granularity = AudioTimestampGranularities.Default;
break;
}
return granularity;
}
private float? GetTemperature(string input)
{
if (!float.TryParse(input, out var temperature))
{
return null;
}
return temperature;
}
}

102
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs Normal file
View file

@ -0,0 +1,102 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
Speed = speed
};
return (voice, options);
}
private GeneratedSpeechVoice GetVoice(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "alloy";
GeneratedSpeechVoice voice;
switch (value)
{
case "echo":
voice = GeneratedSpeechVoice.Echo;
break;
case "fable":
voice = GeneratedSpeechVoice.Fable;
break;
case "onyx":
voice = GeneratedSpeechVoice.Onyx;
break;
case "nova":
voice = GeneratedSpeechVoice.Nova;
break;
case "shimmer":
voice = GeneratedSpeechVoice.Shimmer;
break;
default:
voice = GeneratedSpeechVoice.Alloy;
break;
}
return voice;
}
private GeneratedSpeechFormat GetSpeechFormat(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "mp3";
GeneratedSpeechFormat format;
switch (value)
{
case "wav":
format = GeneratedSpeechFormat.Wav;
break;
case "opus":
format = GeneratedSpeechFormat.Opus;
break;
case "aac":
format = GeneratedSpeechFormat.Aac;
break;
case "flac":
format = GeneratedSpeechFormat.Flac;
break;
case "pcm":
format = GeneratedSpeechFormat.Pcm;
break;
default:
format = GeneratedSpeechFormat.Mp3;
break;
}
return format;
}
private float? GetSpeed(string input)
{
if (!float.TryParse(input, out var speed))
{
return null;
}
return speed;
}
}

19
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs Normal file
View file

@ -0,0 +1,19 @@
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion
{
private readonly IServiceProvider _services;
public string Provider => "openai";
private string _model;
public AudioCompletionProvider(IServiceProvider service)
{
_services = service;
}
public void SetModelName(string model)
{
_model = model;
}
}

6
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = value.FunctionCall.FunctionName,
FunctionArgs = value.FunctionCall.FunctionArguments
};
@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments
};
@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion
responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
};
}

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs
View file

@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding
protected readonly IServiceProvider _services;
protected readonly ILogger<TextEmbeddingProvider> _logger;
private const int DEFAULT_DIMENSION = 3072;
private const int DEFAULT_DIMENSION = 1536;
protected string _model;
protected int _dimension;

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs
View file

@ -53,7 +53,7 @@ public partial class ImageCompletionProvider
var state = _services.GetRequiredService<IConversationStateService>();
var size = GetImageSize(state.GetState("image_size"));
var format = GetImageFormat(state.GetState("image_format"));
var format = GetImageFormat(state.GetState("image_response_format"));
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageEditOptions

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs
View file

@ -33,7 +33,7 @@ public partial class ImageCompletionProvider
var size = GetImageSize(state.GetState("image_size"));
var quality = GetImageQuality(state.GetState("image_quality"));
var style = GetImageStyle(state.GetState("image_style"));
var format = GetImageFormat(state.GetState("image_format"));
var format = GetImageFormat(state.GetState("image_response_format"));
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageGenerationOptions

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs
View file

@ -29,7 +29,7 @@ public partial class ImageCompletionProvider
{
var state = _services.GetRequiredService<IConversationStateService>();
var size = GetImageSize(state.GetState("image_size"));
var format = GetImageFormat(state.GetState("image_format"));
var format = GetImageFormat(state.GetState("image_response_format"));
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageVariationOptions

12
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs
View file

@ -72,9 +72,6 @@ public partial class ImageCompletionProvider : IImageCompletion
case "512x512":
retSize = GeneratedImageSize.W512xH512;
break;
case "1024x1024":
retSize = GeneratedImageSize.W1024xH1024;
break;
case "1024x1792":
retSize = GeneratedImageSize.W1024xH1792;
break;
@ -96,9 +93,6 @@ public partial class ImageCompletionProvider : IImageCompletion
GeneratedImageQuality retQuality;
switch (value)
{
case "standard":
retQuality = GeneratedImageQuality.Standard;
break;
case "hd":
retQuality = GeneratedImageQuality.High;
break;
@ -117,9 +111,6 @@ public partial class ImageCompletionProvider : IImageCompletion
GeneratedImageStyle retStyle;
switch (value)
{
case "natural":
retStyle = GeneratedImageStyle.Natural;
break;
case "vivid":
retStyle = GeneratedImageStyle.Vivid;
break;
@ -138,9 +129,6 @@ public partial class ImageCompletionProvider : IImageCompletion
GeneratedImageFormat retFormat;
switch (value)
{
case "uri":
retFormat = GeneratedImageFormat.Uri;
break;
case "bytes":
retFormat = GeneratedImageFormat.Bytes;
break;

3
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -84,11 +84,12 @@ public class HandleEmailSenderFn : IFunctionCallback
{
if (files.IsNullOrEmpty()) return;
var fileStorage = _services.GetRequiredService<IFileStorageService>();
foreach (var file in files)
{
if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl);
builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType));
Thread.Sleep(100);

7
src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs
View file

@ -1,14 +1,7 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Plugin.EmailHandler.Enums;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.EmailHandler.Hooks;

7
src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs
View file

@ -1,14 +1,7 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Plugin.EmailHandler.Enums;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.EmailHandler.Hooks;

6
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -43,7 +43,7 @@ public class EditImageFn : IFunctionCallback
private void SetImageOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("image_format", "bytes");
state.SetState("image_response_format", "bytes");
state.SetState("image_count", "1");
}
@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback
{
if (image == null) return;
var files = new List<BotSharpFile>()
var files = new List<InputFileModel>()
{
new BotSharpFile
new InputFileModel
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"

Some files were not shown because too many files have changed in this diff Show more