Merge pull request #633 from iceljc/features/add-knowledge-docs

Features/add knowledge docs

src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs

@@ -37,7 +37,7 @@ public interface IFileStorageService
     IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
     string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
     IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
-    bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files);
+    bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files);
 
     /// <summary>
     /// Delete files under messages
@@ -54,11 +54,31 @@ public interface IFileStorageService
 
     #region User
     string GetUserAvatar();
-    bool SaveUserAvatar(InputFileModel file);
+    bool SaveUserAvatar(FileDataModel file);
     #endregion
 
     #region Speech
     bool SaveSpeechFile(string conversationId, string fileName, BinaryData data);
     BinaryData GetSpeechFile(string conversationId, string fileName);
     #endregion
+
+    #region Knowledge
+    bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream);
+
+    /// <summary>
+    /// Delete files in a knowledge collection. If fileId is null, remove all files in the collection.
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileId"></param>
+    /// <returns></returns>
+    bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
+
+    bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData);
+
+    KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId);
+
+    IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider);
+
+    FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId);
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class ExternalFileModel : FileDataModel
+{
+    [JsonPropertyName("file_url")]
+    public string FileUrl { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBinaryDataModel.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class FileBinaryDataModel
+{
+    public string FileName { get; set; }
+    public BinaryData FileBinaryData { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileDataModel.cs

@@ -1,6 +1,6 @@
 namespace BotSharp.Abstraction.Files.Models;
 
-public class InputFileModel : FileBase
+public class FileDataModel : FileBase
 {
     /// <summary>
     /// File name with extension

src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class KnowledgeFileModel
+{
+    public string FileId { get; set; }
+    public string FileName { get; set; }
+    public string FileExtension { get; set; }
+    public string ContentType { get; set; }
+    public string FileUrl { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Knowledges.Enums;
+
+public static class KnowledgeCollectionType
+{
+    public static string QuestionAnswer = "question-answer";
+    public static string Document = "document";
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -6,11 +6,10 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     #region Vector
-    Task<bool> CreateVectorCollection(string collectionName, int dimension);
+    Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
     Task<bool> DeleteVectorCollection(string collectionName);
-    Task<IEnumerable<string>> GetVectorCollections();
+    Task<IEnumerable<string>> GetVectorCollections(string type);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
-    Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
@@ -19,6 +18,16 @@ public interface IKnowledgeService
 
     #region Graph
     Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options);
-    Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions);
+    #endregion
+
+    #region Document
+    Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
+    Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
+    Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName);
+    Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId);
+    #endregion
+
+    #region Common
+    Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs

@@ -0,0 +1,27 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeDocMetaData
+{
+    [JsonPropertyName("collection")]
+    public string Collection { get; set; }
+
+    [JsonPropertyName("file_id")]
+    public string FileId { get; set; }
+
+    [JsonPropertyName("file_name")]
+    public string FileName { get; set; }
+
+    [JsonPropertyName("content_type")]
+    public string ContentType { get; set; }
+
+    [JsonPropertyName("vector_data_ids")]
+    public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
+
+    [JsonPropertyName("create_date")]
+    public DateTime CreateDate { get; set; } = DateTime.UtcNow;
+
+    [JsonPropertyName("create_user_id")]
+    public string CreateUserId { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs

@@ -1,10 +0,0 @@
-using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.VectorStorage.Models;
-
-namespace BotSharp.Abstraction.Knowledges.Models;
-
-public class KnowledgeSearchResult
-{
-    public IEnumerable<VectorSearchResult> VectorResult { get; set; }
-    public GraphSearchResult GraphResult { get; set; }
-}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class UploadKnowledgeResponse
+{
+    [JsonPropertyName("success")]
+    public IEnumerable<string> Success { get; set; } = new List<string>();
+
+    [JsonPropertyName("failed")]
+    public IEnumerable<string> Failed { get; set; } = new List<string>();
+
+    [JsonPropertyName("is_success")]
+    public bool IsSuccess {
+        get
+        {
+            return !Success.IsNullOrEmpty() && Failed.IsNullOrEmpty();
+        }
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs

@@ -8,7 +8,6 @@ public class KnowledgeBaseSettings
     public SettingBase GraphDb { get; set; }
 
     public DefaultKnowledgeBaseSetting Default { get; set; }
-    public List<VectorCollectionSetting> Collections { get; set; } = new();
 }
 
 public class DefaultKnowledgeBaseSetting
@@ -17,12 +16,6 @@ public class DefaultKnowledgeBaseSetting
     public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
 }
 
-public class VectorCollectionSetting
-{
-    public string Name { get; set; }
-    public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
-}
-
 public class KnowledgeTextEmbeddingSetting : SettingBase
 {
     public string Model { get; set; }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -4,6 +4,7 @@ using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
+using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Abstraction.Repositories;
 
@@ -99,4 +100,16 @@ public interface IBotSharpRepository
     bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs);
 
     #endregion
+
+    #region KnowledgeBase
+    /// <summary>
+    /// Save knowledge collection configs. If reset is true, it will remove everything and then save the new configs.
+    /// </summary>
+    /// <param name="configs"></param>
+    /// <param name="reset"></param>
+    /// <returns></returns>
+    bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
+    bool DeleteKnowledgeCollectionConfig(string collectionName);
+    IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs

@@ -44,6 +44,13 @@ public static class StringExtensions
         return str1.Equals(str2, option);
     }
 
+    public static string CleanStr(this string? str)
+    {
+        if (string.IsNullOrWhiteSpace(str)) return string.Empty;
+
+        return str.Replace(" ", "").Replace("\t", "").Replace("\n", "").Replace("\r", "");
+    }
+
     public static string JsonContent(this string text)
     {
         var m = Regex.Match(text, @"\{(?:[^{}]|(?<open>\{)|(?<-open>\}))+(?(open)(?!))\}");

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -13,5 +13,5 @@ public interface IVectorDb
     Task<bool> DeleteCollection(string collectionName);
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
-    Task<bool> DeleteCollectionData(string collectionName, Guid id);
+    Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs

@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionConfigFilter
+{
+    public IEnumerable<string>? CollectionNames { get; set; }
+    public IEnumerable<string>? CollectionTypes { get; set; }
+    public IEnumerable<string>? VectorStroageProviders { get; set; }
+
+    public static VectorCollectionConfigFilter Empty()
+    {
+        return new VectorCollectionConfigFilter();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs

@@ -0,0 +1,46 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionConfigsModel
+{
+    [JsonPropertyName("collections")]
+    public List<VectorCollectionConfig> Collections { get; set; } = new();
+}
+
+public class VectorCollectionConfig
+{
+    /// <summary>
+    /// Must be unique
+    /// </summary>
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    /// <summary>
+    /// Collection type, e.g., question-answer, document
+    /// </summary>
+    [JsonPropertyName("type")]
+    public string Type { get; set; }
+
+    [JsonPropertyName("vector_storage")]
+    public VectorStorageConfig VectorStorage { get; set; }
+
+    [JsonPropertyName("text_embedding")]
+    public KnowledgeEmbeddingConfig TextEmbedding { get; set; }
+}
+
+public class KnowledgeEmbeddingConfig
+{
+    [JsonPropertyName("provider")]
+    public string Provider { get; set; }
+
+    [JsonPropertyName("model")]
+    public string Model { get; set; }
+
+    [JsonPropertyName("dimension")]
+    public int Dimension { get; set; }
+}
+
+public class VectorStorageConfig
+{
+    [JsonPropertyName("provider")]
+    public string Provider { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -5,6 +5,9 @@ public class VectorFilter : StringIdPagination
     [JsonPropertyName("with_vector")]
     public bool WithVector { get; set; }
 
+    /// <summary>
+    /// For keyword search
+    /// </summary>
     [JsonPropertyName("search_pairs")]
     public IEnumerable<KeyValue>? SearchPairs { get; set; }
 }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs

@@ -118,7 +118,7 @@ public partial class LocalFileStorageService
         return foundMsgs;
     }
 
-    public bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files)
+    public bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files)
     {
         if (files.IsNullOrEmpty()) return false;
 

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs

@@ -0,0 +1,191 @@
+using BotSharp.Abstraction.Knowledges.Models;
+using System.IO;
+
+namespace BotSharp.Core.Files.Services;
+
+public partial class LocalFileStorageService
+{
+    public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileId))
+        {
+            return false;
+        }
+
+        try
+        {
+            var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+            var dir = Path.Combine(docDir, fileId);
+            if (ExistDirectory(dir))
+            {
+                Directory.Delete(dir);
+            }
+            Directory.CreateDirectory(dir);
+
+            var filePath = Path.Combine(dir, fileName);
+            using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write);
+            stream.CopyTo(fs);
+            fs.Flush();
+            fs.Close();
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving knowledge file " +
+                $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
+                $"\r\n{ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+    public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+        {
+            return false;
+        }
+
+        var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        if (!ExistDirectory(dir)) return false;
+
+        if (string.IsNullOrEmpty(fileId))
+        {
+            Directory.Delete(dir, true);
+        }
+        else
+        {
+            var fileDir = Path.Combine(dir, fileId);
+            if (ExistDirectory(fileDir))
+            {
+                Directory.Delete(fileDir, true);
+            }
+        }
+
+        return true;
+    }
+
+    public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileId))
+        {
+            return false;
+        }
+
+        var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        var dir = Path.Combine(docDir, fileId);
+        if (!ExistDirectory(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
+
+        var metaFile = Path.Combine(dir, KNOWLEDGE_DOC_META_FILE);
+        var content = JsonSerializer.Serialize(metaData, _jsonOptions);
+        File.WriteAllText(metaFile, content);
+        return true;
+    }
+
+    public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileId))
+        {
+            return null;
+        }
+
+        var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        var metaFile = Path.Combine(docDir, fileId, KNOWLEDGE_DOC_META_FILE);
+        if (!File.Exists(metaFile))
+        {
+            return null;
+        }
+
+        var content = File.ReadAllText(metaFile);
+        var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
+        return metaData;
+    }
+
+    public IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+             || string.IsNullOrWhiteSpace(vectorStoreProvider))
+        {
+            return Enumerable.Empty<KnowledgeFileModel>();
+        }
+
+        var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        if (!ExistDirectory(docDir))
+        {
+            return Enumerable.Empty<KnowledgeFileModel>();
+        }
+
+        var files = new List<KnowledgeFileModel>();
+        foreach (var folder in Directory.GetDirectories(docDir))
+        {
+            var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE);
+            if (!File.Exists(metaFile)) continue;
+
+            var content = File.ReadAllText(metaFile);
+            var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
+            if (metaData == null) continue;
+
+            var fileName = Path.GetFileNameWithoutExtension(metaData.FileName);
+            var fileExtension = Path.GetExtension(metaData.FileName);
+
+            files.Add(new KnowledgeFileModel
+            {
+                FileId = metaData.FileId,
+                FileName = metaData.FileName,
+                FileExtension = fileExtension.Substring(1),
+                ContentType = FileUtility.GetFileContentType(metaData.FileName),
+                FileUrl = BuildKnowledgeFileUrl(collectionName, metaData.FileId)
+            });
+        }
+
+        return files;
+    }
+
+    public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileId))
+        {
+            return null;
+        }
+
+        var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        var fileDir = Path.Combine(docDir, fileId);
+        if (!ExistDirectory(fileDir)) return null;
+
+        var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE);
+        var content = File.ReadAllText(metaFile);
+        var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
+        using var stream = new FileStream(fileDir, FileMode.Open, FileAccess.Read);
+        stream.Position = 0;
+
+        return new FileBinaryDataModel
+        {
+            FileName = metaData.FileName,
+            FileBinaryData = BinaryData.FromStream(stream)
+        };
+    }
+
+
+    #region Private methods
+    private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
+    {
+        return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
+    }
+
+    private string BuildKnowledgeFileUrl(string collectionName, string fileId)
+    {
+        return $"/knowledge/file/{collectionName}/file/{fileId}";
+    }
+    #endregion
+}

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs

@@ -16,7 +16,7 @@ public partial class LocalFileStorageService
         return found;
     }
 
-    public bool SaveUserAvatar(InputFileModel file)
+    public bool SaveUserAvatar(FileDataModel file)
     {
         if (file == null || string.IsNullOrEmpty(file.FileData)) return false;
 

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs

@@ -19,6 +19,17 @@ public partial class LocalFileStorageService : IFileStorageService
     private const string USER_AVATAR_FOLDER = "avatar";
     private const string SESSION_FOLDER = "sessions";
     private const string TEXT_TO_SPEECH_FOLDER = "speeches";
+    private const string KNOWLEDGE_FOLDER = "knowledgebase";
+    private const string KNOWLEDGE_DOC_FOLDER = "document";
+    private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
+
+    private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
+    {
+        PropertyNameCaseInsensitive = true,
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        WriteIndented = true,
+        AllowTrailingCommas = true
+    };
 
     public LocalFileStorageService(
         BotSharpDatabaseSettings dbSettings,

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -3,6 +3,7 @@ using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
+using BotSharp.Abstraction.VectorStorage.Models;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
 namespace BotSharp.Core.Repository;
@@ -232,4 +233,15 @@ public class BotSharpDbContext : Database, IBotSharpRepository
     public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
         throw new NotImplementedException();
     #endregion
+
+    #region Knowledge
+    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
+        throw new NotImplementedException();
+
+    public bool DeleteKnowledgeCollectionConfig(string collectionName) =>
+        throw new NotImplementedException();
+
+    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) =>
+        throw new NotImplementedException();
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -0,0 +1,112 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+using System.IO;
+
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
+{
+    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
+    {
+        var vectorDir = BuildKnowledgeCollectionConfigDir();
+        if (!Directory.Exists(vectorDir))
+        {
+            Directory.CreateDirectory(vectorDir);
+        }
+
+        var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE);
+        if (reset)
+        {
+            File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options));
+            return true;
+        }
+
+        if (!File.Exists(configFile))
+        {
+            File.Create(configFile);
+        }
+
+        var str = File.ReadAllText(configFile);
+        var savedConfigs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
+
+        // Update if collection already exists, otherwise insert
+        foreach (var config in configs)
+        {
+            if (string.IsNullOrWhiteSpace(config.Name)) continue;
+
+            var found = savedConfigs.FirstOrDefault(x => x.Name == config.Name);
+            if (found != null)
+            {
+                found.TextEmbedding = config.TextEmbedding;
+                found.Type = config.Type;
+            }
+            else
+            {
+                savedConfigs.Add(config);
+            }
+        }
+
+        File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options));
+
+        return true;
+    }
+
+    public bool DeleteKnowledgeCollectionConfig(string collectionName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)) return false;
+
+        var vectorDir = BuildKnowledgeCollectionConfigDir();
+        var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE);
+        if (!File.Exists(configFile)) return false;
+
+        var str = File.ReadAllText(configFile);
+        var savedConfigs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
+        savedConfigs = savedConfigs.Where(x => x.Name != collectionName).ToList();
+        File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options));
+
+        return true;
+    }
+
+    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter)
+    {
+        if (filter == null)
+        {
+            return Enumerable.Empty<VectorCollectionConfig>();
+        }
+
+        var vectorDir = BuildKnowledgeCollectionConfigDir();
+        var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE);
+        if (!File.Exists(configFile))
+        {
+            return Enumerable.Empty<VectorCollectionConfig>();
+        }
+
+        // Get data
+        var content = File.ReadAllText(configFile);
+        var configs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(content, _options) ?? new();
+
+        // Apply filters
+        if (!filter.CollectionNames.IsNullOrEmpty())
+        {
+            configs = configs.Where(x => filter.CollectionNames.Contains(x.Name)).ToList();
+        }
+
+        if (!filter.CollectionTypes.IsNullOrEmpty())
+        {
+            configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList();
+        }
+
+        if (!filter.VectorStroageProviders.IsNullOrEmpty())
+        {
+            configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStorage?.Provider)).ToList();
+        }
+
+        return configs;
+    }
+
+    #region Private methods
+    private string BuildKnowledgeCollectionConfigDir()
+    {
+        return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
+    }
+    #endregion
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -40,6 +40,9 @@ public partial class FileRepository : IBotSharpRepository
     private const string AGENT_RESPONSES_FOLDER = "responses";
     private const string AGENT_TASKS_FOLDER = "tasks";
     private const string USERS_FOLDER = "users";
+    private const string KNOWLEDGE_FOLDER = "knowledgebase";
+    private const string VECTOR_FOLDER = "vector";
+    private const string COLLECTION_CONFIG_FILE = "collection-config.json";
 
     public FileRepository(
         IServiceProvider services,

src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs

@@ -62,7 +62,11 @@ namespace BotSharp.OpenAPI.BackgroundServices
             if (cleanSetting == null || !cleanSetting.Enable) return;
 
             var conversationService = scope.ServiceProvider.GetRequiredService<IConversationService>();
-            var conversationIds = await conversationService.GetIdleConversations(cleanSetting.BatchSize, cleanSetting.MessageLimit, cleanSetting.BufferHours, cleanSetting.ExcludeAgentIds);
+            var batchSize = cleanSetting.BatchSize;
+            var limit = cleanSetting.MessageLimit;
+            var bufferHours = cleanSetting.BufferHours;
+            var excludeAgentIds = cleanSetting.ExcludeAgentIds ?? new List<string>();
+            var conversationIds = await conversationService.GetIdleConversations(batchSize, limit, bufferHours, excludeAgentIds);
 
             if (!conversationIds.IsNullOrEmpty())
             {

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -366,8 +366,7 @@ public class ConversationController : ControllerBase
 
     #region Files and attachments
     [HttpPost("/conversation/{conversationId}/attachments")]
-    public IActionResult UploadAttachments([FromRoute] string conversationId,
-        IFormFile[] files)
+    public IActionResult UploadAttachments([FromRoute] string conversationId, IFormFile[] files)
     {
         if (files != null && files.Length > 0)
         {

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Files.Constants;
 using BotSharp.Abstraction.Graph.Models;
 using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
@@ -20,19 +21,19 @@ public class KnowledgeBaseController : ControllerBase
 
     #region Vector
     [HttpGet("knowledge/vector/collections")]
-    public async Task<IEnumerable<string>> GetVectorCollections()
+    public async Task<IEnumerable<string>> GetVectorCollections([FromQuery] string type)
     {
-        return await _knowledgeService.GetVectorCollections();
+        return await _knowledgeService.GetVectorCollections(type);
     }
 
-    [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")]
-    public async Task<bool> CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension)
+    [HttpPost("knowledge/vector/create-collection")]
+    public async Task<bool> CreateVectorCollection([FromBody] CreateVectorCollectionRequest request)
     {
-        return await _knowledgeService.CreateVectorCollection(collection, dimension);
+        return await _knowledgeService.CreateVectorCollection(request.CollectionName, request.CollectionType, request.Dimension, request.Provider, request.Model);
     }
 
     [HttpDelete("knowledge/vector/{collection}/delete-collection")]
-    public async Task<bool> GetVectorCollections([FromRoute] string collection)
+    public async Task<bool> DeleteVectorCollection([FromRoute] string collection)
     {
         return await _knowledgeService.DeleteVectorCollection(collection);
     }
@@ -99,29 +100,6 @@ public class KnowledgeBaseController : ControllerBase
     {
         return await _knowledgeService.DeleteVectorCollectionData(collection, id);
     }
-
-    [HttpPost("/knowledge/vector/{collection}/upload")]
-    public async Task<IActionResult> UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum)
-    {
-        var setttings = _services.GetRequiredService<FileCoreSettings>();
-        var textConverter = _services.GetServices<IPdf2TextConverter>().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider);
-
-        var filePath = Path.GetTempFileName();
-        using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None))
-        {
-            await file.CopyToAsync(stream);
-            await stream.FlushAsync();
-        }
-
-        var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum);
-        await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel
-        {
-            Content = content
-        });
-
-        System.IO.File.Delete(filePath);
-        return Ok(new { count = 1, file.Length });
-    }
     #endregion
 
 
@@ -143,29 +121,56 @@ public class KnowledgeBaseController : ControllerBase
     #endregion
 
 
-    #region Knowledge
-    [HttpPost("/knowledge/search")]
-    public async Task<KnowledgeSearchViewModel> SearchKnowledge([FromBody] SearchKnowledgeRequest request)
+    #region Document
+    [HttpPost("/knowledge/document/{collection}/upload")]
+    public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
     {
-        var vectorOptions = new VectorSearchOptions
-        {
-            Fields = request.VectorParams.Fields,
-            Limit = request.VectorParams.Limit ?? 5,
-            Confidence = request.VectorParams.Confidence ?? 0.5f,
-            WithVector = request.VectorParams.WithVector
-        };
+        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files);
+        return response;
+    }
 
-        var graphOptions = new GraphSearchOptions
-        {
-            Method = request.GraphParams.Method
-        };
+    [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
+    public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
+    {
+        var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
+        return response;
+    }
 
-        var result = await _knowledgeService.SearchKnowledge(request.Text, request.VectorParams.Collection, vectorOptions, graphOptions);
-        return new KnowledgeSearchViewModel
+    [HttpGet("/knowledge/document/{collection}/list")]
+    public async Task<IEnumerable<KnowledgeFileViewModel>> GetKnowledgeDocuments([FromRoute] string collection)
+    {
+        var files = await _knowledgeService.GetKnowledgeDocuments(collection);
+        return files.Select(x => KnowledgeFileViewModel.From(x));
+    }
+
+    [HttpGet("/knowledge/document/{collection}/file/{fileId}")]
+    public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
+    {
+        var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId);
+        return BuildFileResult(file);
+    }
+    #endregion
+
+
+    #region Common
+    [HttpPost("/knowledge/vector/refresh-configs")]
+    public async Task<string> RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request)
+    {
+        var saved = await _knowledgeService.RefreshVectorKnowledgeConfigs(request);
+        return saved ? "Success" : "Fail";
+    }
+    #endregion
+
+
+    #region Private methods
+    private FileContentResult BuildFileResult(FileBinaryDataModel? file)
+    {
+        if (file == null)
         {
-            VectorResult = result?.VectorResult?.Select(x => VectorKnowledgeViewModel.From(x)),
-            GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null
-        };
+            return File(new byte[0], "application/octet-stream", "error.txt");
+        }
+
+        return File(file.FileBinaryData.ToArray(), "application/octet-stream", file.FileName);
     }
     #endregion
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs

@@ -138,7 +138,7 @@ public class UserController : ControllerBase
     public bool UploadUserAvatar([FromBody] UserAvatarModel input)
     {
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
-        var file = new InputFileModel
+        var file = new FileDataModel
         {
             FileName = input.FileName,
             FileData = input.FileData,

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs

@@ -3,5 +3,5 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations;
 public class InputMessageFiles
 {
     public List<MessageState> States { get; set; } = new();
-    public List<InputFileModel> Files { get; set; } = new();
+    public List<FileDataModel> Files { get; set; } = new();
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs

@@ -0,0 +1,21 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class CreateVectorCollectionRequest
+{
+    [JsonPropertyName("collection_name")]
+    public string CollectionName { get; set; }
+
+    [JsonPropertyName("collection_type")]
+    public string CollectionType { get; set; }
+
+    [JsonPropertyName("provider")]
+    public string Provider { get; set; }
+
+    [JsonPropertyName("model")]
+    public string Model { get; set; }
+
+    [JsonPropertyName("dimension")]
+    public int Dimension { get; set; }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs

@@ -0,0 +1,33 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class KnowledgeFileViewModel
+{
+    [JsonPropertyName("file_id")]
+    public string FileId { get; set; }
+
+    [JsonPropertyName("file_name")]
+    public string FileName { get; set; }
+
+    [JsonPropertyName("file_extension")]
+    public string FileExtension { get; set; }
+
+    [JsonPropertyName("content_type")]
+    public string ContentType { get; set; }
+
+    [JsonPropertyName("file_url")]
+    public string FileUrl { get; set; }
+
+    public static KnowledgeFileViewModel From(KnowledgeFileModel model)
+    {
+        return new KnowledgeFileViewModel
+        {
+            FileId = model.FileId,
+            FileName = model.FileName,
+            FileExtension = model.FileExtension,
+            ContentType = model.ContentType,
+            FileUrl = model.FileUrl
+        };
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs

@@ -1,12 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace BotSharp.OpenAPI.ViewModels.Knowledges;
-
-public class KnowledgeSearchViewModel
-{
-    [JsonPropertyName("vector_result")]
-    public IEnumerable<VectorKnowledgeViewModel>? VectorResult { get; set; }
-
-    [JsonPropertyName("graph_result")]
-    public GraphKnowledgeViewModel? GraphResult { get; set; }
-}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs

@@ -1,43 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace BotSharp.OpenAPI.ViewModels.Knowledges;
-
-public class SearchKnowledgeRequest
-{
-    [JsonPropertyName("text")]
-    public string Text { get; set; } = string.Empty;
-
-    #region Vector
-    [JsonPropertyName("vector_params")]
-    public VectorParam VectorParams { get; set; }
-    #endregion
-
-    #region Graph
-    [JsonPropertyName("graph_params")]
-    public GraphParam GraphParams { get; set; }
-    #endregion
-}
-
-public class VectorParam
-{
-    [JsonPropertyName("collection")]
-    public string Collection { get; set; }
-
-    [JsonPropertyName("fields")]
-    public IEnumerable<string>? Fields { get; set; }
-
-    [JsonPropertyName("limit")]
-    public int? Limit { get; set; } = 5;
-
-    [JsonPropertyName("confidence")]
-    public float? Confidence { get; set; } = 0.5f;
-
-    [JsonPropertyName("with_vector")]
-    public bool WithVector { get; set; }
-} 
-
-public class GraphParam
-{
-    [JsonPropertyName("method")]
-    public string Method { get; set; } = string.Empty;
-}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class VectorKnowledgeUploadRequest
+{
+    public IEnumerable<ExternalFileModel> Files { get; set; } = new List<ExternalFileModel>();
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs

@@ -14,7 +14,7 @@ public class NativeWhisperProvider : IAudioCompletion
     private readonly IFileStorageService _fileStorage;
     private readonly ILogger<NativeWhisperProvider> _logger;
 
-    public string Provider => "native";
+    public string Provider => "native-whisper";
 
     public NativeWhisperProvider(
         BotSharpDatabaseSettings dbSettings,

src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs

@@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback
     {
         if (image == null) return;
 
-        var files = new List<InputFileModel>()
+        var files = new List<FileDataModel>()
         {
-            new InputFileModel
+            new FileDataModel
             {
                 FileName = $"{Guid.NewGuid()}.png",
                 FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"

src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs

@@ -77,7 +77,7 @@ public class GenerateImageFn : IFunctionCallback
     {
         if (images.IsNullOrEmpty()) return;
 
-        var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel
+        var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel
         {
             FileName = $"{Guid.NewGuid()}.png",
             FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}"

src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs

@@ -5,21 +5,36 @@ public static class KnowledgeSettingHelper
     public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName)
     {
         var settings = services.GetRequiredService<KnowledgeBaseSettings>();
-        var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding;
+        var db = services.GetRequiredService<IBotSharpRepository>();
+
+        // Get collection config from db
+        var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+        {
+            CollectionNames = [collectionName],
+            VectorStroageProviders = [settings.VectorDb.Provider]
+        });
+
+        var found = configs?.FirstOrDefault()?.TextEmbedding;
+        var provider = found?.Provider ?? string.Empty;
+        var model = found?.Model ?? string.Empty;
+        var dimension = found?.Dimension ?? 0;
+
         if (found == null)
         {
-            found = settings.Default.TextEmbedding;
+            provider = settings.Default.TextEmbedding.Provider;
+            model = settings.Default.TextEmbedding.Model;
+            dimension = settings.Default.TextEmbedding.Dimension;
         }
 
-        var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == found.Provider);
-        var dimension = found.Dimension;
+        // Set up text embedding
+        var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == provider);
 
-        if (found.Dimension <= 0)
+        if (dimension <= 0)
         {
-            dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model);
+            dimension = GetLlmTextEmbeddingDimension(services, provider, model);
         }
 
-        embedding.SetModelName(found.Model);
+        embedding.SetModelName(model);
         embedding.SetDimension(dimension);
         return embedding;
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs

@@ -14,40 +14,48 @@ public static class TextChopper
     private static List<string> ChopByWord(string content, ChunkOption option)
     {
         var chunks = new List<string>();
+        var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
 
-        var words = content.Split(' ')
-            .Where(x => !string.IsNullOrWhiteSpace(x))
-            .ToList();
-
-        var chunk = "";
+        var chunk = string.Empty;
         for (int i = 0; i < words.Count; i++)
         {
             chunk += words[i] + " ";
             if (chunk.Length > option.Size)
             {
                 chunks.Add(chunk.Trim());
-                chunk = "";
+                chunk = string.Empty;
                 i -= option.Conjunction;
             }
         }
 
+        if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))
+        {
+            chunks.Add(chunk);
+        }
+
         return chunks;
     }
 
     private static List<string> ChopByChar(string content, ChunkOption option)
     {
         var chunks = new List<string>();
+        var chunk = string.Empty;
         var currentPos = 0;
+
         while (currentPos < content.Length)
         {
-            var len = content.Length - currentPos > option.Size ?
-                option.Size :
-                content.Length - currentPos;
-            var chunk = content.Substring(currentPos, len);
+            var len = content.Length - currentPos > option.Size ? option.Size : content.Length - currentPos;
+            chunk = content.Substring(currentPos, len);
             chunks.Add(chunk);
             // move backward
             currentPos += option.Size - option.Conjunction;
         }
+
+        if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))
+        {
+            chunks.Add(chunk);
+        }
+
         return chunks;
     }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs

@@ -37,7 +37,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
             SubMenu = new List<PluginMenuDef>
             {
                 new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"),
-                new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships")
+                new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships"),
+                new PluginMenuDef("Documents", link: "page/knowledge-base/documents")
             }
         });
         return true;

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -74,7 +74,7 @@ public class MemoryVectorDb : IVectorDb
         return true;
     }
 
-    public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
+    public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
     {
         return await Task.FromResult(false);
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs

@@ -0,0 +1,12 @@
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var collections = configs.Collections ?? new();
+        var saved = db.AddKnowledgeCollectionConfigs(collections, reset: true);
+        return await Task.FromResult(saved);
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs

@@ -1,69 +0,0 @@
-namespace BotSharp.Plugin.KnowledgeBase.Services;
-
-public partial class KnowledgeService
-{
-    public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
-    {
-        var index = 0;
-        var lines = TextChopper.Chop(knowledge.Content, new ChunkOption
-        {
-            Size = 1024,
-            Conjunction = 32,
-            SplitByWord = true,
-        });
-
-        var db = GetVectorDb();
-        var textEmbedding = GetTextEmbedding(collectionName);
-
-        await db.CreateCollection(collectionName, textEmbedding.GetDimension());
-        foreach (var line in lines)
-        {
-            var vec = await textEmbedding.GetVectorAsync(line);
-            await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
-            index++;
-            Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
-        }
-    }
-
-    public async Task<bool> CreateVectorCollection(string collectionName, int dimension)
-    {
-        try
-        {
-            if (string.IsNullOrWhiteSpace(collectionName))
-            {
-                return false;
-            }
-
-            var db = GetVectorDb();
-            return await db.CreateCollection(collectionName, dimension);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
-            return false;
-        }
-    }
-
-    public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
-    {
-        try
-        {
-            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text))
-            {
-                return false;
-            }
-
-            var textEmbedding = GetTextEmbedding(collectionName);
-            var vector = await textEmbedding.GetVectorAsync(create.Text);
-
-            var db = GetVectorDb();
-            var guid = Guid.NewGuid();
-            return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
-            return false;
-        }
-    }
-}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs

@@ -1,42 +0,0 @@
-namespace BotSharp.Plugin.KnowledgeBase.Services;
-
-public partial class KnowledgeService
-{
-    public async Task<bool> DeleteVectorCollection(string collectionName)
-    {
-        try
-        {
-            if (string.IsNullOrWhiteSpace(collectionName))
-            {
-                return false;
-            }
-
-            var db = GetVectorDb();
-            return await db.DeleteCollection(collectionName);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
-            return false;
-        }
-    }
-
-    public async Task<bool> DeleteVectorCollectionData(string collectionName, string id)
-    {
-        try
-        {
-            if (!Guid.TryParse(id, out var guid))
-            {
-                return false;
-            }
-
-            var db = GetVectorDb();
-            return await db.DeleteCollectionData(collectionName, guid);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
-            return false;
-        }
-    }
-}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -0,0 +1,250 @@
+using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Files.Models;
+using BotSharp.Abstraction.Files.Utilities;
+using System.Net.Http;
+using System.Net.Mime;
+
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
+        {
+            return new UploadKnowledgeResponse
+            {
+                Success = [],
+                Failed = files?.Select(x => x.FileName) ?? new List<string>()
+            };
+        }
+
+        var fileStoreage = _services.GetRequiredService<IFileStorageService>();
+        var userId = await GetUserId();
+        var vectorStoreProvider = _settings.VectorDb.Provider;
+        var successFiles = new List<string>();
+        var failedFiles = new List<string>();
+
+        foreach (var file in files)
+        {
+            if (string.IsNullOrWhiteSpace(file.FileData)
+                && string.IsNullOrWhiteSpace(file.FileUrl))
+            {
+                continue;
+            }
+
+            try
+            {
+                // Get document info
+                var (contentType, bytes) = await GetFileInfo(file);
+                var contents = await GetFileContent(contentType, bytes);
+                
+                // Save document
+                var fileId = Guid.NewGuid().ToString();
+                var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
+                if (!saved)
+                {
+                    failedFiles.Add(file.FileName);
+                    continue;
+                }
+
+                // Save to vector db
+                var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents);
+                if (!dataIds.IsNullOrEmpty())
+                {
+                    fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData
+                    {
+                        Collection = collectionName,
+                        FileId = fileId,
+                        FileName = file.FileName,
+                        ContentType = contentType,
+                        VectorDataIds = dataIds,
+                        CreateDate = DateTime.UtcNow,
+                        CreateUserId = userId
+                    });
+                    successFiles.Add(file.FileName);
+                }
+                else
+                {
+                    failedFiles.Add(file.FileName);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"Error when processing knowledge file ({file.FileName}). {ex.Message}\r\n{ex.InnerException}");
+                failedFiles.Add(file.FileName);
+                continue;
+            }
+        }
+
+        return new UploadKnowledgeResponse
+        {
+            Success = successFiles,
+            Failed = failedFiles
+        };
+    }
+
+
+    public async Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
+        {
+            return false;
+        }
+
+        try
+        {
+            var fileStorage = _services.GetRequiredService<IFileStorageService>();
+            var vectorDb = GetVectorDb();
+            var vectorStoreProvider = _settings.VectorDb.Provider;
+
+            // Get doc meta data
+            var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
+            // Delete doc
+            fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
+
+            if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty())
+            {
+                var guids = metaData.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
+                await vectorDb.DeleteCollectionData(collectionName, guids);
+            }
+            
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting knowledge document " +
+                $"(Collection: {collectionName}, File id: {fileId})" +
+                $"\r\n{ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+
+    public async Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName))
+        {
+            return Enumerable.Empty<KnowledgeFileModel>();
+        }
+
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
+        var vectorStoreProvider = _settings.VectorDb.Provider;
+
+        // Get doc meta data
+        var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
+        return files;
+    }
+
+    public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId)
+    {
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
+        var vectorStoreProvider = _settings.VectorDb.Provider;
+
+        // Get doc binary data
+        var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
+        return file;
+    }
+
+
+    #region Private methods
+    /// <summary>
+    /// Get file content type and file bytes
+    /// </summary>
+    /// <param name="file"></param>
+    /// <returns></returns>
+    private async Task<(string, byte[])> GetFileInfo(ExternalFileModel file)
+    {
+        if (file == null)
+        {
+            return (string.Empty, new byte[0]);
+        }
+
+        if (!string.IsNullOrWhiteSpace(file.FileUrl))
+        {
+            var http = _services.GetRequiredService<IHttpClientFactory>();
+            var contentType = FileUtility.GetFileContentType(file.FileName);
+            using var client = http.CreateClient();
+            var bytes = await client.GetByteArrayAsync(file.FileUrl);
+            return (contentType, bytes);
+        }
+        else if (!string.IsNullOrWhiteSpace(file.FileData))
+        {
+            var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
+            return (contentType, bytes);
+        }
+
+        return (string.Empty, new byte[0]);
+    }
+
+    private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes)
+    {
+        var results = new List<string>();
+
+        if (contentType.IsEqualTo(MediaTypeNames.Text.Plain))
+        {
+            using var stream = new MemoryStream(bytes);
+            using var reader = new StreamReader(stream);
+            var content = await reader.ReadToEndAsync();
+            reader.Close();
+            stream.Close();
+
+            var lines = TextChopper.Chop(content, new ChunkOption
+            {
+                Size = 1024,
+                Conjunction = 32,
+                SplitByWord = true,
+            });
+            results.AddRange(lines);
+        }
+        else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))
+        {
+            // to do
+        }
+        
+        return results;
+    }
+
+    private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
+    {
+        var fileStoreage = _services.GetRequiredService<IFileStorageService>();
+        using var stream = new MemoryStream(bytes);
+        stream.Position = 0;
+
+        var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, stream);
+        stream.Close();
+        return saved;
+    }
+
+    private async Task<IEnumerable<string>> SaveToVectorDb(string collectionName, string fileId, string fileName, IEnumerable<string> contents)
+    {
+        if (contents.IsNullOrEmpty())
+        {
+            return Enumerable.Empty<string>();
+        }
+
+        var dataIds = new List<string>();
+        var vectorDb = GetVectorDb();
+        var textEmbedding = GetTextEmbedding(collectionName);
+
+        for (int i = 0; i < contents.Count(); i++)
+        {
+            var content = contents.ElementAt(i);
+            var vector = await textEmbedding.GetVectorAsync(content);
+            var dataId = Guid.NewGuid();
+            var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
+            {
+                { "fileName", fileName },
+                { "fileId", fileId },
+                { "textNumber", $"{i + 1}" }
+            });
+
+            if (!saved) continue;
+
+            dataIds.Add(dataId.ToString());
+        }
+
+        return dataIds;
+    }
+    #endregion
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs

@@ -1,103 +0,0 @@
-namespace BotSharp.Plugin.KnowledgeBase.Services;
-
-public partial class KnowledgeService
-{
-    public async Task<IEnumerable<string>> GetVectorCollections()
-    {
-        try
-        {
-            var db = GetVectorDb();
-            return await db.GetCollections();
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
-            return Enumerable.Empty<string>();
-        }
-    }
-
-    public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
-    {
-        try
-        {
-            var db = GetVectorDb();
-            var pagedResult =  await db.GetPagedCollectionData(collectionName, filter);
-            return new StringIdPagedItems<VectorSearchResult>
-            {
-                Count = pagedResult.Count,
-                Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)),
-                NextId = pagedResult.NextId,
-            };
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
-            return new StringIdPagedItems<VectorSearchResult>();
-        }
-    }
-
-    public async Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options)
-    {
-        try
-        {
-            var textEmbedding = GetTextEmbedding(collectionName);
-            var vector = await textEmbedding.GetVectorAsync(query);
-
-            // Vector search
-            var db = GetVectorDb();
-            var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
-
-            var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList();
-            return results;
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
-            return new List<VectorSearchResult>();
-        }
-    }
-
-    public async Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options)
-    {
-        try
-        {
-            var db = GetGraphDb();
-            var found = await db.Search(query, options);
-            return new GraphSearchResult
-            {
-                Result = found.Result
-            };
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
-            return new GraphSearchResult();
-        }
-    }
-
-    public async Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions)
-    {
-        try
-        {
-            var textEmbedding = GetTextEmbedding(collectionName);
-            var vector = await textEmbedding.GetVectorAsync(query);
-
-            var vectorDb = GetVectorDb();
-            var vectorRes = await vectorDb.Search(collectionName, vector, vectorOptions.Fields, limit: vectorOptions.Limit ?? 5,
-                            confidence: vectorOptions.Confidence ?? 0.5f, withVector: vectorOptions.WithVector);
-
-            var graphDb = GetGraphDb();
-            var graphRes = await graphDb.Search(query, graphOptions);
-            return new KnowledgeSearchResult
-            {
-                VectorResult = vectorRes.Select(x => VectorSearchResult.CopyFrom(x)),
-                GraphResult = new GraphSearchResult { Result = graphRes.Result }
-            };
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when searching knowledge (Vector collection: {collectionName}) (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
-            return new KnowledgeSearchResult();
-        }
-    }
-}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs

@@ -0,0 +1,22 @@
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options)
+    {
+        try
+        {
+            var db = GetGraphDb();
+            var found = await db.Search(query, options);
+            return new GraphSearchResult
+            {
+                Result = found.Result
+            };
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
+            return new GraphSearchResult();
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs

@@ -1,31 +0,0 @@
-namespace BotSharp.Plugin.KnowledgeBase.Services;
-
-public partial class KnowledgeService
-{
-    public async Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update)
-    {
-        try
-        {
-            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
-            {
-                return false;
-            }
-
-            var db = GetVectorDb();
-            var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
-            if (found.IsNullOrEmpty())
-            {
-                return false;
-            }
-
-            var textEmbedding = GetTextEmbedding(collectionName);
-            var vector = await textEmbedding.GetVectorAsync(update.Text);
-            return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
-            return false;
-        }
-    }
-}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -0,0 +1,218 @@
+using BotSharp.Abstraction.Files;
+
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    #region Collection
+    public async Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName))
+            {
+                return false;
+            }
+
+            var vectorDb = GetVectorDb();
+            var created = await vectorDb.CreateCollection(collectionName, dimension);
+            if (created)
+            {
+                var db = _services.GetRequiredService<IBotSharpRepository>();
+                var userId = await GetUserId();
+
+                db.AddKnowledgeCollectionConfigs(new List<VectorCollectionConfig>
+                {
+                    new VectorCollectionConfig
+                    {
+                        Name = collectionName,
+                        Type = collectionType,
+                        VectorStorage = new VectorStorageConfig
+                        {
+                            Provider = _settings.VectorDb.Provider
+                        },
+                        TextEmbedding = new KnowledgeEmbeddingConfig
+                        {
+                            Provider = provider,
+                            Model = model,
+                            Dimension = dimension
+                        }
+                    }
+                });
+            }
+
+            return created;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+    public async Task<IEnumerable<string>> GetVectorCollections(string type)
+    {
+        try
+        {
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var collectionNames = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+            {
+                CollectionTypes = new[] { type },
+                VectorStroageProviders = new[] { _settings.VectorDb.Provider }
+            }).Select(x => x.Name).ToList();
+
+            var vectorDb = GetVectorDb();
+            var vectorCollections = await vectorDb.GetCollections();
+            return vectorCollections.Where(x => collectionNames.Contains(x));
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
+            return Enumerable.Empty<string>();
+        }
+    }
+
+    public async Task<bool> DeleteVectorCollection(string collectionName)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName))
+            {
+                return false;
+            }
+
+            var vectorDb = GetVectorDb();
+            var deleted = await vectorDb.DeleteCollection(collectionName);
+
+            if (deleted)
+            {
+                var db = _services.GetRequiredService<IBotSharpRepository>();
+                var fileStorage = _services.GetRequiredService<IFileStorageService>();
+                var vectorStoreProvider = _settings.VectorDb.Provider;
+
+                db.DeleteKnowledgeCollectionConfig(collectionName);
+                fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
+            }
+
+            return deleted;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+    #endregion
+
+    #region Collection data
+    public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text))
+            {
+                return false;
+            }
+
+            var textEmbedding = GetTextEmbedding(collectionName);
+            var vector = await textEmbedding.GetVectorAsync(create.Text);
+
+            var db = GetVectorDb();
+            var guid = Guid.NewGuid();
+            return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+    public async Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
+            {
+                return false;
+            }
+
+            var db = GetVectorDb();
+            var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
+            if (found.IsNullOrEmpty())
+            {
+                return false;
+            }
+
+            var textEmbedding = GetTextEmbedding(collectionName);
+            var vector = await textEmbedding.GetVectorAsync(update.Text);
+            return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+    public async Task<bool> DeleteVectorCollectionData(string collectionName, string id)
+    {
+        try
+        {
+            if (!Guid.TryParse(id, out var guid))
+            {
+                return false;
+            }
+
+            var db = GetVectorDb();
+            return await db.DeleteCollectionData(collectionName, new List<Guid> { guid });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+    public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
+    {
+        try
+        {
+            var db = GetVectorDb();
+            var pagedResult = await db.GetPagedCollectionData(collectionName, filter);
+            return new StringIdPagedItems<VectorSearchResult>
+            {
+                Count = pagedResult.Count,
+                Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)),
+                NextId = pagedResult.NextId,
+            };
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return new StringIdPagedItems<VectorSearchResult>();
+        }
+    }
+
+    public async Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options)
+    {
+        try
+        {
+            var textEmbedding = GetTextEmbedding(collectionName);
+            var vector = await textEmbedding.GetVectorAsync(query);
+
+            // Vector search
+            var db = GetVectorDb();
+            var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
+
+            var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList();
+            return results;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return Enumerable.Empty<VectorSearchResult>();
+        }
+    }
+    #endregion
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs

@@ -3,15 +3,18 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 public partial class KnowledgeService : IKnowledgeService
 {
     private readonly IServiceProvider _services;
+    private readonly IUserIdentity _user;
     private readonly KnowledgeBaseSettings _settings;
     private readonly ILogger<KnowledgeService> _logger;
 
     public KnowledgeService(
         IServiceProvider services,
+        IUserIdentity user,
         KnowledgeBaseSettings settings,
         ILogger<KnowledgeService> logger)
     {
         _services = services;
+        _user = user;
         _settings = settings;
         _logger = logger;
     }
@@ -28,8 +31,15 @@ public partial class KnowledgeService : IKnowledgeService
         return db;
     }
 
-    private ITextEmbedding GetTextEmbedding(string collection)
+    private ITextEmbedding GetTextEmbedding(string collectionName)
     {
-        return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection);
+        return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName);
+    }
+
+    private async Task<string> GetUserId()
+    {
+        var userService = _services.GetRequiredService<IUserService>();
+        var user = await userService.GetUser(_user.Id);
+        return user.Id;
     }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -48,7 +48,7 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<bool> DeleteCollectionData(string collectionName, Guid id)
+    public Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Plugin.MongoStorage.Collections;
+
+public class KnowledgeCollectionConfigDocument : MongoBase
+{
+    public string Name { get; set; }
+    public string Type { get; set; }
+    public KnowledgeVectorStorageConfigMongoModel VectorStorage { get; set; }
+    public KnowledgeEmbeddingConfigMongoModel TextEmbedding { get; set; }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs

@@ -0,0 +1,30 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class KnowledgeEmbeddingConfigMongoModel
+{
+    public string Provider { get; set; }
+    public string Model { get; set; }
+    public int Dimension { get; set; }
+
+    public static KnowledgeEmbeddingConfigMongoModel ToMongoModel(KnowledgeEmbeddingConfig model)
+    {
+        return new KnowledgeEmbeddingConfigMongoModel
+        {
+            Provider = model.Provider,
+            Model = model.Model,
+            Dimension = model.Dimension
+        };
+    }
+
+    public static KnowledgeEmbeddingConfig ToDomainModel(KnowledgeEmbeddingConfigMongoModel model)
+    {
+        return new KnowledgeEmbeddingConfig
+        {
+            Provider = model.Provider,
+            Model = model.Model,
+            Dimension = model.Dimension
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs

@@ -0,0 +1,24 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class KnowledgeVectorStorageConfigMongoModel
+{
+    public string Provider { get; set; }
+
+    public static KnowledgeVectorStorageConfigMongoModel ToMongoModel(VectorStorageConfig model)
+    {
+        return new KnowledgeVectorStorageConfigMongoModel
+        {
+            Provider = model.Provider
+        };
+    }
+
+    public static VectorStorageConfig ToDomainModel(KnowledgeVectorStorageConfigMongoModel model)
+    {
+        return new VectorStorageConfig
+        {
+            Provider = model.Provider
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

@@ -153,4 +153,7 @@ public class MongoDbContext
 
     public IMongoCollection<TranslationMemoryDocument> TranslationMemories
         => Database.GetCollection<TranslationMemoryDocument>($"{_collectionPrefix}_TranslationMemories");
+
+    public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
+        => Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -0,0 +1,114 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Repository;
+
+public partial class MongoRepository
+{
+    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
+    {
+        var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
+        var docs = configs?.Where(x => !string.IsNullOrWhiteSpace(x.Name))
+            .Select(x => new KnowledgeCollectionConfigDocument
+            {
+                Id = Guid.NewGuid().ToString(),
+                Name = x.Name,
+                Type = x.Type,
+                VectorStorage = KnowledgeVectorStorageConfigMongoModel.ToMongoModel(x.VectorStorage),
+                TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding)
+            })?.ToList() ?? new List<KnowledgeCollectionConfigDocument>();
+
+        if (reset)
+        {
+            _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
+            _dc.KnowledgeCollectionConfigs.InsertMany(docs);
+            return true;
+        }
+
+        // Update if collection already exists, otherwise insert.
+        var insertDocs = new List<KnowledgeCollectionConfigDocument>();
+        var updateDocs = new List<KnowledgeCollectionConfigDocument>();
+
+        var names = docs.Select(x => x.Name).ToList();
+        filter = Builders<KnowledgeCollectionConfigDocument>.Filter.In(x => x.Name, names);
+        var savedConfigs = _dc.KnowledgeCollectionConfigs.Find(filter).ToList();
+        
+        foreach (var doc in docs)
+        {
+            var found = savedConfigs.FirstOrDefault(x => x.Name == doc.Name);
+            if (found != null)
+            {
+                found.Type = doc.Type;
+                found.VectorStorage = doc.VectorStorage;
+                found.TextEmbedding = doc.TextEmbedding;
+                updateDocs.Add(found);
+            }
+            else
+            {
+                insertDocs.Add(doc);
+            }
+        }
+
+        if (!insertDocs.IsNullOrEmpty())
+        {
+            _dc.KnowledgeCollectionConfigs.InsertMany(docs);
+        }
+
+        if (!updateDocs.IsNullOrEmpty())
+        {
+            foreach (var doc in updateDocs)
+            {
+                filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Id, doc.Id);
+                _dc.KnowledgeCollectionConfigs.ReplaceOne(filter, doc);
+            }
+        }
+        
+        return true;
+    }
+
+    public bool DeleteKnowledgeCollectionConfig(string collectionName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)) return false;
+
+        var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Name, collectionName);
+        var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
+        return deleted.DeletedCount > 0;
+    }
+
+    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter)
+    {
+        if (filter == null)
+        {
+            return Enumerable.Empty<VectorCollectionConfig>();
+        }
+
+        var builder = Builders<KnowledgeCollectionConfigDocument>.Filter;
+        var filters = new List<FilterDefinition<KnowledgeCollectionConfigDocument>> { builder.Empty };
+
+        // Apply filters
+        if (!filter.CollectionNames.IsNullOrEmpty())
+        {
+            filters.Add(builder.In(x => x.Name, filter.CollectionNames));
+        }
+
+        if (!filter.CollectionTypes.IsNullOrEmpty())
+        {
+            filters.Add(builder.In(x => x.Type, filter.CollectionTypes));
+        }
+
+        if (!filter.VectorStroageProviders.IsNullOrEmpty())
+        {
+            filters.Add(builder.In(x => x.VectorStorage.Provider, filter.VectorStroageProviders));
+        }
+
+        // Get data
+        var configs = _dc.KnowledgeCollectionConfigs.Find(Builders<KnowledgeCollectionConfigDocument>.Filter.And(filters)).ToList();
+
+        return configs.Select(x => new VectorCollectionConfig
+        {
+            Name = x.Name,
+            Type = x.Type,
+            VectorStorage = KnowledgeVectorStorageConfigMongoModel.ToDomainModel(x.VectorStorage),
+            TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding)
+        });
+    }
+}

src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs

@@ -21,7 +21,6 @@ public class PrimaryStagePlanFn : IFunctionCallback
         var state = _services.GetRequiredService<IConversationStateService>();
         var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
         var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
-        var fn = _services.GetRequiredService<IRoutingService>();
 
         state.SetState("max_tokens", "4096");
         var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
+using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
 
@@ -10,12 +11,15 @@ public class QdrantDb : IVectorDb
     private QdrantClient _client;
     private readonly QdrantSetting _setting;
     private readonly IServiceProvider _services;
+    private readonly ILogger<QdrantDb> _logger;
 
     public QdrantDb(
         QdrantSetting setting,
+        ILogger<QdrantDb> logger,
         IServiceProvider services)
     {
         _setting = setting;
+        _logger = logger;
         _services = services;
     }
 
@@ -35,21 +39,28 @@ public class QdrantDb : IVectorDb
         return _client;
     }
 
-    public async Task<bool> CreateCollection(string collectionName, int dim)
+    public async Task<bool> CreateCollection(string collectionName, int dimension)
     {
         var client = GetClient();
         var exist = await DoesCollectionExist(client, collectionName);
 
         if (exist) return false;
 
-        // Create a new collection
-        await client.CreateCollectionAsync(collectionName, new VectorParams()
+        try
         {
-            Size = (ulong)dim,
-            Distance = Distance.Cosine
-        });
-
-        return true;
+            // Create a new collection
+            await client.CreateCollectionAsync(collectionName, new VectorParams()
+            {
+                Size = (ulong)dimension,
+                Distance = Distance.Cosine
+            });
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when create collection (Name: {collectionName}, Dimension: {dimension}).");
+            return false;
+        }
     }
 
     public async Task<bool> DeleteCollection(string collectionName)
@@ -145,8 +156,6 @@ public class QdrantDb : IVectorDb
         });
     }
 
-    
-
     public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
     {
         // Insert vectors
@@ -216,10 +225,12 @@ public class QdrantDb : IVectorDb
         return results;
     }
 
-    public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
+    public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
     {
+        if (ids.IsNullOrEmpty()) return false;
+
         var client = GetClient();
-        var result = await client.DeleteAsync(collectionName, id);
+        var result = await client.DeleteAsync(collectionName, ids);
         return result.Status == UpdateStatus.Completed;
     }
 

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -4,6 +4,7 @@ using BotSharp.Abstraction.VectorStorage.Models;
 using Microsoft.SemanticKernel.Memory;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 
 namespace BotSharp.Plugin.SemanticKernel
@@ -84,16 +85,15 @@ namespace BotSharp.Plugin.SemanticKernel
             return true;
         }
 
-        public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
+        public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
         {
-            var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
+            if (ids.IsNullOrEmpty()) return false;
 
-            if (exist)
-            {
-                await _memoryStore.RemoveAsync(collectionName, id.ToString());
-                return true;
-            }
-            return false;
+            var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
+            if (!exist) return false;
+
+            await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString()));
+            return true;
         }
     }
 }

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs

@@ -115,7 +115,7 @@ public partial class TencentCosService
         return foundMsgs;
     }
 
-    public bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files)
+    public bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files)
     {
         if (files.IsNullOrEmpty()) return false;
 

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs

@@ -0,0 +1,36 @@
+using BotSharp.Abstraction.Knowledges.Models;
+
+namespace BotSharp.Plugin.TencentCos.Services;
+
+public partial class TencentCosService
+{
+    public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream)
+    {
+        throw new NotImplementedException();
+    }
+
+    public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
+    {
+        throw new NotImplementedException();
+    }
+
+    public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData)
+    {
+        throw new NotImplementedException();
+    }
+
+    public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
+    {
+        throw new NotImplementedException();
+    }
+
+    public IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider)
+    {
+        throw new NotImplementedException();
+    }
+
+    public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
+    {
+        throw new NotImplementedException();
+    }
+}

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs

@@ -16,7 +16,7 @@ public partial class TencentCosService
         return found;
     }
 
-    public bool SaveUserAvatar(InputFileModel file)
+    public bool SaveUserAvatar(FileDataModel file)
     {
         if (file == null || string.IsNullOrEmpty(file.FileData)) return false;
 

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.Users;
 using BotSharp.Plugin.TencentCos.Settings;
 using System.Net.Mime;

src/WebStarter/appsettings.json

@@ -141,7 +141,8 @@
       "Enable": true,
       "BatchSize": 50,
       "MessageLimit": 2,
-      "BufferHours": 12
+      "BufferHours": 12,
+      "ExcludeAgentIds": []
     },
     "RateLimit": {
       "MaxConversationPerDay": 100,