resolve conflict

src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs

@@ -63,16 +63,17 @@ public interface IFileStorageService
     #endregion
 
     #region Knowledge
-    bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData);
+    bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData);
 
     /// <summary>
-    /// Delete files in a knowledge collection. If fileId is null, remove all files in the collection.
+    /// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection.
     /// </summary>
     /// <param name="collectionName"></param>
+    /// <param name="vectorStoreProvider"></param>
     /// <param name="fileId"></param>
     /// <returns></returns>
-    bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
+    bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
-    string GetKnowledgeBaseFileUrl(string collectionName, string fileId);
+    string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
-    FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId);
+    BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs

@@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Models;
 
 public class KnowledgeFileModel
 {
-    public string FileId { get; set; }
+    public Guid FileId { get; set; }
     public string FileName { get; set; }
     public string FileExtension { get; set; }
     public string ContentType { get; set; }

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -22,9 +22,9 @@ public interface IKnowledgeService
 
     #region Document
     Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
-    Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
+    Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
-    Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId);
+    Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion
 
     #region Common

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs

@@ -6,7 +6,7 @@ public class KnowledgeDocMetaData
     public string Collection { get; set; }
 
     [JsonPropertyName("file_id")]
-    public string FileId { get; set; }
+    public Guid FileId { get; set; }
 
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }
@@ -23,6 +23,9 @@ public class KnowledgeDocMetaData
     [JsonPropertyName("vector_data_ids")]
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
 
+    [JsonPropertyName("web_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? WebUrl { get; set; }
 
     [JsonPropertyName("create_date")]
     public DateTime CreateDate { get; set; } = DateTime.UtcNow;

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs

@@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 
 public class KnowledgeFileFilter : Pagination
 {
-    public IEnumerable<string>? FileIds { get; set; }
+    public IEnumerable<Guid>? FileIds { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -113,7 +113,15 @@ public interface IBotSharpRepository
     bool DeleteKnowledgeCollectionConfig(string collectionName);
     IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
 
-    public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
+    bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
-    public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
+    /// <summary>
+    /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="vectorStoreProvider"></param>
+    /// <param name="fileId"></param>
+    /// <returns></returns>
+    bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null);
+    PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
     #endregion
 }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs

@@ -1,23 +1,21 @@
-using BotSharp.Abstraction.Knowledges.Models;
 using System.IO;
 
 namespace BotSharp.Core.Files.Services;
 
 public partial class LocalFileStorageService
 {
-    public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
+    public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
-            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
-            || string.IsNullOrWhiteSpace(fileId))
         {
             return false;
         }
 
         try
         {
-            var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+            var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-            var dir = Path.Combine(docDir, fileId);
+            var dir = Path.Combine(docDir, fileId.ToString());
             if (ExistDirectory(dir))
             {
                 Directory.Delete(dir);
@@ -40,7 +38,7 @@ public partial class LocalFileStorageService
         }
     }
 
-    public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
+    public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider))
@@ -48,16 +46,16 @@ public partial class LocalFileStorageService
             return false;
         }
 
-        var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
         if (!ExistDirectory(dir)) return false;
 
-        if (string.IsNullOrEmpty(fileId))
+        if (fileId == null)
         {
             Directory.Delete(dir, true);
         }
         else
         {
-            var fileDir = Path.Combine(dir, fileId);
+            var fileDir = Path.Combine(dir, fileId.ToString());
             if (ExistDirectory(fileDir))
             {
                 Directory.Delete(fileDir, true);
@@ -67,10 +65,10 @@ public partial class LocalFileStorageService
         return true;
     }
 
-    public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
+    public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
-             || string.IsNullOrWhiteSpace(fileId))
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
         {
             return string.Empty;
         }
@@ -78,39 +76,31 @@ public partial class LocalFileStorageService
         return $"/knowledge/document/{collectionName}/file/{fileId}";
     }
 
-    public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
+    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
-            || string.IsNullOrWhiteSpace(fileId))
+            || string.IsNullOrWhiteSpace(fileName))
         {
             return null;
         }
 
-        var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
+        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = Path.Combine(docDir, fileId);
+        var fileDir = Path.Combine(docDir, fileId.ToString());
         if (!ExistDirectory(fileDir)) return null;
 
-        var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE);
+        var file = Path.Combine(fileDir, fileName);
-        var content = File.ReadAllText(metaFile);
-        var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
-        var file = Path.Combine(fileDir, metaData.FileName);
         using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
         stream.Position = 0;
 
-        return new FileBinaryDataModel
+        return BinaryData.FromStream(stream);
-        {
-            FileName = metaData.FileName,
-            ContentType = metaData.ContentType,
-            FileBinaryData = BinaryData.FromStream(stream)
-        };
     }
 
 
     #region Private methods
-    private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
+    private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
     {
-        return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
+        return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
     }
     #endregion
 }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs

@@ -21,7 +21,6 @@ public partial class LocalFileStorageService : IFileStorageService
     private const string TEXT_TO_SPEECH_FOLDER = "speeches";
     private const string KNOWLEDGE_FOLDER = "knowledgebase";
     private const string KNOWLEDGE_DOC_FOLDER = "document";
-    private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
 
     private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
     {

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -234,7 +234,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
         throw new NotImplementedException();
     #endregion
 
-    #region Knowledge
+    #region KnowledgeBase
     public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
         throw new NotImplementedException();
 
@@ -247,6 +247,9 @@ public class BotSharpDbContext : Database, IBotSharpRepository
     public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) =>
         throw new NotImplementedException();
 
+    public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) =>
+        throw new NotImplementedException();
+
     public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) =>
         throw new NotImplementedException();
     #endregion

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -110,14 +110,13 @@ public partial class FileRepository
     {
         if (metaData == null
             || string.IsNullOrWhiteSpace(metaData.Collection)
-            || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
+            || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
-            || string.IsNullOrWhiteSpace(metaData.FileId))
         {
             return false;
         }
 
-        var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr());
+        var dir = BuildKnowledgeCollectionFileDir(metaData.Collection, metaData.VectorStoreProvider);
-        var docDir = Path.Combine(dir, metaData.FileId);
+        var docDir = Path.Combine(dir, metaData.FileId.ToString());
         if (!Directory.Exists(docDir))
         {
             Directory.CreateDirectory(docDir);
@@ -129,6 +128,33 @@ public partial class FileRepository
         return true;
     }
 
+    public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+        {
+            return false;
+        }
+
+        var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+        if (!Directory.Exists(dir)) return false;
+
+        if (fileId == null)
+        {
+            Directory.Delete(dir, true);
+        }
+        else
+        {
+            var fileDir = Path.Combine(dir, fileId.ToString());
+            if (Directory.Exists(fileDir))
+            {
+                Directory.Delete(fileDir, true);
+            }
+        }
+
+        return true;
+    }
+
     public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
@@ -137,7 +163,7 @@ public partial class FileRepository
             return new PagedItems<KnowledgeDocMetaData>();
         }
 
-        var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider);
+        var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
         if (!Directory.Exists(dir))
         {
             return new PagedItems<KnowledgeDocMetaData>();
@@ -181,9 +207,9 @@ public partial class FileRepository
         return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
     }
 
-    private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider)
+    private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
     {
-        return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
+        return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
     }
     #endregion
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -137,7 +137,7 @@ public class KnowledgeBaseController : ControllerBase
     }
 
     [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
-    public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
+    public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
     {
         var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
         return response;
@@ -160,7 +160,7 @@ public class KnowledgeBaseController : ControllerBase
     }
 
     [HttpGet("/knowledge/document/{collection}/file/{fileId}")]
-    public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
+    public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
     {
         var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId);
         var stream = file.FileBinaryData.ToStream();

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs

@@ -5,7 +5,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges;
 public class KnowledgeFileViewModel
 {
     [JsonPropertyName("file_id")]
-    public string FileId { get; set; }
+    public Guid FileId { get; set; }
 
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs

@@ -44,10 +44,7 @@ public class WelcomeHook : ConversationHookBase
 
             foreach (var message in messages)
             {
-                var richContent = new RichContent<IRichMessage>(message)
+                var richContent = new RichContent<IRichMessage>(message);
-                {
-                };
-
                 var json = JsonSerializer.Serialize(new ChatResponseModel()
                 {
                     ConversationId = conversation.Id,

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -42,7 +42,7 @@ public partial class KnowledgeService
                 var contents = await GetFileContent(contentType, bytes);
                 
                 // Save document
-                var fileId = Guid.NewGuid().ToString();
+                var fileId = Guid.NewGuid();
                 var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
                 if (!saved)
                 {
@@ -89,9 +89,9 @@ public partial class KnowledgeService
     }
 
 
-    public async Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId)
+    public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId)
     {
-        if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
+        if (string.IsNullOrWhiteSpace(collectionName))
         {
             return false;
         }
@@ -104,21 +104,23 @@ public partial class KnowledgeService
             var vectorStoreProvider = _settings.VectorDb.Provider;
 
             // Get doc meta data
-            var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
+            var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
             {
-                FileIds = new[] { fileId }
+                FileIds = [ fileId ],
+                Size = 1
             });
 
             // Delete doc
-            fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
+            fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId);
-
+            
-            var found = pagedData?.Items?.FirstOrDefault();
+            var found = pageData?.Items?.FirstOrDefault();
             if (found != null && !found.VectorDataIds.IsNullOrEmpty())
             {
                 var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
                 await vectorDb.DeleteCollectionData(collectionName, guids);
             }
-            
+
+            db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId);
             return true;
         }
         catch (Exception ex)
@@ -148,13 +150,19 @@ public partial class KnowledgeService
             Size = filter.Size
         });
 
-        var files = pagedData.Items?.Select(x => new KnowledgeFileModel
+        var files = pagedData.Items?.Select(x =>
         {
-            FileId = x.FileId,
+            var fileUrl = x.ContentType == MediaTypeNames.Text.Html ?
-            FileName = x.FileName,
+                        x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName);  
-            FileExtension = Path.GetExtension(x.FileName),
+
-            ContentType = x.ContentType,
+            return new KnowledgeFileModel
-            FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId)
+            {
+                FileId = x.FileId,
+                FileName = x.FileName,
+                FileExtension = Path.GetExtension(x.FileName),
+                ContentType = x.ContentType,
+                FileUrl = fileUrl
+            };
         })?.ToList() ?? new List<KnowledgeFileModel>();
 
         return new PagedItems<KnowledgeFileModel>
@@ -164,14 +172,29 @@ public partial class KnowledgeService
         };
     }
 
-    public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId)
+    public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
     {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var vectorStoreProvider = _settings.VectorDb.Provider;
 
         // Get doc binary data
-        var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
+        var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
-        return file;
+        {
+            FileIds = [ fileId ],
+            Size = 1
+        });
+
+        var metaData = pageData?.Items?.FirstOrDefault();
+        if (metaData == null) return null;
+
+        var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
+        return new FileBinaryDataModel
+        {
+            FileName = metaData.FileName,
+            ContentType = metaData.ContentType,
+            FileBinaryData = binaryData
+        };
     }
 
 
@@ -246,16 +269,16 @@ public partial class KnowledgeService
     #endregion
 
 
-    private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
+    private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes)
     {
         var fileStoreage = _services.GetRequiredService<IFileStorageService>();
         var data = BinaryData.FromBytes(bytes);
-        var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, data);
+        var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data);
         return saved;
     }
 
     private async Task<IEnumerable<string>> SaveToVectorDb(
-        string collectionName, string fileId, string fileName, IEnumerable<string> contents,
+        string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
         string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
     {
         if (contents.IsNullOrEmpty())
@@ -275,7 +298,7 @@ public partial class KnowledgeService
             var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
             {
                 { KnowledgePayloadName.DataSource, vectorDataSource },
-                { KnowledgePayloadName.FileId, fileId },
+                { KnowledgePayloadName.FileId, fileId.ToString() },
                 { KnowledgePayloadName.FileName, fileName },
                 { KnowledgePayloadName.FileSource, fileSource },
                 { "textNumber", $"{i + 1}" }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -92,7 +92,8 @@ public partial class KnowledgeService
                 var vectorStoreProvider = _settings.VectorDb.Provider;
 
                 db.DeleteKnowledgeCollectionConfig(collectionName);
-                fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
+                fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider);
+                db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider);
             }
 
             return deleted;

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs

@@ -1,14 +1,15 @@
 namespace BotSharp.Plugin.MongoStorage.Collections;
 
-public class KnowledgeCollectionFileDocument : MongoBase
+public class KnowledgeCollectionFileMetaDocument : MongoBase
 {
     public string Collection { get; set; }
-    public string FileId { get; set; }
+    public Guid FileId { get; set; }
     public string FileName { get; set; }
     public string FileSource { get; set; }
     public string ContentType { get; set; }
     public string VectorStoreProvider { get; set; }
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
+    public string? WebUrl { get; set; }
     public DateTime CreateDate { get; set; }
     public string CreateUserId { get; set; }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

@@ -157,6 +157,6 @@ public class MongoDbContext
     public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
         => Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
 
-    public IMongoCollection<KnowledgeCollectionFileDocument> KnowledgeCollectionFiles
+    public IMongoCollection<KnowledgeCollectionFileMetaDocument> KnowledgeCollectionFileMeta
-        => Database.GetCollection<KnowledgeCollectionFileDocument>($"{_collectionPrefix}_KnowledgeCollectionFiles");
+        => Database.GetCollection<KnowledgeCollectionFileMetaDocument>($"{_collectionPrefix}_KnowledgeCollectionFileMeta");
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -120,13 +120,12 @@ public partial class MongoRepository
     {
         if (metaData == null
             || string.IsNullOrWhiteSpace(metaData.Collection)
-            || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
+            || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
-            || string.IsNullOrWhiteSpace(metaData.FileId))
         {
             return false;
         }
 
-        var doc = new KnowledgeCollectionFileDocument
+        var doc = new KnowledgeCollectionFileMetaDocument
         {
             Id = Guid.NewGuid().ToString(),
             Collection = metaData.Collection,
@@ -136,14 +135,39 @@ public partial class MongoRepository
             ContentType = metaData.ContentType,
             VectorStoreProvider = metaData.VectorStoreProvider,
             VectorDataIds = metaData.VectorDataIds,
+            WebUrl = metaData.WebUrl,
             CreateDate = metaData.CreateDate,
             CreateUserId = metaData.CreateUserId
         };
 
-        _dc.KnowledgeCollectionFiles.InsertOne(doc);
+        _dc.KnowledgeCollectionFileMeta.InsertOne(doc);
         return true;
     }
 
+    public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+        {
+            return false;
+        }
+
+        var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
+        var filters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
+        {
+            builder.Eq(x => x.Collection, collectionName),
+            builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
+        };
+
+        if (fileId != null)
+        {
+            filters.Add(builder.Eq(x => x.FileId, fileId));
+        }
+
+        var res = _dc.KnowledgeCollectionFileMeta.DeleteMany(builder.And(filters));
+        return res.DeletedCount > 0;
+    }
+
     public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
@@ -152,9 +176,8 @@ public partial class MongoRepository
             return new PagedItems<KnowledgeDocMetaData>();
         }
 
-        var builder = Builders<KnowledgeCollectionFileDocument>.Filter;
+        var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
-        
+        var docFilters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
-        var docFilters = new List<FilterDefinition<KnowledgeCollectionFileDocument>>()
         {
             builder.Eq(x => x.Collection, collectionName),
             builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
@@ -167,9 +190,9 @@ public partial class MongoRepository
         }
 
         var filterDef = builder.And(docFilters);
-        var sortDef = Builders<KnowledgeCollectionFileDocument>.Sort.Descending(x => x.CreateDate);
+        var sortDef = Builders<KnowledgeCollectionFileMetaDocument>.Sort.Descending(x => x.CreateDate);
-        var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
+        var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
-        var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef);
+        var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef);
 
         var files = docs?.Select(x => new KnowledgeDocMetaData
         {
@@ -180,6 +203,7 @@ public partial class MongoRepository
             ContentType = x.ContentType,
             VectorStoreProvider = x.VectorStoreProvider,
             VectorDataIds = x.VectorDataIds,
+            WebUrl = x.WebUrl,
             CreateDate = x.CreateDate,
             CreateUserId = x.CreateUserId
         })?.ToList() ?? new List<KnowledgeDocMetaData>();

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs

@@ -2,23 +2,101 @@ namespace BotSharp.Plugin.TencentCos.Services;
 
 public partial class TencentCosService
 {
-    public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
+    public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
     {
-        throw new NotImplementedException();
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+        {
+            return false;
+        }
+
+        try
+        {
+            var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+            var dir = $"{docDir}/{fileId}";
+            if (ExistDirectory(dir))
+            {
+                _cosClient.BucketClient.DeleteDir(dir);
+            }
+
+            var file = $"{dir}/{fileName}";
+            var res = _cosClient.BucketClient.UploadBytes(file, fileData.ToArray());
+            return res;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving knowledge file " +
+                $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
+                $"\r\n{ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
     }
 
-    public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
+    public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
     {
-        throw new NotImplementedException();
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+        {
+            return false;
+        }
+
+        var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+        if (!ExistDirectory(dir)) return false;
+
+        if (fileId == null)
+        {
+            _cosClient.BucketClient.DeleteDir(dir);
+        }
+        else
+        {
+            var fileDir = $"{dir}/{fileId}";
+            if (ExistDirectory(fileDir))
+            {
+                _cosClient.BucketClient.DeleteDir(fileDir);
+            }
+        }
+
+        return true;
     }
 
-    public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
+    public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
-        throw new NotImplementedException();
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileName))
+        {
+            return string.Empty;
+        }
+
+        var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
+        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
     }
 
-    public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
+    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
-        throw new NotImplementedException();
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileName))
+        {
+            return null;
+        }
+
+        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+        var fileDir = $"{docDir}/{fileId}";
+        if (!ExistDirectory(fileDir)) return null;
+
+        var file = $"{fileDir}/{fileName}";
+        var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
+        if (bytes == null) return null;
+
+        return BinaryData.FromBytes(bytes);
     }
+
+
+    #region Private methods
+    private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
+    {
+        return $"{KNOWLEDGE_FOLDER}/{KNOWLEDGE_DOC_FOLDER}/{vectorStoreProvider.CleanStr()}/{collectionName.CleanStr()}";
+    }
+    #endregion
 }

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs

@@ -28,6 +28,8 @@ public partial class TencentCosService : IFileStorageService
     private const string USER_AVATAR_FOLDER = "avatar";
     private const string SESSION_FOLDER = "sessions";
     private const string TEXT_TO_SPEECH_FOLDER = "speeches";
+    private const string KNOWLEDGE_FOLDER = "knowledgebase";
+    private const string KNOWLEDGE_DOC_FOLDER = "document";
 
     public TencentCosService(
         TencentCosSettings settings,