1. split knowledge menu

2. refine vector db
This commit is contained in:
Jicheng Lu 2024-08-20 13:32:22 -05:00
parent 5029455050
commit e96b217dfe
10 changed files with 39 additions and 31 deletions

View file

@ -25,9 +25,9 @@ public class PluginMenuDef
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<PluginMenuDef>? SubMenu { get; set; } public List<PluginMenuDef>? SubMenu { get; set; }
public PluginMenuDef(string lable, string? link = null, string? icon = null, int weight = 0) public PluginMenuDef(string label, string? link = null, string? icon = null, int weight = 0)
{ {
Label = lable; Label = label;
Link = link; Link = link;
Icon = icon; Icon = icon;
Weight = weight; Weight = weight;

View file

@ -9,7 +9,7 @@ public interface IVectorDb
Task<IEnumerable<string>> GetCollections(); Task<IEnumerable<string>> GetCollections();
Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter); Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
Task CreateCollection(string collectionName, int dim); Task CreateCollection(string collectionName, int dim);
Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null); Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
Task<bool> DeleteCollectionData(string collectionName, string id); Task<bool> DeleteCollectionData(string collectionName, Guid id);
} }

View file

@ -29,9 +29,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback
var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp; var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
await vectorDb.CreateCollection(collectionName, vector[0].Length); await vectorDb.CreateCollection(collectionName, vector[0].Length);
var id = Guid.NewGuid().ToString(); var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0],
var result = await vectorDb.Upsert(collectionName, id, vector[0], args.Question,
args.Question,
new Dictionary<string, string> new Dictionary<string, string>
{ {
{ KnowledgePayloadName.Answer, args.Answer } { KnowledgePayloadName.Answer, args.Answer }

View file

@ -20,8 +20,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase"); return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
}); });
var a = config["KnowledgeBase"];
services.AddScoped<ITextChopper, TextChopperService>(); services.AddScoped<ITextChopper, TextChopperService>();
services.AddScoped<IKnowledgeService, KnowledgeService>(); services.AddScoped<IKnowledgeService, KnowledgeService>();
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>(); services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
@ -32,7 +30,14 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
public bool AttachMenu(List<PluginMenuDef> menu) public bool AttachMenu(List<PluginMenuDef> menu)
{ {
var section = menu.First(x => x.Label == "Apps"); var section = menu.First(x => x.Label == "Apps");
menu.Add(new PluginMenuDef("Knowledge Base", link: "/page/knowledge-base", icon: "bx bx-book-open", weight: section.Weight + 1)); menu.Add(new PluginMenuDef("Knowledge Base", icon: "bx bx-book-open", weight: section.Weight + 1)
{
SubMenu = new List<PluginMenuDef>
{
new PluginMenuDef("Vector", link: "page/knowledge-base/vector"),
new PluginMenuDef("Graph", link: "page/knowledge-base/graph")
}
});
return true; return true;
} }
} }

View file

@ -53,11 +53,11 @@ public class MemoryVectorDb : IVectorDb
return await Task.FromResult(results); return await Task.FromResult(results);
} }
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null) public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{ {
_vectors[collectionName].Add(new VecRecord _vectors[collectionName].Add(new VecRecord
{ {
Id = id, Id = id.ToString(),
Vector = vector, Vector = vector,
Text = text Text = text
}); });
@ -65,7 +65,7 @@ public class MemoryVectorDb : IVectorDb
return true; return true;
} }
public async Task<bool> DeleteCollectionData(string collectionName, string id) public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{ {
return await Task.FromResult(false); return await Task.FromResult(false);
} }

View file

@ -19,8 +19,7 @@ public partial class KnowledgeService
foreach (var line in lines) foreach (var line in lines)
{ {
var vec = await textEmbedding.GetVectorAsync(line); var vec = await textEmbedding.GetVectorAsync(line);
var id = Guid.NewGuid().ToString(); await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
await db.Upsert(collectionName, id, vec, line);
index++; index++;
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
} }

View file

@ -6,8 +6,13 @@ public partial class KnowledgeService
{ {
try try
{ {
if (!Guid.TryParse(id, out var guid))
{
return false;
}
var db = GetVectorDb(); var db = GetVectorDb();
return await db.DeleteCollectionData(collectionName, id); return await db.DeleteCollectionData(collectionName, guid);
} }
catch (Exception ex) catch (Exception ex)
{ {

View file

@ -32,12 +32,12 @@ public class FaissDb : IVectorDb
throw new NotImplementedException(); throw new NotImplementedException();
} }
public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null) public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
public Task<bool> DeleteCollectionData(string collectionName, string id) public Task<bool> DeleteCollectionData(string collectionName, Guid id)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }

View file

@ -92,14 +92,14 @@ public class QdrantDb : IVectorDb
} }
} }
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null) public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{ {
// Insert vectors // Insert vectors
var point = new PointStruct() var point = new PointStruct()
{ {
Id = new PointId() Id = new PointId()
{ {
Uuid = id Uuid = id.ToString()
}, },
Vectors = vector, Vectors = vector,
Payload = Payload =
@ -137,7 +137,11 @@ public class QdrantDb : IVectorDb
return results; return results;
} }
var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); var points = await client.SearchAsync(collectionName,
vector,
limit: (ulong)limit,
scoreThreshold: confidence,
vectorsSelector: new WithVectorsSelector { Enable = withVector });
var pickFields = fields != null; var pickFields = fields != null;
foreach (var point in points) foreach (var point in points)
@ -174,15 +178,10 @@ public class QdrantDb : IVectorDb
return results; return results;
} }
public async Task<bool> DeleteCollectionData(string collectionName, string id) public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{ {
if (!Guid.TryParse(id, out var guid))
{
return false;
}
var client = GetClient(); var client = GetClient();
var result = await client.DeleteAsync(collectionName, guid); var result = await client.DeleteAsync(collectionName, id);
return result.Status == UpdateStatus.Completed; return result.Status == UpdateStatus.Completed;
} }

View file

@ -2,6 +2,7 @@ using BotSharp.Abstraction.Utilities;
using BotSharp.Abstraction.VectorStorage; using BotSharp.Abstraction.VectorStorage;
using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.Abstraction.VectorStorage.Models;
using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Memory;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -62,7 +63,7 @@ namespace BotSharp.Plugin.SemanticKernel
return resultTexts; return resultTexts;
} }
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload) public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)
{ {
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector)); await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
@ -70,13 +71,13 @@ namespace BotSharp.Plugin.SemanticKernel
return true; return true;
} }
public async Task<bool> DeleteCollectionData(string collectionName, string id) public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{ {
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
if (exist) if (exist)
{ {
await _memoryStore.RemoveAsync(collectionName, id); await _memoryStore.RemoveAsync(collectionName, id.ToString());
return true; return true;
} }
return false; return false;