1. split knowledge menu
2. refine vector db
This commit is contained in:
parent
5029455050
commit
e96b217dfe
|
|
@ -25,9 +25,9 @@ public class PluginMenuDef
|
||||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||||
public List<PluginMenuDef>? SubMenu { get; set; }
|
public List<PluginMenuDef>? SubMenu { get; set; }
|
||||||
|
|
||||||
public PluginMenuDef(string lable, string? link = null, string? icon = null, int weight = 0)
|
public PluginMenuDef(string label, string? link = null, string? icon = null, int weight = 0)
|
||||||
{
|
{
|
||||||
Label = lable;
|
Label = label;
|
||||||
Link = link;
|
Link = link;
|
||||||
Icon = icon;
|
Icon = icon;
|
||||||
Weight = weight;
|
Weight = weight;
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ public interface IVectorDb
|
||||||
Task<IEnumerable<string>> GetCollections();
|
Task<IEnumerable<string>> GetCollections();
|
||||||
Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
|
Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
|
||||||
Task CreateCollection(string collectionName, int dim);
|
Task CreateCollection(string collectionName, int dim);
|
||||||
Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
|
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
|
||||||
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
|
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
|
||||||
Task<bool> DeleteCollectionData(string collectionName, string id);
|
Task<bool> DeleteCollectionData(string collectionName, Guid id);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,9 +29,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback
|
||||||
var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
|
var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
|
||||||
await vectorDb.CreateCollection(collectionName, vector[0].Length);
|
await vectorDb.CreateCollection(collectionName, vector[0].Length);
|
||||||
|
|
||||||
var id = Guid.NewGuid().ToString();
|
var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0],
|
||||||
var result = await vectorDb.Upsert(collectionName, id, vector[0],
|
args.Question,
|
||||||
args.Question,
|
|
||||||
new Dictionary<string, string>
|
new Dictionary<string, string>
|
||||||
{
|
{
|
||||||
{ KnowledgePayloadName.Answer, args.Answer }
|
{ KnowledgePayloadName.Answer, args.Answer }
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
|
||||||
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
|
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
|
||||||
});
|
});
|
||||||
|
|
||||||
var a = config["KnowledgeBase"];
|
|
||||||
|
|
||||||
services.AddScoped<ITextChopper, TextChopperService>();
|
services.AddScoped<ITextChopper, TextChopperService>();
|
||||||
services.AddScoped<IKnowledgeService, KnowledgeService>();
|
services.AddScoped<IKnowledgeService, KnowledgeService>();
|
||||||
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
|
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
|
||||||
|
|
@ -32,7 +30,14 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
|
||||||
public bool AttachMenu(List<PluginMenuDef> menu)
|
public bool AttachMenu(List<PluginMenuDef> menu)
|
||||||
{
|
{
|
||||||
var section = menu.First(x => x.Label == "Apps");
|
var section = menu.First(x => x.Label == "Apps");
|
||||||
menu.Add(new PluginMenuDef("Knowledge Base", link: "/page/knowledge-base", icon: "bx bx-book-open", weight: section.Weight + 1));
|
menu.Add(new PluginMenuDef("Knowledge Base", icon: "bx bx-book-open", weight: section.Weight + 1)
|
||||||
|
{
|
||||||
|
SubMenu = new List<PluginMenuDef>
|
||||||
|
{
|
||||||
|
new PluginMenuDef("Vector", link: "page/knowledge-base/vector"),
|
||||||
|
new PluginMenuDef("Graph", link: "page/knowledge-base/graph")
|
||||||
|
}
|
||||||
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -53,11 +53,11 @@ public class MemoryVectorDb : IVectorDb
|
||||||
return await Task.FromResult(results);
|
return await Task.FromResult(results);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||||
{
|
{
|
||||||
_vectors[collectionName].Add(new VecRecord
|
_vectors[collectionName].Add(new VecRecord
|
||||||
{
|
{
|
||||||
Id = id,
|
Id = id.ToString(),
|
||||||
Vector = vector,
|
Vector = vector,
|
||||||
Text = text
|
Text = text
|
||||||
});
|
});
|
||||||
|
|
@ -65,7 +65,7 @@ public class MemoryVectorDb : IVectorDb
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> DeleteCollectionData(string collectionName, string id)
|
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||||
{
|
{
|
||||||
return await Task.FromResult(false);
|
return await Task.FromResult(false);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,8 +19,7 @@ public partial class KnowledgeService
|
||||||
foreach (var line in lines)
|
foreach (var line in lines)
|
||||||
{
|
{
|
||||||
var vec = await textEmbedding.GetVectorAsync(line);
|
var vec = await textEmbedding.GetVectorAsync(line);
|
||||||
var id = Guid.NewGuid().ToString();
|
await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
|
||||||
await db.Upsert(collectionName, id, vec, line);
|
|
||||||
index++;
|
index++;
|
||||||
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
|
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,13 @@ public partial class KnowledgeService
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
if (!Guid.TryParse(id, out var guid))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
var db = GetVectorDb();
|
var db = GetVectorDb();
|
||||||
return await db.DeleteCollectionData(collectionName, id);
|
return await db.DeleteCollectionData(collectionName, guid);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -32,12 +32,12 @@ public class FaissDb : IVectorDb
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||||
{
|
{
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<bool> DeleteCollectionData(string collectionName, string id)
|
public Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||||
{
|
{
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -92,14 +92,14 @@ public class QdrantDb : IVectorDb
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||||
{
|
{
|
||||||
// Insert vectors
|
// Insert vectors
|
||||||
var point = new PointStruct()
|
var point = new PointStruct()
|
||||||
{
|
{
|
||||||
Id = new PointId()
|
Id = new PointId()
|
||||||
{
|
{
|
||||||
Uuid = id
|
Uuid = id.ToString()
|
||||||
},
|
},
|
||||||
Vectors = vector,
|
Vectors = vector,
|
||||||
Payload =
|
Payload =
|
||||||
|
|
@ -137,7 +137,11 @@ public class QdrantDb : IVectorDb
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
|
var points = await client.SearchAsync(collectionName,
|
||||||
|
vector,
|
||||||
|
limit: (ulong)limit,
|
||||||
|
scoreThreshold: confidence,
|
||||||
|
vectorsSelector: new WithVectorsSelector { Enable = withVector });
|
||||||
|
|
||||||
var pickFields = fields != null;
|
var pickFields = fields != null;
|
||||||
foreach (var point in points)
|
foreach (var point in points)
|
||||||
|
|
@ -174,15 +178,10 @@ public class QdrantDb : IVectorDb
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> DeleteCollectionData(string collectionName, string id)
|
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||||
{
|
{
|
||||||
if (!Guid.TryParse(id, out var guid))
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
var client = GetClient();
|
var client = GetClient();
|
||||||
var result = await client.DeleteAsync(collectionName, guid);
|
var result = await client.DeleteAsync(collectionName, id);
|
||||||
return result.Status == UpdateStatus.Completed;
|
return result.Status == UpdateStatus.Completed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ using BotSharp.Abstraction.Utilities;
|
||||||
using BotSharp.Abstraction.VectorStorage;
|
using BotSharp.Abstraction.VectorStorage;
|
||||||
using BotSharp.Abstraction.VectorStorage.Models;
|
using BotSharp.Abstraction.VectorStorage.Models;
|
||||||
using Microsoft.SemanticKernel.Memory;
|
using Microsoft.SemanticKernel.Memory;
|
||||||
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
|
@ -62,7 +63,7 @@ namespace BotSharp.Plugin.SemanticKernel
|
||||||
return resultTexts;
|
return resultTexts;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload)
|
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)
|
||||||
{
|
{
|
||||||
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
|
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
|
||||||
await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
|
await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
|
||||||
|
|
@ -70,13 +71,13 @@ namespace BotSharp.Plugin.SemanticKernel
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> DeleteCollectionData(string collectionName, string id)
|
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||||
{
|
{
|
||||||
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
|
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
|
||||||
|
|
||||||
if (exist)
|
if (exist)
|
||||||
{
|
{
|
||||||
await _memoryStore.RemoveAsync(collectionName, id);
|
await _memoryStore.RemoveAsync(collectionName, id.ToString());
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue