From a17bca4430308edff5538b246f48c5f316e1f543 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 16 Oct 2024 17:52:18 +0800 Subject: [PATCH 01/50] fix: Update to activated status after successfully forgetting password --- .../Repository/MongoRepository.User.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 55c454bd..c064856c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -111,7 +111,8 @@ public partial class MongoRepository { var filter = Builders.Filter.Eq(x => x.Id, userId); var update = Builders.Update.Set(x => x.Password, password) - .Set(x => x.UpdatedTime, DateTime.UtcNow); + .Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.Verified, true); _dc.Users.UpdateOne(filter, update); } From cb7285f33693a7919907a5832469d24a49d6051e Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Fri, 18 Oct 2024 15:43:34 +0800 Subject: [PATCH 02/50] add RegionCode --- .../BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs index ffcd71a2..d76ec389 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs @@ -18,6 +18,7 @@ public class UserDocument : MongoBase public string Role { get; set; } = null!; public string? VerificationCode { get; set; } public bool Verified { get; set; } + public string? RegionCode { get; set; } public string? AffiliateId { get; set; } public bool IsDisabled { get; set; } public DateTime CreatedTime { get; set; } From d481928f168916d6e2852fa5774302fe8ede4b75 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 18 Oct 2024 11:32:41 -0500 Subject: [PATCH 03/50] Product V3 --- .../Browsing/Models/WebPageResponseFilter.cs | 10 +++++++ .../PlaywrightDriver/PlaywrightInstance.cs | 28 +++++++++++-------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs index cd97a01d..935d2282 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs @@ -4,4 +4,14 @@ public class WebPageResponseFilter { public string Url { get; set; } = null!; public string[]? QueryParameters { get; set; } + + /// + /// contains, starts, ends, equals + /// + public string UrlMatchPattern { get; set; } = "contains"; + + /// + /// Handle Content-Type: text/x-component + /// + public int PartIndex { get; set; } = -1; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 4a67ffeb..2bbb0acf 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -146,37 +146,41 @@ public class PlaywrightInstance : IDisposable { if (e.Status != 204 && e.Headers.ContainsKey("content-type") && - e.Headers["content-type"].Contains("application/json") && (e.Request.ResourceType == "fetch" || e.Request.ResourceType == "xhr") && (excludeResponseUrls == null || !excludeResponseUrls.Any(url => e.Url.ToLower().Contains(url))) && (includeResponseUrls == null || includeResponseUrls.Any(url => e.Url.ToLower().Contains(url)))) { Serilog.Log.Information($"{e.Request.Method}: {e.Url}"); - JsonElement? json = null; + try { - if (e.Status == 200 && e.Ok) - { - json = await e.JsonAsync(); - } - else - { - Serilog.Log.Warning($"Response status: {e.Status} {e.StatusText}, OK: {e.Ok}"); - } - var result = new WebPageResponseData { Url = e.Url.ToLower(), PostData = e.Request?.PostData ?? string.Empty, - ResponseData = JsonSerializer.Serialize(json), ResponseInMemory = responseInMemory }; + if (e.Headers["content-type"].Contains("application/json")) + { + if (e.Status == 200 && e.Ok) + { + var json = await e.JsonAsync(); + result.ResponseData = JsonSerializer.Serialize(json); + } + } + else + { + var html = await e.TextAsync(); + result.ResponseData = html; + } + if (responseContainer != null && responseInMemory) { responseContainer.Add(result); } + Serilog.Log.Warning($"Response status: {e.Status} {e.StatusText}, OK: {e.Ok}"); var webPageResponseHooks = _services.GetServices(); foreach (var hook in webPageResponseHooks) { From 14bd97d088ed187f1ebd5b03f5145c0ed241bc31 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 18 Oct 2024 14:21:11 -0500 Subject: [PATCH 04/50] Set default Direction as 'down' --- .../BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index aa4154ff..0964389b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -7,7 +7,7 @@ public class PageActionArgs public BroswerActionEnum Action { get; set; } public string? Content { get; set; } - public string? Direction { get; set; } + public string Direction { get; set; } = "down"; public string Url { get; set; } = null!; From 91250af6ffef6d91c395622d89fb9fc6482c0647 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 18 Oct 2024 18:36:48 -0500 Subject: [PATCH 05/50] PartSearch --- .../Browsing/Models/WebPageResponseData.cs | 5 +++++ .../Browsing/Models/WebPageResponseFilter.cs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs index 1d03b0b7..14932118 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs @@ -7,4 +7,9 @@ public class WebPageResponseData public string ResponseData { get; set; } = null!; public bool ResponseInMemory { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + + public override string ToString() + { + return $"{Url} {ResponseData.Length}"; + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs index 935d2282..d9561847 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs @@ -13,5 +13,5 @@ public class WebPageResponseFilter /// /// Handle Content-Type: text/x-component /// - public int PartIndex { get; set; } = -1; + public Func? PartSearch { get; set; } = null; } From b4800efb6dd5a44c43da0fa279a52ce11693ccf3 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Sat, 19 Oct 2024 17:16:55 +0800 Subject: [PATCH 06/50] add affiliate employee --- .../Repositories/IBotSharpRepository.cs | 2 +- .../Users/IUserIdentity.cs | 3 ++ .../Users/IUserService.cs | 1 + .../BotSharp.Abstraction/Users/Models/User.cs | 1 + .../FileRepository/FileRepository.User.cs | 4 +- .../Users/Services/UserIdentity.cs | 9 ++++ .../Users/Services/UserService.cs | 54 ++++++++++++------- .../Repository/MongoRepository.User.cs | 8 +-- 8 files changed, 56 insertions(+), 26 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index de351649..abb144e4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -24,7 +24,7 @@ public interface IBotSharpRepository User? GetAffiliateUserByPhone(string phone) => throw new NotImplementedException(); User? GetUserById(string id) => throw new NotImplementedException(); List GetUserByIds(List ids) => throw new NotImplementedException(); - User? GetUserByAffiliateId(string affiliateId) => throw new NotImplementedException(); + List GetUsersByAffiliateId(string affiliateId) => throw new NotImplementedException(); User? GetUserByUserName(string userName) => throw new NotImplementedException(); void CreateUser(User user) => throw new NotImplementedException(); void UpdateUserVerified(string userId) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs index 347e7554..2c51b7b0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs @@ -11,4 +11,7 @@ public interface IUserIdentity string? UserLanguage { get; } string? Phone { get; } string? AffiliateId { get; } + string? EmployeeId { get; } + string Type { get; } + string Role { get; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 5a3315b7..1572771f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -9,6 +9,7 @@ public interface IUserService Task CreateUser(User user); Task ActiveUser(UserActivationModel model); Task GetAffiliateToken(string authorization); + Task GetAdminToken(string authorization); Task GetToken(string authorization); Task GetMyProfile(); Task VerifyUserNameExisting(string userName); diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs index cb48701f..69e41eca 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs @@ -22,6 +22,7 @@ public class User public string? VerificationCode { get; set; } public bool Verified { get; set; } public string? AffiliateId { get; set; } + public string? EmployeeId { get; set; } public bool IsDisabled { get; set; } public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; public DateTime CreatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs index c8eb28dc..42d56d98 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs @@ -31,9 +31,9 @@ public partial class FileRepository return Users.Where(x => ids.Contains(x.Id) || (x.ExternalId != null && ids.Contains(x.ExternalId)))?.ToList() ?? new List(); } - public User? GetUserByAffiliateId(string affiliateId) + public List GetUsersByAffiliateId(string affiliateId) { - return Users.FirstOrDefault(x => x.AffiliateId == affiliateId); + return Users.Where(x => x.AffiliateId == affiliateId).ToList(); } public User? GetUserByUserName(string userName = null) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index 531fd286..9b7615af 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -72,4 +72,13 @@ public class UserIdentity : IUserIdentity [JsonPropertyName("affiliateId")] public string? AffiliateId => _claims?.FirstOrDefault(x => x.Type == "affiliateId")?.Value; + + [JsonPropertyName("employeeId")] + public string? EmployeeId => _claims?.FirstOrDefault(x => x.Type == "employeeId")?.Value; + + [JsonPropertyName("type")] + public string? Type => _claims?.FirstOrDefault(x => x.Type == "type")?.Value; + + [JsonPropertyName("role")] + public string? Role => _claims?.FirstOrDefault(x => x.Type == "role")?.Value; } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index bc6fc317..ef5a3d77 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -106,19 +106,14 @@ public class UserService : IUserService return true; } - public async Task GetAffiliateToken(string authorization) + public async Task GetAffiliateToken(string authorization) { var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization)); var (id, password) = base64.SplitAsTuple(":"); var db = _services.GetRequiredService(); var record = db.GetAffiliateUserByPhone(id); - if (record == null) - { - record = db.GetUserByPhone(id); - } - - var isCanLoginAffiliateRoleType = record != null && !record.IsDisabled && record.Type != UserType.Client; - if (!isCanLoginAffiliateRoleType) + var isCanLogin = record != null && !record.IsDisabled && record.Type == UserType.Affiliate; + if (!isCanLogin) { return default; } @@ -128,6 +123,35 @@ public class UserService : IUserService return default; } + return await Task.FromResult(BuildToken(record)); + } + + public async Task GetAdminToken(string authorization) + { + var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization)); + var (id, password) = base64.SplitAsTuple(":"); + var db = _services.GetRequiredService(); + var record = db.GetUserByPhone(id); + var isCanLogin = record != null && !record.IsDisabled + && record.Type == UserType.Internal && new List + { + UserRole.Root,UserRole.Admin + }.Contains(record.Role); + if (!isCanLogin) + { + return default; + } + + if (Utilities.HashTextMd5($"{password}{record.Salt}") != record.Password) + { + return default; + } + + return await Task.FromResult(BuildToken(record)); + } + + private Token BuildToken(User record) + { var accessToken = GenerateJwtToken(record); var jwt = new JwtSecurityTokenHandler().ReadJwtToken(accessToken); var token = new Token @@ -225,16 +249,7 @@ public class UserService : IUserService return default; } - var accessToken = GenerateJwtToken(record); - var jwt = new JwtSecurityTokenHandler().ReadJwtToken(accessToken); - var token = new Token - { - AccessToken = accessToken, - ExpireTime = jwt.Payload.Exp.Value, - TokenType = "Bearer", - Scope = "api" - }; - + var token = BuildToken(record); foreach (var hook in hooks) { hook.BeforeSending(token); @@ -258,7 +273,8 @@ public class UserService : IUserService new Claim("role", user.Role ?? UserRole.User), new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), new Claim("phone", user.Phone ?? string.Empty), - new Claim("affiliateId", user.AffiliateId ?? string.Empty) + new Claim("affiliateId", user.AffiliateId ?? string.Empty), + new Claim("employeeId", user.EmployeeId ?? string.Empty) }; var validators = _services.GetServices(); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 9b6b62c1..f9e1a466 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -37,11 +37,11 @@ public partial class MongoRepository return users?.Any() == true ? users.Select(x => x.ToUser()).ToList() : new List(); } - public User? GetUserByAffiliateId(string affiliateId) + public List GetUsersByAffiliateId(string affiliateId) { - var user = _dc.Users.AsQueryable() - .FirstOrDefault(x => x.AffiliateId == affiliateId); - return user != null ? user.ToUser() : null; + var users = _dc.Users.AsQueryable() + .Where(x => x.AffiliateId == affiliateId).ToList(); + return users?.Any() == true ? users.Select(x => x.ToUser()).ToList() : new List(); } public User? GetUserByUserName(string userName) From 0623f3dbefc83c19229b797594890b80b2389856 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Sat, 19 Oct 2024 18:49:18 +0800 Subject: [PATCH 07/50] fix issue by self test --- .../BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs | 3 ++- .../Repository/MongoRepository.User.cs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs index ffcd71a2..8526788b 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs @@ -19,9 +19,10 @@ public class UserDocument : MongoBase public string? VerificationCode { get; set; } public bool Verified { get; set; } public string? AffiliateId { get; set; } + public string? EmployeeId { get; set; } public bool IsDisabled { get; set; } public DateTime CreatedTime { get; set; } - public DateTime UpdatedTime { get; set; } + public DateTime UpdatedTime { get; set; } public User ToUser() { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index f9e1a466..ffdc78e6 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -71,6 +71,7 @@ public partial class MongoRepository VerificationCode = user.VerificationCode, Verified = user.Verified, AffiliateId = user.AffiliateId, + EmployeeId = user.EmployeeId, IsDisabled = user.IsDisabled, CreatedTime = DateTime.UtcNow, UpdatedTime = DateTime.UtcNow From bf84d4a87ada08dca81d53ad6b08ed2f7fdcc3f3 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 19 Oct 2024 10:06:24 -0500 Subject: [PATCH 08/50] UserAuthenticated --- .../BotSharp.Abstraction.csproj | 1 + .../Users/IAuthenticationHook.cs | 49 ++++++++++++++++++- .../Users/Services/UserService.cs | 2 +- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index e94cf839..474dcf56 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -32,6 +32,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs index afb0e40e..36b07f11 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs @@ -1,15 +1,60 @@ using BotSharp.Abstraction.Users.Models; +using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; namespace BotSharp.Abstraction.Users; public interface IAuthenticationHook { + /// + /// Interupt the authentication process, and return the user object if the user is authenticated + /// + /// + /// + /// Task Authenticate(string id, string password); - void AddClaims(List claims); - void BeforeSending(Token token); + + /// + /// Add extra claims to user + /// + /// + /// + bool AddClaims(List claims) + => true; + + /// + /// User authenticated successfully + /// + /// + /// + bool UserAuthenticated(JwtSecurityToken token) + => true; + + /// + /// Bfore user updating + /// + /// + /// Task UserUpdating(User user); + + /// + /// After user created + /// + /// + /// Task UserCreated(User user); + + /// + /// Reset password + /// + /// + /// Task VerificationCodeResetPassword(User user); + + /// + /// Delete users + /// + /// + /// Task DelUsers(List userIds); } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 10dbe8de..5d08c51f 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -275,7 +275,7 @@ public class UserService : IUserService foreach (var hook in hooks) { - hook.BeforeSending(token); + hook.UserAuthenticated(jwt); } return token; From 780718ef978c9495ce420a403d42ccccc974fa93 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 19 Oct 2024 12:39:42 -0500 Subject: [PATCH 09/50] ScrollPage --- .../PlaywrightWebDriver.ScrollPage.cs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs index c50bf48b..48538ce8 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -12,12 +12,20 @@ public partial class PlaywrightWebDriver if (args.Direction == "down") { // Get the total page height - int scrollY = await page.EvaluateAsync("document.body.scrollHeight"); + int scrollY = await page.EvaluateAsync("window.screen.height"); - // Scroll to the bottom + // Scroll a page down await page.Mouse.WheelAsync(0, scrollY); } else if (args.Direction == "up") + { + // Get the total page height + int scrollY = await page.EvaluateAsync("window.screen.height"); + + // Scroll a page up + await page.Mouse.WheelAsync(0, -scrollY); + } + else if (args.Direction == "bottom") { // Get the total page height int scrollY = await page.EvaluateAsync("document.body.scrollHeight"); @@ -25,6 +33,14 @@ public partial class PlaywrightWebDriver // Scroll to the bottom await page.Mouse.WheelAsync(0, -scrollY); } + else if (args.Direction == "top") + { + // Get the total page height + int scrollY = await page.EvaluateAsync("document.body.scrollHeight"); + + // Scroll to the top + await page.Mouse.WheelAsync(0, -scrollY); + } else if (args.Direction == "left") { await page.EvaluateAsync(@" From a16b1e435881a86de0bb8bb9eca7209878bfbf74 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Sun, 20 Oct 2024 13:02:13 +0800 Subject: [PATCH 10/50] fix issue by self test --- .../BotSharp.Core/Users/Services/UserIdentity.cs | 2 +- .../BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index 9b7615af..9ee2238b 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -80,5 +80,5 @@ public class UserIdentity : IUserIdentity public string? Type => _claims?.FirstOrDefault(x => x.Type == "type")?.Value; [JsonPropertyName("role")] - public string? Role => _claims?.FirstOrDefault(x => x.Type == "role")?.Value; + public string? Role => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Role)?.Value; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs index 8526788b..a3d84e7d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs @@ -22,7 +22,7 @@ public class UserDocument : MongoBase public string? EmployeeId { get; set; } public bool IsDisabled { get; set; } public DateTime CreatedTime { get; set; } - public DateTime UpdatedTime { get; set; } + public DateTime UpdatedTime { get; set; } public User ToUser() { @@ -41,6 +41,7 @@ public class UserDocument : MongoBase Type = Type, Role = Role, AffiliateId = AffiliateId, + EmployeeId = EmployeeId, IsDisabled = IsDisabled, VerificationCode = VerificationCode, Verified = Verified, From 78f5cd1751ecea360fb4ae8bdbc2c6982d569d8d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 20 Oct 2024 15:24:21 +0000 Subject: [PATCH 11/50] category --- .../Browsing/Models/ElementLocatingArgs.cs | 3 +++ .../Browsing/Models/WebPageResponseFilter.cs | 3 +++ .../PlaywrightWebDriver.LocateElement.cs | 10 +++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs index 53d72520..b50f2366 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs @@ -1,5 +1,8 @@ +using System.Diagnostics; + namespace BotSharp.Abstraction.Browsing.Models; +[DebuggerStepThrough] public class ElementLocatingArgs { [JsonPropertyName("match_rule")] diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs index d9561847..54951115 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs @@ -1,5 +1,8 @@ +using System.Diagnostics; + namespace BotSharp.Abstraction.Browsing.Models; +[DebuggerStepThrough] public class WebPageResponseFilter { public string Url { get; set; } = null!; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs index a39da9c0..ff43864e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs @@ -128,7 +128,15 @@ public partial class PlaywrightWebDriver } else { - result.Selector = locator.ToString().Split("Locator@").Last(); + foreach (var element in await locator.AllAsync()) + { + var html = await element.InnerHTMLAsync(); + _logger.LogWarning(html); + // fix if html has & + result.Body = HttpUtility.HtmlDecode(html); + break; + } + result.IsSuccess = true; } } From 3e1fd96ef861c4aa3a6d3eb501ac02ce04430d07 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Mon, 21 Oct 2024 19:59:05 +0800 Subject: [PATCH 12/50] fix: Update CreateUser API --- .../BotSharp.Abstraction/Users/Models/User.cs | 1 + .../BotSharp.Core/Users/Services/UserService.cs | 3 ++- .../ViewModels/Users/UserCreationModel.cs | 11 ++++++----- .../Collections/UserDocument.cs | 2 ++ .../Repository/MongoRepository.User.cs | 5 ++++- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs index cb48701f..29577f7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs @@ -21,6 +21,7 @@ public class User public string Role { get; set; } = UserRole.User; public string? VerificationCode { get; set; } public bool Verified { get; set; } + public string? RegionCode { get; set; } public string? AffiliateId { get; set; } public bool IsDisabled { get; set; } public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 5d08c51f..4ee5970e 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -91,7 +91,8 @@ public class UserService : IUserService record.Email = user.Email?.ToLower(); if (!string.IsNullOrWhiteSpace(user.Phone)) { - record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value; + //record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value; + record.Phone = Regex.Match(user.Phone, @"\d+").Value; } record.Salt = Guid.NewGuid().ToString("N"); record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}"); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs index b157ef59..f2d9f048 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs @@ -5,18 +5,18 @@ namespace BotSharp.OpenAPI.ViewModels.Users; public class UserCreationModel { public string? UserName { get; set; } - public string FirstName { get; set; } = string.Empty; + public string FirstName { get; set; } = string.Empty; public string? LastName { get; set; } public string? Email { get; set; } public string? Phone { get; set; } public string Password { get; set; } = string.Empty; public string Type { get; set; } = UserType.Client; public string Role { get; set; } = UserRole.User; - + public string? RegionCode { get; set; } public User ToUser() { - return new User - { + return new User + { UserName = UserName, FirstName = FirstName, LastName = LastName, @@ -24,7 +24,8 @@ public class UserCreationModel Phone = Phone, Password = Password, Role = Role, - Type = Type + Type = Type, + RegionCode = RegionCode }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs index ffcd71a2..1e6f6655 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs @@ -18,6 +18,7 @@ public class UserDocument : MongoBase public string Role { get; set; } = null!; public string? VerificationCode { get; set; } public bool Verified { get; set; } + public string? RegionCode { get; set; } public string? AffiliateId { get; set; } public bool IsDisabled { get; set; } public DateTime CreatedTime { get; set; } @@ -43,6 +44,7 @@ public class UserDocument : MongoBase IsDisabled = IsDisabled, VerificationCode = VerificationCode, Verified = Verified, + RegionCode = RegionCode, }; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index c064856c..b62c0eb2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -70,6 +70,7 @@ public partial class MongoRepository Type = user.Type, VerificationCode = user.VerificationCode, Verified = user.Verified, + RegionCode = user.RegionCode, AffiliateId = user.AffiliateId, IsDisabled = user.IsDisabled, CreatedTime = DateTime.UtcNow, @@ -87,7 +88,9 @@ public partial class MongoRepository .Set(x => x.Phone, user.Phone) .Set(x => x.Salt, user.Salt) .Set(x => x.Password, user.Password) - .Set(x => x.VerificationCode, user.VerificationCode); + .Set(x => x.VerificationCode, user.VerificationCode) + .Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.RegionCode, user.RegionCode); _dc.Users.UpdateOne(filter, update); } From 41709c95dddc9cf3a84bee106ee93feea40dcd67 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 21 Oct 2024 11:29:04 -0500 Subject: [PATCH 13/50] Fix compile issue. --- .../BotSharp.Core/Users/Services/UserService.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 9c1c8723..4e7c1248 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -161,7 +161,9 @@ public class UserService : IUserService return default; } - return await Task.FromResult(BuildToken(record)); + var (token, jwt) = BuildToken(record); + + return await Task.FromResult(token); } public async Task GetAdminToken(string authorization) @@ -185,10 +187,12 @@ public class UserService : IUserService return default; } - return await Task.FromResult(BuildToken(record)); + var (token, jwt) = BuildToken(record); + + return await Task.FromResult(token); } - private Token BuildToken(User record) + private (Token, JwtSecurityToken) BuildToken(User record) { var accessToken = GenerateJwtToken(record); var jwt = new JwtSecurityTokenHandler().ReadJwtToken(accessToken); @@ -199,7 +203,7 @@ public class UserService : IUserService TokenType = "Bearer", Scope = "api" }; - return token; + return (token, jwt); } public async Task GetToken(string authorization) @@ -287,7 +291,7 @@ public class UserService : IUserService return default; } - var token = BuildToken(record); + var (token, jwt) = BuildToken(record); foreach (var hook in hooks) { hook.UserAuthenticated(jwt); From f5d153f7d5af619d6682b50a6e7b9a76ff3e0eed Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 21 Oct 2024 14:46:09 -0500 Subject: [PATCH 14/50] add redis cache folder --- .../Infrastructures/SharpCacheAttribute.cs | 2 +- .../Users/Services/UserService.cs | 2 +- .../Filters/UserSingleLoginFilter.cs | 116 +++++++++--------- 3 files changed, 59 insertions(+), 61 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs index dd30f491..c8b29ecd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs @@ -71,7 +71,7 @@ public class SharpCacheAttribute : MoAttribute private string GetCacheKey(SharpCacheSettings settings, MethodContext context) { - var key = settings.Prefix + "-" + context.Method.Name; + var key = settings.Prefix + ":" + context.Method.Name; foreach (var arg in context.Arguments) { if (arg is null) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 4e7c1248..e565a9b5 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -359,7 +359,7 @@ public class UserService : IUserService private string GetUserTokenExpiresCacheKey(string userId) { - return $"user_{userId}_token_expires"; + return $"user:{userId}_token_expires"; } public async Task GetUserTokenExpires() diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs index e8db0801..dda71f48 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs @@ -1,80 +1,78 @@ using BotSharp.Abstraction.Users.Settings; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.Extensions.Configuration; using Microsoft.Net.Http.Headers; using System.IdentityModel.Tokens.Jwt; -namespace BotSharp.OpenAPI.Filters -{ - public class UserSingleLoginFilter : IAuthorizationFilter - { - private readonly IUserService _userService; - private readonly IServiceProvider _services; +namespace BotSharp.OpenAPI.Filters; - public UserSingleLoginFilter(IUserService userService, IServiceProvider services) +public class UserSingleLoginFilter : IAuthorizationFilter +{ + private readonly IUserService _userService; + private readonly IServiceProvider _services; + + public UserSingleLoginFilter(IUserService userService, IServiceProvider services) + { + _userService = userService; + _services = services; + } + + public void OnAuthorization(AuthorizationFilterContext context) + { + var isAllowAnonymous = context.ActionDescriptor.EndpointMetadata + .Any(em => em.GetType() == typeof(AllowAnonymousAttribute)); + + if (isAllowAnonymous) { - _userService = userService; - _services = services; + return; } - public void OnAuthorization(AuthorizationFilterContext context) + var bearerToken = GetBearerToken(context); + if (!string.IsNullOrWhiteSpace(bearerToken)) { - var isAllowAnonymous = context.ActionDescriptor.EndpointMetadata - .Any(em => em.GetType() == typeof(AllowAnonymousAttribute)); + var config = _services.GetRequiredService(); + var token = GetJwtToken(bearerToken); - if (isAllowAnonymous) + if (config.AllowMultipleDeviceLoginUserIds.Contains(token.Claims.First(x => x.Type == "nameid").Value)) { return; } - var bearerToken = GetBearerToken(context); - if (!string.IsNullOrWhiteSpace(bearerToken)) + var validTo = token.ValidTo.ToLongTimeString(); + var currentExpires = GetUserExpires().ToLongTimeString(); + + if (validTo != currentExpires) { - var config = _services.GetRequiredService(); - var token = GetJwtToken(bearerToken); - - if (config.AllowMultipleDeviceLoginUserIds.Contains(token.Claims.First(x => x.Type == "nameid").Value)) - { - return; - } - - var validTo = token.ValidTo.ToLongTimeString(); - var currentExpires = GetUserExpires().ToLongTimeString(); - - if (validTo != currentExpires) - { - Serilog.Log.Warning($"Token expired. Token expires at {validTo}, current expires at {currentExpires}"); - // login confict - context.Result = new ConflictResult(); - } + Serilog.Log.Warning($"Token expired. Token expires at {validTo}, current expires at {currentExpires}"); + // login confict + context.Result = new ConflictResult(); } } - - private string GetBearerToken(AuthorizationFilterContext context) - { - if (context.HttpContext.Request.Headers.TryGetValue(HeaderNames.Authorization, out var bearerToken) - && !string.IsNullOrWhiteSpace(bearerToken.ToString())) - { - var tokenType = bearerToken.ToString().Split(" ").First(); - if (tokenType == JwtBearerDefaults.AuthenticationScheme) - { - return bearerToken.ToString().Split(" ").Last(); - } - } - return null; - } - - private JwtSecurityToken GetJwtToken(string jwtToken) - { - var handler = new JwtSecurityTokenHandler(); - var token = handler.ReadJwtToken(jwtToken); - return token; - } - - private DateTime GetUserExpires() - { - return _userService.GetUserTokenExpires().GetAwaiter().GetResult(); - } + } + + private string GetBearerToken(AuthorizationFilterContext context) + { + if (context.HttpContext.Request.Headers.TryGetValue(HeaderNames.Authorization, out var bearerToken) + && !string.IsNullOrWhiteSpace(bearerToken.ToString())) + { + var tokenType = bearerToken.ToString().Split(" ").First(); + if (tokenType == JwtBearerDefaults.AuthenticationScheme) + { + return bearerToken.ToString().Split(" ").Last(); + } + } + return null; + } + + private JwtSecurityToken GetJwtToken(string jwtToken) + { + var handler = new JwtSecurityTokenHandler(); + var token = handler.ReadJwtToken(jwtToken); + return token; + } + + private DateTime GetUserExpires() + { + return _userService.GetUserTokenExpires().GetAwaiter().GetResult(); } } From 75bf0425070fff623309836943938f0ffd427b36 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Tue, 22 Oct 2024 17:43:15 +0800 Subject: [PATCH 15/50] fix: add RegionCode --- .../ViewModels/Users/UserResetPasswordModel.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs index 8a58820a..df0601c5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs @@ -1,5 +1,3 @@ -using System.Data; - namespace BotSharp.OpenAPI.ViewModels.Users; public class UserResetPasswordModel @@ -8,6 +6,7 @@ public class UserResetPasswordModel public string? Phone { get; set; } public string Password { get; set; } = string.Empty; public string VerificationCode { get; set; } + public string? RegionCode { get; set; } public User ToUser() { From 4b2a94a5391595b4bb35b7165910d2b7ec658714 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Tue, 22 Oct 2024 18:43:34 +0800 Subject: [PATCH 16/50] fix: add regionCode for ModifyUserPhone API --- .../Repositories/IBotSharpRepository.cs | 2 +- .../BotSharp.Abstraction/Users/IUserService.cs | 2 +- .../BotSharp.Core/Users/Services/UserService.cs | 11 ++++++++--- .../BotSharp.OpenAPI/Controllers/UserController.cs | 4 ++-- .../Repository/MongoRepository.User.cs | 5 +++-- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 4ef68d88..003eddf4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -32,7 +32,7 @@ public interface IBotSharpRepository void UpdateUserVerificationCode(string userId, string verficationCode) => throw new NotImplementedException(); void UpdateUserPassword(string userId, string password) => throw new NotImplementedException(); void UpdateUserEmail(string userId, string email) => throw new NotImplementedException(); - void UpdateUserPhone(string userId, string Iphone) => throw new NotImplementedException(); + void UpdateUserPhone(string userId, string Iphone, string regionCode) => throw new NotImplementedException(); void UpdateUserIsDisable(string userId, bool isDisable) => throw new NotImplementedException(); void UpdateUsersIsDisable(List userIds, bool isDisable) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index d5547051..8123b272 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -19,7 +19,7 @@ public interface IUserService Task SendVerificationCodeResetPasswordLogin(); Task ResetUserPassword(User user); Task ModifyUserEmail(string email); - Task ModifyUserPhone(string phone); + Task ModifyUserPhone(string phone, string regionCode); Task UpdatePassword(string newPassword, string verificationCode); Task GetUserTokenExpires(); Task UpdateUsersIsDisable(List userIds, bool isDisable); diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 43e2dfb7..a56c57a8 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -189,7 +189,7 @@ public class UserService : IUserService } var (token, jwt) = BuildToken(record); - + return await Task.FromResult(token); } @@ -618,8 +618,12 @@ public class UserService : IUserService return true; } - public async Task ModifyUserPhone(string phone) + public async Task ModifyUserPhone(string phone, string regionCode) { + if (string.IsNullOrWhiteSpace(regionCode)) + { + throw new Exception("regionCode is required"); + } var curUser = await GetMyProfile(); var db = _services.GetRequiredService(); var record = db.GetUserById(curUser.Id); @@ -631,6 +635,7 @@ public class UserService : IUserService } record.Phone = phone; + record.RegionCode = regionCode; var hooks = _services.GetServices(); foreach (var hook in hooks) @@ -638,7 +643,7 @@ public class UserService : IUserService await hook.UserUpdating(record); } - db.UpdateUserPhone(record.Id, record.Phone); + db.UpdateUserPhone(record.Id, record.Phone, regionCode); return true; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 67c0f4e3..253c19be 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -153,9 +153,9 @@ public class UserController : ControllerBase } [HttpPost("/user/phone/modify")] - public async Task ModifyUserPhone([FromQuery] string phone) + public async Task ModifyUserPhone([FromQuery] string phone, [FromQuery] string regionCode) { - return await _userService.ModifyUserPhone(phone); + return await _userService.ModifyUserPhone(phone, regionCode); } [HttpPost("/user/update/isdisable")] diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index cf33312c..7b1f66db 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -128,11 +128,12 @@ public partial class MongoRepository _dc.Users.UpdateOne(filter, update); } - public void UpdateUserPhone(string userId, string phone) + public void UpdateUserPhone(string userId, string phone, string regionCode) { var filter = Builders.Filter.Eq(x => x.Id, userId); var update = Builders.Update.Set(x => x.Phone, phone) - .Set(x => x.UpdatedTime, DateTime.UtcNow); + .Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.RegionCode, regionCode); _dc.Users.UpdateOne(filter, update); } From 0f930b4c46c3260d914a0e202d74835877f0eda6 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Tue, 22 Oct 2024 18:56:00 +0800 Subject: [PATCH 17/50] fix: update ModifyUserPhone API --- .../BotSharp.Core/Users/Services/UserService.cs | 2 ++ .../Repository/MongoRepository.User.cs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index a56c57a8..c48001bf 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -636,6 +636,8 @@ public class UserService : IUserService record.Phone = phone; record.RegionCode = regionCode; + record.UserName = phone; + record.FirstName = phone; var hooks = _services.GetServices(); foreach (var hook in hooks) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 7b1f66db..1d0e2596 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -133,7 +133,9 @@ public partial class MongoRepository var filter = Builders.Filter.Eq(x => x.Id, userId); var update = Builders.Update.Set(x => x.Phone, phone) .Set(x => x.UpdatedTime, DateTime.UtcNow) - .Set(x => x.RegionCode, regionCode); + .Set(x => x.RegionCode, regionCode) + .Set(x => x.UserName, phone) + .Set(x => x.FirstName, phone); _dc.Users.UpdateOne(filter, update); } From 0fbe0e6f0cdf45662eb34a4bf4ef071baf21996c Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 23 Oct 2024 14:57:57 +0800 Subject: [PATCH 18/50] feat: add RegionCode by UserIdentity. --- .../BotSharp.Abstraction/Users/IUserIdentity.cs | 1 + .../BotSharp.Core/Users/Services/UserIdentity.cs | 5 ++++- .../BotSharp.Core/Users/Services/UserService.cs | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs index 47e70e42..f42b7f25 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs @@ -17,4 +17,5 @@ public interface IUserIdentity string? EmployeeId { get; } string Type { get; } string Role { get; } + string? RegionCode { get; } } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index 6ddeae03..965be261 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -63,7 +63,7 @@ public class UserIdentity : IUserIdentity get { _contextAccessor.HttpContext.Request.Headers.TryGetValue("User-Language", out var languages); - return languages.FirstOrDefault() ?? "en-US"; + return languages.FirstOrDefault() ?? "en-US"; } } @@ -81,4 +81,7 @@ public class UserIdentity : IUserIdentity [JsonPropertyName("role")] public string? Role => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Role)?.Value; + + [JsonPropertyName("regionCode")] + public string? RegionCode => _claims?.FirstOrDefault(x => x.Type == "regionCode")?.Value; } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index c48001bf..dbd8b696 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -267,7 +267,8 @@ public class UserService : IUserService ExternalId = user.ExternalId, Password = user.Password, Type = user.Type, - Role = user.Role + Role = user.Role, + RegionCode = user.RegionCode }; await CreateUser(record); } @@ -317,7 +318,8 @@ public class UserService : IUserService new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), new Claim("phone", user.Phone ?? string.Empty), new Claim("affiliateId", user.AffiliateId ?? string.Empty), - new Claim("employeeId", user.EmployeeId ?? string.Empty) + new Claim("employeeId", user.EmployeeId ?? string.Empty), + new Claim("regionCode", user.RegionCode ?? "CN") }; var validators = _services.GetServices(); From 317f6b147ec186394a3692e4295d590ebf4e1e32 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 23 Oct 2024 15:04:36 +0800 Subject: [PATCH 19/50] fix: Set default values --- .../BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs index f2d9f048..c55da5fa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs @@ -12,7 +12,7 @@ public class UserCreationModel public string Password { get; set; } = string.Empty; public string Type { get; set; } = UserType.Client; public string Role { get; set; } = UserRole.User; - public string? RegionCode { get; set; } + public string RegionCode { get; set; } = "CN"; public User ToUser() { return new User From 83cf6ae91246f4640bfb947e5a98ff0390154118 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 23 Oct 2024 15:35:05 +0800 Subject: [PATCH 20/50] fix: Set default values --- .../BotSharp.OpenAPI/Controllers/UserController.cs | 2 +- .../BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 253c19be..cb254f50 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -153,7 +153,7 @@ public class UserController : ControllerBase } [HttpPost("/user/phone/modify")] - public async Task ModifyUserPhone([FromQuery] string phone, [FromQuery] string regionCode) + public async Task ModifyUserPhone([FromQuery] string phone, [FromQuery] string regionCode = "CN") { return await _userService.ModifyUserPhone(phone, regionCode); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs index df0601c5..0632a8f8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs @@ -6,7 +6,7 @@ public class UserResetPasswordModel public string? Phone { get; set; } public string Password { get; set; } = string.Empty; public string VerificationCode { get; set; } - public string? RegionCode { get; set; } + public string RegionCode { get; set; } = "CN"; public User ToUser() { From 33945d41eee38d81adedb79a91a80a383c5c0304 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 23 Oct 2024 16:14:25 +0800 Subject: [PATCH 21/50] fix: Set default values --- src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs index 7ed9c02d..0baa7457 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs @@ -21,7 +21,7 @@ public class User public string Role { get; set; } = UserRole.User; public string? VerificationCode { get; set; } public bool Verified { get; set; } - public string? RegionCode { get; set; } + public string RegionCode { get; set; } = "CN"; public string? AffiliateId { get; set; } public string? EmployeeId { get; set; } public bool IsDisabled { get; set; } From 82aab1f54accd6bcb4258ad1a4e2913ad818f00e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 26 Oct 2024 09:32:18 -0500 Subject: [PATCH 22/50] Upgrade MongoDB.Driver to v2.28.0 --- .../BotSharp.Plugin.MongoStorage.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj index b7bf8e5a..ab14d864 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj @@ -12,7 +12,7 @@ - + From 6961ef5e19a47146f18970a828ca9d010b76b8aa Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 27 Oct 2024 21:07:30 -0500 Subject: [PATCH 23/50] PostData --- .../Browsing/Models/BrowserActionResult.cs | 4 ++-- .../BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs | 2 ++ .../BotSharp.Abstraction/Browsing/Models/MessageInfo.cs | 2 ++ .../BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs | 2 ++ .../Browsing/Models/WebPageResponseFilter.cs | 2 ++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs index bd34e936..1867a687 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs @@ -6,8 +6,8 @@ public class BrowserActionResult public bool IsSuccess { get; set; } public string? Message { get; set; } public string? StackTrace { get; set; } - public string Selector { get; set; } - public string Body { get; set; } + public string? Selector { get; set; } + public string? Body { get; set; } public bool IsHighlighted { get; set; } public override string ToString() diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs index a63fffc8..2935496c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs @@ -1,7 +1,9 @@ using BotSharp.Abstraction.Browsing.Enums; +using System.Diagnostics; namespace BotSharp.Abstraction.Browsing.Models; +[DebuggerStepThrough] public class ElementActionArgs { public BroswerActionEnum Action { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs index afb42dac..92c9d42d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs @@ -1,7 +1,9 @@ using BotSharp.Abstraction.Infrastructures; +using System.Diagnostics; namespace BotSharp.Abstraction.Browsing.Models; +[DebuggerStepThrough] public class MessageInfo : ICacheKey { public string AgentId { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 0964389b..698939e5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -1,7 +1,9 @@ using BotSharp.Abstraction.Browsing.Enums; +using System.Diagnostics; namespace BotSharp.Abstraction.Browsing.Models; +[DebuggerStepThrough] public class PageActionArgs { public BroswerActionEnum Action { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs index 54951115..00b78cc4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs @@ -8,6 +8,8 @@ public class WebPageResponseFilter public string Url { get; set; } = null!; public string[]? QueryParameters { get; set; } + public string[]? PostData { get; set; } + /// /// contains, starts, ends, equals /// From d527893b377d3e27d82cf5edb6a3dc8b2f2b22c0 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Mon, 28 Oct 2024 10:22:39 +0800 Subject: [PATCH 24/50] fix: add regionCode --- .../ViewModels/Users/UserResetPasswordModel.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs index 0632a8f8..7837c916 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserResetPasswordModel.cs @@ -15,7 +15,8 @@ public class UserResetPasswordModel Email = Email, Phone = Phone, Password = Password, - VerificationCode = VerificationCode + VerificationCode = VerificationCode, + RegionCode = RegionCode }; } } From ee3a9507cac34d39fca63cc5d9f36eb6c21b2d8f Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Mon, 28 Oct 2024 18:12:53 +0800 Subject: [PATCH 25/50] fix: Compatible +86 --- .../Repository/MongoRepository.User.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 1d0e2596..33cd16e8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -13,7 +13,12 @@ public partial class MongoRepository public User? GetUserByPhone(string phone) { - var user = _dc.Users.AsQueryable().FirstOrDefault(x => x.Phone == phone && x.Type != UserType.Affiliate); + string phoneSecond = string.Empty; + if (phone.Substring(0, 3) != "+86") + { + phoneSecond = $"+86{phone}"; + } + var user = _dc.Users.AsQueryable().FirstOrDefault(x => (x.Phone == phone || x.Phone == phoneSecond) && x.Type != UserType.Affiliate); return user != null ? user.ToUser() : null; } From d72c4ee610820d02147eda0dd9188693f51b8c1b Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Mon, 28 Oct 2024 19:44:01 +0800 Subject: [PATCH 26/50] fix: update phone number display --- .../BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index 393bbb86..df60c26c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -47,7 +47,7 @@ public class UserViewModel FirstName = user.FirstName, LastName = user.LastName, Email = user.Email, - Phone = user.Phone, + Phone = user.Phone.Substring(0, 3) == "+86" ? user.Phone.Substring(3) : user.Phone, Type = user.Type, Role = user.Role, Source = user.Source, From 22cc3dccf47b85ffe8ae0a3ce364891bdd60e9c2 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 28 Oct 2024 13:26:10 -0500 Subject: [PATCH 27/50] Print outerHTML when multiple elements found. --- .../PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs index ff43864e..8695fc5c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs @@ -122,7 +122,7 @@ public partial class PlaywrightWebDriver foreach (var element in await locator.AllAsync()) { - var content = await element.InnerHTMLAsync(); + var content = await element.EvaluateAsync("element => element.outerHTML"); _logger.LogError(content); } } @@ -130,7 +130,7 @@ public partial class PlaywrightWebDriver { foreach (var element in await locator.AllAsync()) { - var html = await element.InnerHTMLAsync(); + var html = await element.EvaluateAsync("element => element.outerHTML"); _logger.LogWarning(html); // fix if html has & result.Body = HttpUtility.HtmlDecode(html); From d213d2f8fd1b7cc2a37a58c7fc4d058cd02a4f76 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 28 Oct 2024 14:00:30 -0500 Subject: [PATCH 28/50] Fix mullti-context active page issue for PlaywrightInstance --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 2bbb0acf..1b0e79ac 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -9,7 +9,7 @@ public class PlaywrightInstance : IDisposable public IServiceProvider Services => _services; Dictionary _contexts = new Dictionary(); Dictionary> _pages = new Dictionary>(); - IPage? _activePage = null; + Dictionary _activePage = new Dictionary(); /// /// ContextId and BrowserContext @@ -30,14 +30,14 @@ public class PlaywrightInstance : IDisposable { if (string.IsNullOrEmpty(pattern)) { - return _activePage ?? _contexts[contextId].Pages.LastOrDefault(); + return _activePage.ContainsKey(contextId) ? _activePage[contextId] : _contexts[contextId].Pages.LastOrDefault(); } foreach (var page in _contexts[contextId].Pages) { if (page.Url.ToLower() == pattern.ToLower()) { - _activePage = page; + _activePage[contextId] = page; page.BringToFrontAsync().Wait(); return page; } @@ -92,7 +92,7 @@ public class PlaywrightInstance : IDisposable _contexts[ctxId].Page += async (sender, page) => { - _activePage = page; + _activePage[ctxId] = page; _pages[ctxId].Add(page); page.Close += async (sender, e) => { @@ -240,7 +240,7 @@ public class PlaywrightInstance : IDisposable if (page != null) { await page.CloseAsync(); - _activePage = _pages[ctxId].LastOrDefault(); + _activePage[ctxId] = _pages[ctxId].LastOrDefault(); } } } From bc3d935b4a8bf9b7b83189dc75310520a861ca1d Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Tue, 29 Oct 2024 18:30:53 +0800 Subject: [PATCH 29/50] fix: add RegionCode for UserViewModel --- .../BotSharp.OpenAPI/Controllers/UserController.cs | 2 ++ .../BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index cb254f50..8a712919 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -90,6 +90,8 @@ public class UserController : ControllerBase LastName = identiy.LastName, Source = claims.First().Issuer, ExternalId = identiy.Id, + RegionCode = identiy.RegionCode, + Phone = identiy.Phone, }); } return UserViewModel.FromUser(user); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index df60c26c..f6f8d268 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -27,6 +27,8 @@ public class UserViewModel [JsonPropertyName("update_date")] public DateTime UpdateDate { get; set; } + public string RegionCode { get; set; } = "CN"; + public static UserViewModel FromUser(User user) { if (user == null) @@ -54,7 +56,8 @@ public class UserViewModel ExternalId = user.ExternalId, CreateDate = user.CreatedTime, UpdateDate = user.UpdatedTime, - Avatar = "/user/avatar" + Avatar = "/user/avatar", + RegionCode = user.RegionCode }; } } From 5d32d64ef8048b96164266132145b86cf512f856 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Tue, 29 Oct 2024 20:31:44 +0800 Subject: [PATCH 30/50] fix user phone is null --- .../BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index df60c26c..e10a195c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -47,7 +47,7 @@ public class UserViewModel FirstName = user.FirstName, LastName = user.LastName, Email = user.Email, - Phone = user.Phone.Substring(0, 3) == "+86" ? user.Phone.Substring(3) : user.Phone, + Phone = user.Phone?.Substring(0, 3) == "+86" ? user.Phone.Substring(3) : user.Phone, Type = user.Type, Role = user.Role, Source = user.Source, From e3e1a56038037577d09fdcfdecbb90985456bd75 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 28 Oct 2024 23:28:09 -0400 Subject: [PATCH 31/50] Add BotSharp.Plugin.MicrosoftExtensionsAI --- BotSharp.sln | 11 ++ ...tSharp.Plugin.MicrosoftExtensionsAI.csproj | 23 +++ ...osoftExtensionsAIChatCompletionProvider.cs | 185 ++++++++++++++++++ .../MicrosoftExtensionsAIPlugin.cs | 29 +++ ...osoftExtensionsAITextCompletionProvider.cs | 71 +++++++ ...rosoftExtensionsAITextEmbeddingProvider.cs | 59 ++++++ .../ProviderHelper.cs | 26 +++ 7 files changed, 404 insertions(+) create mode 100644 src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj create mode 100644 src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/ProviderHelper.cs diff --git a/BotSharp.sln b/BotSharp.sln index d281450e..a1c1ddb9 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -65,6 +65,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.GoogleAI", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.SemanticKernel", "src\Plugins\BotSharp.Plugin.SemanticKernel\BotSharp.Plugin.SemanticKernel.csproj", "{73EE2CD0-3B27-4F02-A67B-762CBDD740D0}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.MicrosoftExtensionsAI", "src\Plugins\BotSharp.Plugin.MicrosoftExtensionsAI\BotSharp.Plugin.MicrosoftExtensionsAI.csproj", "{72CA059E-6AAA-406C-A1EB-A2243E652F5F}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.SemanticKernel.UnitTests", "tests\BotSharp.Plugin.SemanticKernel.UnitTests\BotSharp.Plugin.SemanticKernel.UnitTests.csproj", "{BC57D428-A1A4-4D38-A2D0-AC6CA943F247}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Twilio", "src\Plugins\BotSharp.Plugin.Twilio\BotSharp.Plugin.Twilio.csproj", "{E627F1E3-BE03-443A-83A2-86A855A278EB}" @@ -283,6 +285,14 @@ Global {73EE2CD0-3B27-4F02-A67B-762CBDD740D0}.Release|Any CPU.Build.0 = Release|Any CPU {73EE2CD0-3B27-4F02-A67B-762CBDD740D0}.Release|x64.ActiveCfg = Release|Any CPU {73EE2CD0-3B27-4F02-A67B-762CBDD740D0}.Release|x64.Build.0 = Release|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Debug|x64.ActiveCfg = Debug|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Debug|x64.Build.0 = Debug|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Release|Any CPU.Build.0 = Release|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Release|x64.ActiveCfg = Release|Any CPU + {72CA059E-6AAA-406C-A1EB-A2243E652F5F}.Release|x64.Build.0 = Release|Any CPU {BC57D428-A1A4-4D38-A2D0-AC6CA943F247}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BC57D428-A1A4-4D38-A2D0-AC6CA943F247}.Debug|Any CPU.Build.0 = Debug|Any CPU {BC57D428-A1A4-4D38-A2D0-AC6CA943F247}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -491,6 +501,7 @@ Global {DB3DE37B-1208-4ED3-9615-A52AD0AAD69C} = {5CD330E1-9E5A-4112-8346-6E31CA98EF78} {8BC29F8A-78D6-422C-B522-10687ADC38ED} = {D5293208-2BEF-42FC-A64C-5954F61720BA} {73EE2CD0-3B27-4F02-A67B-762CBDD740D0} = {D5293208-2BEF-42FC-A64C-5954F61720BA} + {72CA059E-6AAA-406C-A1EB-A2243E652F5F} = {D5293208-2BEF-42FC-A64C-5954F61720BA} {BC57D428-A1A4-4D38-A2D0-AC6CA943F247} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {E627F1E3-BE03-443A-83A2-86A855A278EB} = {64264688-0F5C-4AB0-8F2B-B59B717CCE00} {F06B22CB-B143-4680-8FFF-35B9E50E6C47} = {51AFE054-AE99-497D-A593-69BAEFB5106F} diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj new file mode 100644 index 00000000..53518b38 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj @@ -0,0 +1,23 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GeneratePackageOnBuild) + $(SolutionDir)packages + $(NoWarn);NU5104 + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs new file mode 100644 index 00000000..67f1e513 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -0,0 +1,185 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.MLTasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using System.Collections.Generic; +using System.Threading.Tasks; +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; + +namespace BotSharp.Plugin.MicrosoftExtensionsAI; + +/// +/// Provides an implementation of for Microsoft.Extensions.AI. +/// +public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletion +{ + private readonly IChatClient _client; + private readonly ILogger _logger; + private readonly IServiceProvider _services; + private string? _model; + + /// + /// Creates an instance of the class. + /// + public MicrosoftExtensionsAIChatCompletionProvider( + IChatClient client, + ILogger logger, + IServiceProvider services) + { + _client = client; + _model = _client.Metadata.ModelId; + _logger = logger; + _services = services; + } + + /// + public string Provider => "microsoft.extensions.ai"; + + /// + public void SetModelName(string model) => _model = model; + + /// + public async Task GetChatCompletions(Agent agent, List conversations) + { + // Before chat completion hook + var hooks = _services.GetServices().ToArray(); + await Task.WhenAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations))); + + // Configure options + var state = _services.GetRequiredService(); + var options = new ChatOptions() + { + Temperature = float.Parse(state.GetState("temperature", "0.0")), + MaxOutputTokens = int.Parse(state.GetState("max_tokens", "1024")) + }; + + if (_services.GetService() is { } agentService) + { + foreach (var function in agent.Functions) + { + if (agentService.RenderFunction(agent, function)) + { + var property = agentService.RenderFunctionProperty(agent, function); + (options.Tools ??= []).Add(new NopAIFunction(new(function.Name) + { + Description = function.Description, + Parameters = property?.Properties.RootElement.Deserialize>()?.Select(p => new AIFunctionParameterMetadata(p.Key) + { + Schema = p.Value, + }).ToList() ?? [], + })); + } + } + } + + // Configure messages + List messages = []; + + if (_services.GetRequiredService().RenderedInstruction(agent) is string instruction && + instruction.Length > 0) + { + messages.Add(new(ChatRole.System, instruction)); + } + + if (!string.IsNullOrEmpty(agent.Knowledges)) + { + messages.Add(new(ChatRole.System, agent.Knowledges)); + } + + foreach (var sample in ProviderHelper.GetChatSamples(agent.Samples)) + { + messages.Add(new(sample.Role == AgentRole.Assistant ? ChatRole.Assistant : ChatRole.User, sample.Content)); + } + + var fileStorage = _services.GetService(); + bool allowMultiModal = fileStorage is not null && _services.GetService()?.GetSetting(Provider, _model ?? "default")?.MultiModal is true; + foreach (var x in conversations) + { + if (x.Role == AgentRole.Function && x.FunctionName is not null) + { + messages.Add(new(ChatRole.Assistant, + [ + new FunctionCallContent(x.FunctionName, x.FunctionName, JsonSerializer.Deserialize>(x.FunctionArgs ?? "{}")), + new FunctionResultContent(x.FunctionName, x.FunctionName, x.Content) + ])); + } + else if (x.Role == AgentRole.System || x.Role == AgentRole.Assistant) + { + messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.Content)); + } + else if (x.Role == AgentRole.User) + { + List contents = [new TextContent(!string.IsNullOrWhiteSpace(x.Payload) ? x.Payload : x.Content)]; + if (allowMultiModal) + { + foreach (var file in x.Files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + contents.Add(new ImageContent(file.FileData)); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var bytes = fileStorage!.GetFileBytes(file.FileStorageUrl); + contents.Add(new ImageContent(bytes, contentType)); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + contents.Add(new ImageContent(file.FileUrl)); + } + } + } + + messages.Add(new(ChatRole.User, contents) { AuthorName = x.FunctionName }); + } + } + + var completion = await _client.CompleteAsync(messages); + + RoleDialogModel result = new(AgentRole.Assistant, string.Concat(completion.Message.Contents.OfType())) + { + CurrentAgentId = agent.Id + }; + + if (completion.Message.Contents.OfType().FirstOrDefault() is { } fcc) + { + result.Role = AgentRole.Function; + result.MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + result.FunctionName = fcc.Name; + result.FunctionArgs = fcc.Arguments is not null ? JsonSerializer.Serialize(fcc.Arguments) : null; + } + + // After chat completion hook + await Task.WhenAll(hooks.Select(hook => hook.AfterGenerated(result, new() { Model = _model ?? "default" }))); + + return result; + } + + /// + public Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) => + throw new NotImplementedException(); + + /// + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) => + throw new NotImplementedException(); + + private sealed class NopAIFunction(AIFunctionMetadata metadata) : AIFunction + { + public override AIFunctionMetadata Metadata { get; } = metadata; + + protected override Task InvokeCoreAsync(IEnumerable> arguments, CancellationToken cancellationToken) => + throw new NotSupportedException(); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIPlugin.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIPlugin.cs new file mode 100644 index 00000000..105e0084 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIPlugin.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Plugins; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace BotSharp.Plugin.MicrosoftExtensionsAI; + +/// +/// Use Microsoft.Extensions.AI as BotSharp plugin +/// +public sealed class MicrosoftExtensionsAIPlugin : IBotSharpPlugin +{ + /// + public string Id => "B7F2AB8D-1BBA-41CE-9642-2D5E6B5F86A0"; + + /// + public string Name => "Microsoft.Extensions.AI"; + + /// + public string Description => "Microsoft.Extensions.AI Service"; + + /// + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs new file mode 100644 index 00000000..152213c6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs @@ -0,0 +1,71 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.MLTasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.MicrosoftExtensionsAI; + +/// +/// Provides an implementation of for Microsoft.Extensions.AI. +/// +public sealed class MicrosoftExtensionsAITextCompletionProvider : ITextCompletion +{ + private readonly IChatClient _chatClient; + private readonly IServiceProvider _services; + private readonly ITokenStatistics _tokenStatistics; + private string? _model = null; + + /// + public string Provider => "microsoft-extensions-ai"; + + /// + /// Creates an instance of the class. + /// + public MicrosoftExtensionsAITextCompletionProvider( + IChatClient chatClient, + IServiceProvider services, + ITokenStatistics tokenStatistics) + { + _chatClient = chatClient; + _services = services; + _tokenStatistics = tokenStatistics; + } + + /// + public async Task GetCompletion(string text, string agentId, string messageId) + { + var hooks = _services.GetServices().ToArray(); + + // Before chat completion hook + Agent agent = new() { Id = agentId }; + RoleDialogModel userMessage = new(AgentRole.User, text) { MessageId = messageId }; + await Task.WhenAll(hooks.Select(hook => hook.BeforeGenerating(agent, [userMessage]))); + + _tokenStatistics.StartTimer(); + var completion = await _chatClient.CompleteAsync(text); + var result = string.Concat(completion.Message.Contents.OfType()); + _tokenStatistics.StopTimer(); + + // After chat completion hook + await Task.WhenAll(hooks.Select(hook => + hook.AfterGenerated(new(AgentRole.Assistant, result), new() { Model = _model ?? "default" }))); + + return result; + } + + /// + public void SetModelName(string model) + { + if (!string.IsNullOrWhiteSpace(model)) + { + _model = model; + } + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs new file mode 100644 index 00000000..958c4694 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs @@ -0,0 +1,59 @@ +using BotSharp.Abstraction.MLTasks; +using Microsoft.Extensions.AI; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.MicrosoftExtensionsAI; + +/// +/// Provides an implementation of for Microsoft.Extensions.AI. +/// +public sealed class MicrosoftExtensionsAITextEmbeddingProvider : ITextEmbedding +{ + private readonly IEmbeddingGenerator> _generator; + private string? _model; + private int? _dimensions; + + /// + /// Creates an instance of the class. + /// + public MicrosoftExtensionsAITextEmbeddingProvider(IEmbeddingGenerator> generator) => + _generator = generator; + + /// + public string Provider => "microsoft-extensions-ai"; + + /// + public async Task GetVectorAsync(string text) => + (await _generator.GenerateEmbeddingVectorAsync(text, CreateOptions())).ToArray(); + + /// + public async Task> GetVectorsAsync(List texts) + { + var embeddings = await _generator.GenerateAsync(texts, CreateOptions()); + return embeddings.Select(e => e.Vector.ToArray()).ToList(); + } + + /// + public void SetModelName(string model) => _model = model; + + /// + public void SetDimension(int dimension) + { + if (dimension > 0) + { + _dimensions = dimension; + } + } + + /// + public int GetDimension() => _dimensions ?? 0; + + private EmbeddingGenerationOptions CreateOptions() => + new() + { + ModelId = _model, + Dimensions = _dimensions, + }; +} diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/ProviderHelper.cs new file mode 100644 index 00000000..17f0b82e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/ProviderHelper.cs @@ -0,0 +1,26 @@ +using BotSharp.Abstraction.Conversations.Models; +using System.Collections.Generic; + +namespace BotSharp.Plugin.MicrosoftExtensionsAI; + +internal static class ProviderHelper +{ + public static IEnumerable GetChatSamples(List lines) + { + foreach (string line in lines) + { + if (!string.IsNullOrWhiteSpace(line)) + { + int pos = line.IndexOf(' '); + if (pos > 0) + { + string role = line.Substring(0, pos - 1).Trim(); + if (role != "##") // skip comments + { + yield return new RoleDialogModel(role, line.Substring(line.IndexOf(' ') + 1).Trim()); + } + } + } + } + } +} From 7c226444bce1938ae69aef6a1e4d724b704c7b85 Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Tue, 29 Oct 2024 11:34:17 -0500 Subject: [PATCH 32/50] naming change-domain knowledge --- .../BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Functions/PrimaryStagePlanFn.cs | 8 ++++---- .../Functions/SecondaryStagePlanFn.cs | 6 +++--- .../BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs | 10 +++++----- .../templates/two_stage.1st.plan.liquid | 6 +++--- .../templates/two_stage.summarize.liquid | 4 ++-- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs index 6234611c..1167d2e1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -5,7 +5,7 @@ public interface IKnowledgeHook Task> CollectChunkedKnowledge() => Task.FromResult(new List()); - Task> GetRelevantKnowledges(RoleDialogModel message, string text) + Task> GetDomainKnowledges(RoleDialogModel message, string text) => Task.FromResult(new List()); Task> GetGlobalKnowledges(RoleDialogModel message) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 39584b84..606caad9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -24,7 +24,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback var knowledges = new List(); foreach (var hook in hooks) { - var k = await hook.GetRelevantKnowledges(message, args.Question); + var k = await hook.GetDomainKnowledges(message, args.Question); knowledges.AddRange(k); } knowledges = knowledges.Distinct().ToList(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 74afc5ff..664ad7b9 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -36,13 +36,13 @@ public class PrimaryStagePlanFn : IFunctionCallback { foreach (var hook in hooks) { - var k = await hook.GetRelevantKnowledges(message, question); + var k = await hook.GetDomainKnowledges(message, question); knowledges.AddRange(k); } } knowledges = knowledges.Distinct().ToList(); var knowledgeState = String.Join("\r\n", knowledges); - state.SetState("relevant_knowledges", knowledgeState); + state.SetState("domain_knowledges", knowledgeState); // Get first stage planning prompt var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); @@ -64,7 +64,7 @@ public class PrimaryStagePlanFn : IFunctionCallback return true; } - private async Task GetFirstStagePlanPrompt(RoleDialogModel message, string taskDescription, List relevantKnowledges) + private async Task GetFirstStagePlanPrompt(RoleDialogModel message, string taskDescription, List domainKnowledges) { var agentService = _services.GetRequiredService(); var render = _services.GetRequiredService(); @@ -86,7 +86,7 @@ public class PrimaryStagePlanFn : IFunctionCallback { { "task_description", taskDescription }, { "global_knowledges", globalKnowledges }, - { "relevant_knowledges", relevantKnowledges }, + { "domain_knowledges", domainKnowledges }, { "response_format", responseFormat } }); } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index df4a879f..f0972ccf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -36,15 +36,15 @@ public class SecondaryStagePlanFn : IFunctionCallback var knowledges = new List(); foreach (var hook in hooks) { - var k = await hook.GetRelevantKnowledges(message, taskSecondary.SolutionQuestion); + var k = await hook.GetDomainKnowledges(message, taskSecondary.SolutionQuestion); knowledges.AddRange(k); } knowledges = knowledges.Distinct().ToList(); var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges); - var knowledgeState = states.GetState("relevant_knowledges"); + var knowledgeState = states.GetState("domain_knowledges"); knowledgeState += String.Join("\r\n", knowledges); - states.SetState("relevant_knowledges", knowledgeState); + states.SetState("domain_knowledges", knowledgeState); // Get second stage planning prompt var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index bcf7588a..258f4c33 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -35,8 +35,8 @@ public class SummaryPlanFn : IFunctionCallback var steps = states.GetState("planning_result").JsonArrayContent(); var allTables = new List(); var ddlStatements = string.Empty; - var relevantKnowledge = states.GetState("planning_result"); - relevantKnowledge += "\r\n" + states.GetState("relevant_knowledges"); + var domainKnowledge = states.GetState("planning_result"); + domainKnowledge += "\r\n" + states.GetState("domain_knowledges"); var dictionaryItems = states.GetState("dictionary_items"); var excelImportResult = states.GetState("excel_import_result"); @@ -55,7 +55,7 @@ public class SummaryPlanFn : IFunctionCallback ddlStatements += "\r\n" + msgCopy.Content; // Summarize and generate query - var prompt = await GetSummaryPlanPrompt(msgCopy, taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult); + var prompt = await GetSummaryPlanPrompt(msgCopy, taskRequirement, domainKnowledge, dictionaryItems, ddlStatements, excelImportResult); _logger.LogInformation($"Summary plan prompt:\r\n{prompt}"); var plannerAgent = new Agent @@ -76,7 +76,7 @@ public class SummaryPlanFn : IFunctionCallback return true; } - private async Task GetSummaryPlanPrompt(RoleDialogModel message, string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult) + private async Task GetSummaryPlanPrompt(RoleDialogModel message, string taskDescription, string domainKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult) { var agentService = _services.GetRequiredService(); var render = _services.GetRequiredService(); @@ -104,7 +104,7 @@ public class SummaryPlanFn : IFunctionCallback { "task_description", taskDescription }, { "summary_requirements", string.Join("\r\n", additionalRequirements) }, { "global_knowledges", globalKnowledges }, - { "relevant_knowledges", relevantKnowledge }, + { "domain_knowledges", domainKnowledge }, { "dictionary_items", dictionaryItems }, { "table_structure", ddlStatement }, { "excel_import_result", excelImportResult } diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 9ded6e16..08a962d3 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -1,7 +1,7 @@ You are a Task Planner. you will breakdown user business requirements into excutable sub-tasks. Thinking process: -1. Reference to "Task Knowledge" if there is relevant knowledge; +1. Reference to "Task Knowledge" if there is domain knowledge; 2. Breakdown task into subtasks. - The subtask should contain all needed parameters for subsequent steps. - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_breakdown_task to true. @@ -25,10 +25,10 @@ Global Knowledge: {% endfor %} {%- endif %} -{% if relevant_knowledges != empty -%} +{% if domain_knowledges != empty -%} ===== Task Knowledge: -{% for k in relevant_knowledges %} +{% for k in domain_knowledges %} {{ k }} {% endfor %} {%- endif %} diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid index 6b901eb8..a585cd23 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid @@ -12,8 +12,8 @@ Global Knowledges: {{ global_knowledges }} ===== -Relevant Knowledges: -{{ relevant_knowledges }} +Domain Knowledges: +{{ domain_knowledges }} ===== Dictionary Items: From 505e047ee7cd693bb00c9443cc8a09c60acb33b2 Mon Sep 17 00:00:00 2001 From: Bo Yin <103488@smsassist.com> Date: Tue, 29 Oct 2024 13:55:55 -0500 Subject: [PATCH 33/50] support intent in welcome webhook --- .../Controllers/TwilioVoiceController.cs | 65 ++++++++++++++----- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index cf318e35..1d100712 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -32,7 +32,7 @@ public class TwilioVoiceController : TwilioController /// [ValidateRequest] [HttpPost("twilio/voice/welcome")] - public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) + public async Task InitiateConversation(VoiceRequest request, [FromQuery] string states, [FromQuery] string intent) { if (request?.CallSid == null) { @@ -41,8 +41,29 @@ public class TwilioVoiceController : TwilioController string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); - var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; - var response = twilio.ReturnNoninterruptedInstructions(new List { "twilio/welcome.mp3" }, url, true, timeout: 2); + VoiceResponse response; + if (string.IsNullOrWhiteSpace(intent)) + { + var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; + response = twilio.ReturnNoninterruptedInstructions(new List { "twilio/welcome.mp3" }, url, true, timeout: 2); + } + else + { + int seqNum = 0; + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + await sessionManager.StageCallerMessageAsync(conversationId, seqNum, intent); + var callerMessage = new CallerMessage() + { + ConversationId = conversationId, + SeqNumber = seqNum, + Content = intent, + From = request.From, + States = ParseStates(states) + }; + await messageQueue.EnqueueAsync(callerMessage); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + } return TwiML(response); } @@ -72,19 +93,10 @@ public class TwilioVoiceController : TwilioController ConversationId = conversationId, SeqNumber = seqNum, Content = messageContent, - Digits = request.Digits, - From = request.From + Digits = request.Digits, + From = request.From, + States = ParseStates(states) }; - - if (!string.IsNullOrEmpty(states)) - { - var kvp = states.Split(':'); - if (kvp.Length == 2) - { - callerMessage.States.Add(kvp[0], kvp[1]); - } - } - await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); @@ -159,7 +171,7 @@ public class TwilioVoiceController : TwilioController { speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3"); } - + var fileName = $"indication_{seqNum}_{segIndex}.mp3"; fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); @@ -214,7 +226,7 @@ public class TwilioVoiceController : TwilioController response = twilio.ReturnInstructions(new List { $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" - }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true, hints:reply.Hints); + }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true, hints: reply.Hints); } } @@ -233,4 +245,23 @@ public class TwilioVoiceController : TwilioController }; return result; } + + private Dictionary ParseStates(string? states) + { + var result = new Dictionary(); + if (string.IsNullOrWhiteSpace(states)) + { + return result; + } + var kvps = states.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + foreach (var kvp in kvps) + { + var parts = kvp.Split(':'); + if (parts.Length == 2) + { + result.Add(parts[0], parts[1]); + } + } + return result; + } } From 88b946c36d509fbab15deee7dc6383eaac8df2da Mon Sep 17 00:00:00 2001 From: Bo Yin <103488@smsassist.com> Date: Tue, 29 Oct 2024 17:25:35 -0500 Subject: [PATCH 34/50] fix states query parameter --- .../Controllers/TwilioVoiceController.cs | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 1d100712..e7d93e06 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -32,7 +32,7 @@ public class TwilioVoiceController : TwilioController /// [ValidateRequest] [HttpPost("twilio/voice/welcome")] - public async Task InitiateConversation(VoiceRequest request, [FromQuery] string states, [FromQuery] string intent) + public async Task InitiateConversation(VoiceRequest request, [FromQuery] string[] states, [FromQuery] string intent) { if (request?.CallSid == null) { @@ -44,7 +44,7 @@ public class TwilioVoiceController : TwilioController VoiceResponse response; if (string.IsNullOrWhiteSpace(intent)) { - var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; + var url = $"twilio/voice/{conversationId}/receive/0?{GenerateStatesParameter(states)}"; response = twilio.ReturnNoninterruptedInstructions(new List { "twilio/welcome.mp3" }, url, true, timeout: 2); } else @@ -62,14 +62,14 @@ public class TwilioVoiceController : TwilioController States = ParseStates(states) }; await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}"), HttpMethod.Post); } return TwiML(response); } [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] - public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request, [FromQuery] int attempts = 1) + public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string[] states, VoiceRequest request, [FromQuery] int attempts = 1) { var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); @@ -99,7 +99,7 @@ public class TwilioVoiceController : TwilioController }; await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}"), HttpMethod.Post); } else { @@ -117,11 +117,11 @@ public class TwilioVoiceController : TwilioController speechPaths.Add($"twilio/say-it-again-{Random.Shared.Next(1, 5)}.mp3"); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{lastRepy.SpeechFileName}"); } - response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}", true); + response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/receive/{seqNum}?{GenerateStatesParameter(states)}", true); } else { - response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}&attempts={++attempts}", true); + response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/receive/{seqNum}?{GenerateStatesParameter(states)}&attempts={++attempts}", true); } } return TwiML(response); @@ -130,7 +130,7 @@ public class TwilioVoiceController : TwilioController [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, - [FromQuery] string states, VoiceRequest request) + [FromQuery] string[] states, VoiceRequest request) { var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); @@ -185,7 +185,7 @@ public class TwilioVoiceController : TwilioController segIndex++; } } - response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true); + response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}", true); await sessionManager.RemoveReplyIndicationAsync(conversationId, seqNum); } else @@ -208,7 +208,7 @@ public class TwilioVoiceController : TwilioController instructions.Add($"twilio/typing-{typingIndex}.mp3"); } - response = twilio.ReturnInstructions(instructions, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true); + response = twilio.ReturnInstructions(instructions, $"twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}", true); } } else @@ -226,7 +226,7 @@ public class TwilioVoiceController : TwilioController response = twilio.ReturnInstructions(new List { $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" - }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true, hints: reply.Hints); + }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?{GenerateStatesParameter(states)}", true, hints: reply.Hints); } } @@ -246,17 +246,16 @@ public class TwilioVoiceController : TwilioController return result; } - private Dictionary ParseStates(string? states) + private Dictionary ParseStates(string[] states) { var result = new Dictionary(); - if (string.IsNullOrWhiteSpace(states)) + if (states is null || !states.Any()) { return result; } - var kvps = states.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - foreach (var kvp in kvps) + foreach (var kvp in states) { - var parts = kvp.Split(':'); + var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); if (parts.Length == 2) { result.Add(parts[0], parts[1]); @@ -264,4 +263,13 @@ public class TwilioVoiceController : TwilioController } return result; } + + private string GenerateStatesParameter(string[] states) + { + if (states is null || states.Length == 0) + { + return null; + } + return string.Join("&", states.Select(x => $"states={x}")); + } } From b6cc3de4f366393c8a4e308ed412169c80924591 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 10:51:06 +0800 Subject: [PATCH 35/50] fix: Login bug --- src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index dbd8b696..d8e19d0e 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -213,7 +213,7 @@ public class UserService : IUserService var (id, password) = base64.SplitAsTuple(":"); var db = _services.GetRequiredService(); - var record = id.Contains("@") ? db.GetUserByEmail(id) : db.GetUserByUserName(id); + var record = id.Contains("@") ? db.GetUserByEmail(id) : db.GetUserByPhone(id); if (record == null) { record = db.GetUserByUserName(id); From 9a1cf696a2c4b73a9cbf1e8a1f8606a8a685d0bc Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 10:58:28 +0800 Subject: [PATCH 36/50] fix: login bug --- .../BotSharp.Core/Users/Services/UserService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index d8e19d0e..469a72c4 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -213,10 +213,10 @@ public class UserService : IUserService var (id, password) = base64.SplitAsTuple(":"); var db = _services.GetRequiredService(); - var record = id.Contains("@") ? db.GetUserByEmail(id) : db.GetUserByPhone(id); + var record = id.Contains("@") ? db.GetUserByEmail(id) : db.GetUserByUserName(id); if (record == null) { - record = db.GetUserByUserName(id); + record = db.GetUserByPhone(id); } if (record != null && record.Type == UserType.Affiliate) From 656e96aec5f72b03c1bb93f75169afbbbc2eee37 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 16:45:24 +0800 Subject: [PATCH 37/50] fix: Activated user changes phone number to inactive user bug --- src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 469a72c4..8bdc6f28 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -483,7 +483,7 @@ public class UserService : IUserService var db = _services.GetRequiredService(); var UserByphone = db.GetUserByPhone(phone); - if (UserByphone != null && UserByphone.Verified) + if (UserByphone != null/* && UserByphone.Verified*/) { return true; } From 30be27ea40d75b23d8b4d5a9a7fd2ae78881dbe8 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 17:35:08 +0800 Subject: [PATCH 38/50] fix: ActiveUser bug --- src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 8bdc6f28..3fbf92c7 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -407,7 +407,7 @@ public class UserService : IUserService var record = id.Contains("@") ? db.GetUserByEmail(id) : db.GetUserByUserName(id); if (record == null) { - record = db.GetUserByUserName(id); + record = db.GetUserByPhone(id); } if (record == null) From e31942c656ec8edaca0039230176b853778e6b1c Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 18:00:05 +0800 Subject: [PATCH 39/50] fix: data is empty set the default value --- .../BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index eb9656fe..5e9a17f0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -57,7 +57,7 @@ public class UserViewModel CreateDate = user.CreatedTime, UpdateDate = user.UpdatedTime, Avatar = "/user/avatar", - RegionCode = user.RegionCode + RegionCode = string.IsNullOrWhiteSpace(user.RegionCode) ? "CN" : user.RegionCode }; } } From 025684407480934eb8bac81eb53ce80744ee3e2d Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 18:22:24 +0800 Subject: [PATCH 40/50] fix: NullReferenceException --- .../Repository/MongoRepository.User.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 33cd16e8..7c19dc2a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -14,7 +14,7 @@ public partial class MongoRepository public User? GetUserByPhone(string phone) { string phoneSecond = string.Empty; - if (phone.Substring(0, 3) != "+86") + if (phone.Length >= 4 && phone.Substring(0, 3) != "+86") { phoneSecond = $"+86{phone}"; } From 79d053daa74f4a998781e236f955d5ee57c61d82 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 18:42:23 +0800 Subject: [PATCH 41/50] fix: Support updating regionCode --- src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 3fbf92c7..674f60c5 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -631,7 +631,7 @@ public class UserService : IUserService var record = db.GetUserById(curUser.Id); var existPhone = db.GetUserByPhone(phone); - if (record == null || existPhone != null) + if (record == null || (existPhone != null && existPhone.RegionCode == regionCode)) { return false; } From 3ec50552161045268be39571d2fc15d76b42f234 Mon Sep 17 00:00:00 2001 From: YouWeiDH Date: Wed, 30 Oct 2024 19:39:26 +0800 Subject: [PATCH 42/50] hdong:Add flag CreateUserAutomatically to control user create in user/me . --- .../BotSharp.Abstraction/Users/Settings/AccountSetting.cs | 1 + .../BotSharp.OpenAPI/Controllers/UserController.cs | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs index 17151d7e..a7f6b717 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs @@ -7,4 +7,5 @@ public class AccountSetting /// public bool NewUserVerification { get; set; } public string[] AllowMultipleDeviceLoginUserIds { get; set; } = []; + public bool CreateUserAutomatically { get; set; } = true; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 8a712919..de31ef6b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Users.Settings; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; using System.ComponentModel.DataAnnotations; @@ -10,10 +11,12 @@ public class UserController : ControllerBase { private readonly IServiceProvider _services; private readonly IUserService _userService; - public UserController(IUserService userService, IServiceProvider services) + private readonly AccountSetting _setting; + public UserController(IUserService userService, IServiceProvider services, AccountSetting setting) { _services = services; _userService = userService; + _setting = setting; } [AllowAnonymous] @@ -77,7 +80,7 @@ public class UserController : ControllerBase public async Task GetMyUserProfile() { var user = await _userService.GetMyProfile(); - if (user == null) + if (user == null && _setting.CreateUserAutomatically) { var identiy = _services.GetRequiredService(); var accessor = _services.GetRequiredService(); From cee71b1add3e064a35260f1213df36cac7e3b290 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 20:36:32 +0800 Subject: [PATCH 43/50] fix: Compatible with new data --- .../Repository/MongoRepository.User.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 7c19dc2a..4ebb62c3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -18,6 +18,10 @@ public partial class MongoRepository { phoneSecond = $"+86{phone}"; } + else + { + phoneSecond = phone.Replace("+86", ""); + } var user = _dc.Users.AsQueryable().FirstOrDefault(x => (x.Phone == phone || x.Phone == phoneSecond) && x.Type != UserType.Affiliate); return user != null ? user.ToUser() : null; } From e1d4956d91b45603ca3922ff81e4657ea89236fa Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 30 Oct 2024 20:50:23 +0800 Subject: [PATCH 44/50] fix: NullReferenceException --- .../Repository/MongoRepository.User.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 4ebb62c3..f5dbc508 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -14,7 +14,12 @@ public partial class MongoRepository public User? GetUserByPhone(string phone) { string phoneSecond = string.Empty; - if (phone.Length >= 4 && phone.Substring(0, 3) != "+86") + // 如果电话号码长度小于 4，直接返回 null + if (phone?.Length < 4) + { + return null; + } + if (phone.Substring(0, 3) != "+86") { phoneSecond = $"+86{phone}"; } From c85f12f572facda80faf9b6efddac70de76263b5 Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Wed, 30 Oct 2024 17:27:10 -0500 Subject: [PATCH 45/50] formattor update --- .../templates/query_result_formatting.liquid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid index 5c6ad05c..b9abb8ac 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid @@ -1,5 +1,5 @@ -Output in human readable format. If there is large amount of rows, shape it in tabular, otherwise, output in plain text. -Put user task description in the first line in the same language, for example, if user is using Chinese, you have to output the result in Chinese. +Output in human readable format. If there is large amount of rows, shape it in tabular, otherwise, output in plain text. Keep all the query data results. +Put user task description in the first line in the same language, for example, if user is using Chinese, you have to output the result in Chinese. Otherwise, use English. User Task Description: {{ requirement_detail }} \ No newline at end of file From b9d8331e7720520e4316e1ff01e2e3f6c6b8fdf6 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Thu, 31 Oct 2024 09:58:48 +0800 Subject: [PATCH 46/50] fix: VerifyEmailExisting bug --- src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 674f60c5..a2a257c0 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -466,7 +466,7 @@ public class UserService : IUserService var db = _services.GetRequiredService(); var emailName = db.GetUserByEmail(email); - if (emailName != null && emailName.Verified) + if (emailName != null /*&& emailName.Verified*/) { return true; } From afb9f446a18eae4dbb4f65feb4f3c891315de2ad Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Thu, 31 Oct 2024 18:37:37 +0800 Subject: [PATCH 47/50] fix: Existing exception --- .../BotSharp.Core/Users/Services/UserService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index a2a257c0..f7f4a6de 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -466,7 +466,7 @@ public class UserService : IUserService var db = _services.GetRequiredService(); var emailName = db.GetUserByEmail(email); - if (emailName != null /*&& emailName.Verified*/) + if (emailName != null && emailName.Verified) { return true; } @@ -483,7 +483,7 @@ public class UserService : IUserService var db = _services.GetRequiredService(); var UserByphone = db.GetUserByPhone(phone); - if (UserByphone != null/* && UserByphone.Verified*/) + if (UserByphone != null && UserByphone.Verified) { return true; } From 0d38df1f2250c7a3475df38bb6dd30fd8155dd7e Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Thu, 31 Oct 2024 11:35:01 -0500 Subject: [PATCH 48/50] Add Database Hook --- .../Repositories/IDatabaseHook.cs | 7 +++ .../BotSharp.Plugin.Planner.csproj | 12 ---- .../BotSharp.Plugin.SqlDriver.csproj | 17 ++++++ .../Functions/ExecuteQueryFn.cs | 22 +++++--- .../Functions/GetTableDefinitionFn.cs | 56 +++++++++++++++++-- .../Hooks/SqlDriverPlanningHook.cs | 8 --- .../Settings/SqlDriverSetting.cs | 3 +- .../templates/database.dictionary.sql.liquid | 0 .../templates/database.summarize.mysql.liquid | 0 .../database.summarize.redshift.liquid | 12 ++++ .../database.summarize.sqlserver.liquid | 0 11 files changed, 104 insertions(+), 33 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs rename src/Plugins/{BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9 => BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f}/templates/database.dictionary.sql.liquid (100%) rename src/Plugins/{BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9 => BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f}/templates/database.summarize.mysql.liquid (100%) create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.redshift.liquid rename src/Plugins/{BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9 => BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f}/templates/database.summarize.sqlserver.liquid (100%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs new file mode 100644 index 00000000..7484b754 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Repositories; + +public interface IDatabaseHook +{ + // Get database type + string GetDatabaseType(RoleDialogModel message); +} diff --git a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj index 01cb77de..72c4e710 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj +++ b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj @@ -16,9 +16,6 @@ - - - @@ -44,15 +41,6 @@ PreserveNewest - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index c0b755f6..245fccc7 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -29,6 +29,10 @@ + + + + @@ -70,6 +74,18 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest @@ -81,6 +97,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index d1b21bdc..646e48c0 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.SqlDriver.Models; @@ -6,6 +7,7 @@ using Dapper; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using MySqlConnector; +using Npgsql; namespace BotSharp.Plugin.SqlDriver.Functions; @@ -27,18 +29,18 @@ public class ExecuteQueryFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - var refinedArgs = await RefineSqlStatement(message, args); - - var settings = _services.GetRequiredService(); + var dbHook = _services.GetRequiredService(); + var dbType = dbHook.GetDatabaseType(message); try { - var results = settings.DatabaseType switch + var results = dbType.ToLower() switch { - "MySql" => RunQueryInMySql(refinedArgs.SqlStatements), - "SqlServer" => RunQueryInSqlServer(refinedArgs.SqlStatements), - _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + "mysql" => RunQueryInMySql(refinedArgs.SqlStatements), + "sqlserver" => RunQueryInSqlServer(refinedArgs.SqlStatements), + "redshift" => RunQueryInRedshift(refinedArgs.SqlStatements), + _ => throw new NotImplementedException($"Database type {dbType} is not supported.") }; if (refinedArgs.SqlStatements.Length == 1 && refinedArgs.SqlStatements[0].StartsWith("DROP TABLE")) @@ -102,6 +104,12 @@ public class ExecuteQueryFn : IFunctionCallback using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); return connection.Query(string.Join("\r\n", sqlTexts)); } + private IEnumerable RunQueryInRedshift(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new NpgsqlConnection(settings.RedshiftConnectionString); + return connection.Query(string.Join("\r\n", sqlTexts)); + } private async Task RefineSqlStatement(RoleDialogModel message, ExecuteQueryArgs args) { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 8d277b9d..6550b45b 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,7 +1,12 @@ +using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.SqlDriver.Models; using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore.Metadata.Internal; using Microsoft.Extensions.Logging; using MySqlConnector; +using Npgsql; +using System.ComponentModel.DataAnnotations; +using System.Data.Common; namespace BotSharp.Plugin.SqlDriver.Functions; @@ -25,14 +30,16 @@ public class GetTableDefinitionFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var tables = args.Tables; var agentService = _services.GetRequiredService(); - var settings = _services.GetRequiredService(); + var dbHook = _services.GetRequiredService(); + var dbType = dbHook.GetDatabaseType(message); // Get table DDL from database - var tableDdls = settings.DatabaseType switch + var tableDdls = dbType switch { - "MySql" => GetDdlFromMySql(tables), - "SqlServer" => GetDdlFromSqlServer(tables), - _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + "mysql" => GetDdlFromMySql(tables), + "sqlserver" => GetDdlFromSqlServer(tables), + "redshift" => GetDdlFromRedshift(tables), + _ => throw new NotImplementedException($"Database type {dbType} is not supported.") }; message.Content = string.Join("\r\n\r\n", tableDdls); @@ -125,4 +132,43 @@ public class GetTableDefinitionFn : IFunctionCallback connection.Close(); return tableDdls; } + + private List GetDdlFromRedshift(string[] tables) + { + var settings = _services.GetRequiredService(); + var tableDdls = new List(); + var schemas = "'onebi_hour','onebi_day'"; + using var connection = new NpgsqlConnection(settings.RedshiftConnectionString); + connection.Open(); + + foreach (var table in tables) + { + try + { + var sql = $"SELECT columnname, external_type, columnnum FROM svv_external_columns WHERE schemaname in ({schemas}) AND tablename = '{table}';"; + + using var command = new NpgsqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + var ddl = $"Table {table} contains below fields: \r\n"; + while (reader.Read()) + { + /*var result = reader.GetString(0); + tableDdls.Add(result);*/ + var columnName = reader.GetString(0); + var dataType = reader.GetString(1); + ddl += $"{columnName} {dataType}\n"; + } + tableDdls.Add(ddl); + + reader.Close(); + command.Dispose(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}"); + } + } + connection.Close(); + return tableDdls; + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index 29160780..6a558f5d 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -14,14 +14,6 @@ public class SqlDriverPlanningHook : IPlanningHook _services = services; } - public async Task GetSummaryAdditionalRequirements(string planner) - { - var settings = _services.GetRequiredService(); - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(BuiltInAgentId.Planner); - return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{settings.DatabaseType.ToLower()}")?.Content ?? string.Empty; - } - public async Task OnPlanningCompleted(string planner, RoleDialogModel msg) { var settings = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs index c6917894..24f12420 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.SqlHero.Settings; public class SqlDriverSetting { - public string DatabaseType { get; set; } = "MySql"; + public string DatabaseType { get; set; } = "mysql"; public string MySqlConnectionString { get; set; } = null!; public string MySqlExecutionConnectionString { get; set; } = null!; public string MySqlTempConnectionString { get; set; } = null!; @@ -10,6 +10,7 @@ public class SqlDriverSetting public string SqlServerConnectionString { get; set; } = null!; public string SqlServerExecutionConnectionString { get; set; } = null!; public string SqlLiteConnectionString { get; set; } = null!; + public string RedshiftConnectionString { get; set; } = null!; public bool ExecuteSqlSelectAutonomous { get; set; } = false; public bool FormattingResult { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.dictionary.sql.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.dictionary.sql.liquid diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.mysql.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.mysql.liquid diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.redshift.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.redshift.liquid new file mode 100644 index 00000000..05170b2f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.redshift.liquid @@ -0,0 +1,12 @@ +Try if you can generate a Redshift single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in Amazon Redshift. + +*** the alias of the table name in the sql query should be identical. *** +*** The generated sql query MUST be based on the provided table structure. *** +*** All queries return a maximum of 1000 records. *** +*** Only select user friendly columns. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.sqlserver.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/database.summarize.sqlserver.liquid From 8af8ef0baa915a61a883072f9d9f18328c920f00 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 1 Nov 2024 10:43:11 -0500 Subject: [PATCH 49/50] Fix planner issue of Claude output array is not stable --- .../functions/plan_primary_stage.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json index a934169a..60597b6f 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json @@ -10,7 +10,7 @@ }, "questions": { "type": "array", - "description": "Break down user requirements in details and in multiple ways, don't miss any entity type/value", + "description": "Break down user requirements in details and in multiple ways, don't miss any entity type/value. The output format must be string array.", "items": { "type": "string", "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge.One question should contain only one main topic that with one entity type." @@ -18,7 +18,7 @@ }, "norm_questions": { "type": "array", - "description": "normalize the generated questions, remove specific entity value.", + "description": "normalize the generated questions, remove specific entity value. The output format must be string array.", "items": { "type": "string", "description": "Normalized question" From d8f9c1a5544995eba493aae5b74cab2a14b5ab7d Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Fri, 1 Nov 2024 11:34:21 -0500 Subject: [PATCH 50/50] minor change --- .../templates/two_stage.1st.plan.liquid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 08a962d3..190edf8c 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -1,13 +1,13 @@ You are a Task Planner. you will breakdown user business requirements into excutable sub-tasks. Thinking process: -1. Reference to "Task Knowledge" if there is domain knowledge; +1. Reference to "Domain Knowledge" if there is relevant knowledge; 2. Breakdown task into subtasks. - The subtask should contain all needed parameters for subsequent steps. - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_breakdown_task to true. - If there is extra knowledge or relationship needed between steps, set the need_breakdown_task to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_breakdown_task to true. - - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_breakdown_task to true. + - You should find the relationships between data structure based on the domain knowledge strictly. If lack of information, set the need_breakdown_task to true. - If you need to lookup the dictionary to verify or get the enum/term/dictionary value(exclude example data from attachment), set the need_lookup_dictionary to true. - Don't set need_lookup_dictionary to true for attachment data. - Seperate the dictionary lookup and need additional information/knowledge into different subtask. @@ -27,7 +27,7 @@ Global Knowledge: {% if domain_knowledges != empty -%} ===== -Task Knowledge: +Domain Knowledge: {% for k in domain_knowledges %} {{ k }} {% endfor %}