Add PipeSettings to hold common settings for training or predit.

BotSharp.Core/Abstractions/INlpPipeline.cs

@@ -16,6 +16,11 @@ namespace BotSharp.Core.Abstractions
     {
         IConfiguration Configuration { get; set; }
 
+        /// <summary>
+        /// Common settings for Pipeline
+        /// </summary>
+        PipeSettings Settings { get; set; }
+
         /// <summary>
         /// Process
         /// </summary>

BotSharp.Core/BotSharp.Core.csproj

@@ -35,7 +35,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="DotNetToolkit" Version="1.5.0" />
+    <PackageReference Include="DotNetToolkit" Version="1.5.1" />
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="1.8.0" />
     <PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="2.1.1" />
     <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />

BotSharp.Core/Engines/BotPreditor.cs

@@ -34,16 +34,24 @@ namespace BotSharp.Core.Engines
 
             var data = JObject.FromObject(new
             {
+                Text = request.Query.FirstOrDefault()
             });
 
             await provider.Train(agent, data, providerPipe);
             meta.Pipeline.RemoveAt(0);
 
+            var settings = new PipeSettings
+            {
+                ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id),
+                AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
+            };
+
             // pipe process
             meta.Pipeline.ForEach(async pipeMeta =>
             {
                 var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
                 pipe.Configuration = provider.Configuration;
+                pipe.Settings = settings;
                 await pipe.Predict(agent, data, pipeMeta);
             });
 

BotSharp.Core/Engines/BotTrainer.cs

@@ -71,16 +71,21 @@ namespace BotSharp.Core.Engines
                 Pipeline = new List<PipeModel>() { pipeModel }
             };
 
-            var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
+            var settings = new PipeSettings
-            if (!Directory.Exists(dirTrain))
             {
-                Directory.CreateDirectory(dirTrain);
+                TrainDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id),
+                ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id),
+                AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
+            };
+
+            if (!Directory.Exists(settings.TrainDir))
+            {
+                Directory.CreateDirectory(settings.TrainDir);
             }
 
-            var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
+            if (!Directory.Exists(settings.ModelDir))
-            if (!Directory.Exists(dirModel))
             {
-                Directory.CreateDirectory(dirModel);
+                Directory.CreateDirectory(settings.ModelDir);
             }
 
             // pipe process
@@ -93,6 +98,7 @@ namespace BotSharp.Core.Engines
             {
                 var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
                 pipe.Configuration = provider.Configuration;
+                pipe.Settings = settings;
                 pipeModel = new PipeModel
                 {
                     Name = pipeName,

BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs

@@ -20,6 +20,7 @@ namespace BotSharp.Core.Engines.CRFsuite
     public class CRFsuiteEntityRecognizer : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
@@ -32,11 +33,9 @@ namespace BotSharp.Core.Engines.CRFsuite
             List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
             List<List<TrainingData>> list = new List<List<TrainingData>>();
 
-            var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
+            string rawTrainingDataFileName = Path.Join(Settings.TrainDir, "ner-crf.corpus.txt");
-            var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
+            string parsedTrainingDataFileName = Path.Join(Settings.TrainDir, "ner-crf.parsed.txt");
-            string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt");
+            string modelFileName = Path.Join(Settings.ModelDir, meta.Model);
-            string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt");
-            string modelFileName = Path.Join(dirModel, meta.Model);
 
             using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
             {
@@ -65,9 +64,7 @@ namespace BotSharp.Core.Engines.CRFsuite
             new MachineLearning.CRFsuite.Ner()
                 .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" "));
 
-            var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
+            CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
-
-            CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
 
             Console.WriteLine($"Saved model to {modelFileName}");
             meta.Meta = new JObject();

BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs

@@ -14,21 +14,22 @@ namespace BotSharp.Core.Engines.Classifiers
     public class FasttextClassifier : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
-        public Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
         {
-            throw new NotImplementedException();
+            string modelFileName = Path.Join(Settings.ModelDir, meta.Model);
+            var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), "predict {modelFileName}.bin test.txt");
+
+            return true;
         }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             meta.Model = "classification-fasttext.model";
-            var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
-            var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
-            var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
 
-            string parsedTrainingDataFileName = Path.Join(dirTrain, $"classification-fasttext.parsed.txt");
+            string parsedTrainingDataFileName = Path.Join(Settings.TrainDir, $"classification-fasttext.parsed.txt");
-            string modelFileName = Path.Join(dirModel, meta.Model);
+            string modelFileName = Path.Join(Settings.ModelDir, meta.Model);
 
             // assemble corpus
             StringBuilder corpus = new StringBuilder();
@@ -36,7 +37,7 @@ namespace BotSharp.Core.Engines.Classifiers
 
             File.WriteAllText(parsedTrainingDataFileName, corpus.ToString());
 
-            var output = CmdHelper.Run(Path.Join(algorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
+            var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
 
             Console.WriteLine($"Saved model to {modelFileName}");
             meta.Meta = new JObject();

BotSharp.Core/Engines/PipeSettings.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines
+{
+    public class PipeSettings
+    {
+        public string TrainDir { get; set; }
+        public string ModelDir { get; set; }
+        public string AlgorithmDir { get; set; }
+    }
+}

BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs

@@ -15,6 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
     public class SpaCyEntitizer : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
         {

BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs

@@ -17,6 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         List<String> entitiesInTrainingSet = new List<string>();
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {

BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs

@@ -15,6 +15,8 @@ namespace BotSharp.Core.Engines.SpaCy
     public class SpaCyProvider : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
+
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);

BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs

@@ -15,6 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
     public class SpaCyTagger : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {

BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs

@@ -16,6 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
     public class SpaCyTextCategorizer : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {

BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs

@@ -17,6 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
     public class SpaCyTokenizer : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {

BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs

@@ -14,6 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy
     public class SpacyFeaturizer : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
 
         public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {