Intergate NBClassifier into NLP pipeline.

BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs

@@ -10,5 +10,7 @@ namespace BotSharp.Algorithm.Bayes
         public List<Probability> LabelDist { get; set; }
 
         public Dictionary<string, double> CondProbDictionary { get; set; }
+
+        public List<String> Values { get; set; }
     }
 }

BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs

@@ -22,25 +22,22 @@ namespace BotSharp.Core.Engines.BotSharp
         public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
         {
             meta.Model = "classification-nb.model";
-
             string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
 
-            var encoder = new OneHotEncoder();
+            var options = new ClassifyOptions
-            encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence
+            {
+                ModelFilePath = modelFileName
+            };
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+
+            var sentences = doc.Sentences.Select(x => new Sentence
             {
                 Label = x.Intent.Label,
                 Text = x.Text,
                 Words = x.Tokens
             }).ToList();
-            encoder.EncodeAll();
 
-            var options = new ClassifyOptions
+            classifier.Train(sentences);
-            {
-                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
-            };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
-
-            classifier.Train(encoder.Sentences);
 
             Console.WriteLine($"Saved model to {modelFileName}");
             meta.Meta = new JObject();
@@ -51,6 +48,28 @@ namespace BotSharp.Core.Engines.BotSharp
 
         public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
         {
+            var options = new ClassifyOptions
+            {
+                ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model)
+            };
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+
+            var sentence = doc.Sentences.Select(s => new Sentence
+            {
+                Text = s.Text,
+                Words = s.Tokens
+            }).First();
+
+
+            var result = classifier.Classify(sentence);
+
+            doc.Sentences[0].Intent = new TextClassificationResult
+            {
+                Classifier = "BotSharpNBayesClassifier",
+                Label = result.First().Item1,
+                Confidence = (decimal)result.First().Item2
+            };
+
             return true;
         }
     }

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -36,17 +36,14 @@ namespace BotSharp.NLP.UnitTest
             
             sentences.Shuffle();
 
-            var encoder = new OneHotEncoder();
-            encoder.Sentences = sentences;
-            encoder.EncodeAll();
-
             var options = new ClassifyOptions
             {
+                ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
             };
             var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
             
-            var dataset = sentences.Split(0.7M);
+            var dataset = sentences.Split(1M);
             classifier.Train(dataset.Item1);
 
             int correct = 0;

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -29,20 +29,20 @@ namespace BotSharp.NLP.Classify
 
         public void Train(List<Sentence> sentences)
         {
-            var vectors = new List<Tuple<string, double[]>>();
+            _classifier.Train(sentences, _options);
-
+            _classifier.SaveModel(_options);
-            var sents = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
-
-            _classifier.Train(sents, new double[] { 0, 1 }, _options);
         }
 
         public List<Tuple<string, double>> Classify(Sentence sentence)
         {
             var options = new ClassifyOptions
             {
+                ModelFilePath = _options.ModelFilePath
             };
 
-            var classes = _classifier.Classify(sentence.Vector, options);
+            _classifier.LoadModel(options);
+
+            var classes = _classifier.Classify(sentence, options);
 
             return classes.OrderByDescending(x => x.Item2).ToList();
         }

BotSharp.NLP/Classify/IClassifier.cs

@@ -10,16 +10,20 @@ namespace BotSharp.NLP.Classify
         /// <summary>
         /// Training by feature vector
         /// </summary>
-        /// <param name="featureSets"></param>
+        /// <param name="sentences"></param>
         /// <param name="options"></param>
-        void Train(List<Tuple<string, double[]>> featureSets, double[] values, ClassifyOptions options);
+        void Train(List<Sentence> sentences, ClassifyOptions options);
 
         /// <summary>
         /// Predict by feature vector
         /// </summary>
-        /// <param name="features"></param>
+        /// <param name="sentence"></param>
         /// <param name="options"></param>
         /// <returns></returns>
-        List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options);
+        List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options);
+
+        String SaveModel(ClassifyOptions options);
+
+        Object LoadModel(ClassifyOptions options);
     }
 }

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -22,6 +22,8 @@ using BotSharp.Algorithm.Estimators;
 using BotSharp.Algorithm.Extensions;
 using BotSharp.Algorithm.Features;
 using BotSharp.Algorithm.Statistics;
+using BotSharp.NLP.Txt2Vec;
+using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -46,14 +48,24 @@ namespace BotSharp.NLP.Classify
 
         private Dictionary<string, double> condProbDictionary = new Dictionary<string, double>();
 
-        public void Train(List<Tuple<string, double[]>> featureSets, double[] values, ClassifyOptions options)
+        private List<string> words;
+        private double[] features = new double[] { 0, 1 };
+
+        public void Train(List<Sentence> sentences, ClassifyOptions options)
         {
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = sentences;
+            words = encoder.EncodeAll();
+
+            var featureSets = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
+
             labelDist = featureSets.GroupBy(x => x.Item1)
                 .Select(x => new Probability
                 {
                     Value = x.Key,
                     Freq = x.Count()
                 })
+                .OrderBy(x => x.Value)
                 .ToList();
 
             nb.LabelDist = labelDist;
@@ -70,30 +82,27 @@ namespace BotSharp.NLP.Classify
             {
                 for (int x = 0; x < featureCount; x++)
                 {
-                    for (int v = 0; v < values.Length; v++)
+                    for (int v = 0; v < features.Length; v++)
                     {
-                        string key = $"{label.Value} f{x} {values[v]}";
+                        string key = $"{label.Value} f{x} {features[v]}";
-                        condProbDictionary[key] = nb.CalCondProb(x, label.Value, values[v]);
+                        condProbDictionary[key] = nb.CalCondProb(x, label.Value, features[v]);
                     }
                 }
             });
-
-            // save the model
-            var model = new MultinomiaNaiveBayesModel
-            {
-                LabelDist = labelDist,
-                CondProbDictionary = condProbDictionary
-            };
         }
 
-        public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
+        public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
         {
+            var encoder = new OneHotEncoder();
+            encoder.Words = words;
+            encoder.Encode(sentence);
+
             var results = new List<Tuple<string, double>>();
 
             // calculate prop
             labelDist.ForEach(lf =>
             {
-                var prob = nb.CalPosteriorProb(lf.Value, features, lf.Prob, condProbDictionary);
+                var prob = nb.CalPosteriorProb(lf.Value, sentence.Vector, lf.Prob, condProbDictionary);
                 results.Add(new Tuple<string, double>(lf.Value, prob));
             });
 
@@ -105,6 +114,47 @@ namespace BotSharp.NLP.Classify
 
             return results;
         }
+
+        public string SaveModel(ClassifyOptions options)
+        {
+            // save the model
+            var model = new MultinomiaNaiveBayesModel
+            {
+                LabelDist = labelDist,
+                CondProbDictionary = condProbDictionary,
+                Values = words
+            };
+
+            //save the file
+            using (var bw = new BinaryWriter(new FileStream(options.ModelFilePath, FileMode.Create)))
+            {
+                var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model));
+                bw.Write(bytes);
+            }
+
+            return options.ModelFilePath;
+        }
+
+        public Object LoadModel(ClassifyOptions options)
+        {
+            string json = String.Empty;
+
+            //read the file
+            using (var br = new BinaryReader(new FileStream(options.ModelFilePath, FileMode.Open)))
+            {
+                byte[] bytes = br.ReadBytes((int)br.BaseStream.Length);
+
+                json = Encoding.UTF8.GetString(bytes);
+            }
+
+            var model = JsonConvert.DeserializeObject<MultinomiaNaiveBayesModel>(json);
+
+            labelDist = model.LabelDist;
+            condProbDictionary = model.CondProbDictionary;
+            words = model.Values;
+
+            return model;
+        }
     }
 
     public class FeaturesWithLabel

BotSharp.NLP/Classify/SVMClassifier.cs

@@ -48,12 +48,12 @@ namespace BotSharp.NLP.Classify
             return Prediction.PredictLabelsProbability(options.Model, scaled);
         }
 
-        public void Train(List<Tuple<string, double[]>> featureSets, double[] values, ClassifyOptions options)
+        public void Train(List<Sentence> sentences, ClassifyOptions options)
         {
             // SVMClassifierTrain(featureSets, options);
         }
 
-        public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
+        public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
         {
             throw new NotImplementedException();
         }
@@ -154,5 +154,15 @@ namespace BotSharp.NLP.Classify
 
             return labeledFeatureSet;
         }
+
+        public string SaveModel(ClassifyOptions options)
+        {
+            throw new NotImplementedException();
+        }
+
+        object IClassifier.LoadModel(ClassifyOptions options)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

BotSharp.NLP/Txt2Vec/OneHotEncoder.cs

@@ -15,17 +15,17 @@ namespace BotSharp.NLP.Txt2Vec
     {
         public List<Sentence> Sentences { get; set; }
 
-        private List<string> words;
+        public List<string> Words { get; set; }
 
         public void Encode(Sentence sentence)
         {
             InitDictionary();
 
-            var vector = words.Select(x => 0D).ToArray();
+            var vector = Words.Select(x => 0D).ToArray();
 
             sentence.Words.ForEach(w =>
             {
-                int index = words.IndexOf(w.Text.ToLower());
+                int index = Words.IndexOf(w.Text.ToLower());
                 if(index > 0)
                 {
                     vector[index] = 1;
@@ -35,24 +35,29 @@ namespace BotSharp.NLP.Txt2Vec
             sentence.Vector = vector;
         }
 
-        public void EncodeAll()
+        public List<string> EncodeAll()
         {
             InitDictionary();
+            
             Sentences.ForEach(sent => Encode(sent));
             //Parallel.ForEach(Sentences, sent => Encode(sent));
+
+            return Words;
         }
 
-        private void InitDictionary()
+        private List<string> InitDictionary()
         {
-            if (words == null)
+            if (Words == null)
             {
-                words = new List<string>();
+                Words = new List<string>();
                 Sentences.ForEach(x =>
                 {
-                    words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
+                    Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
                 });
-                words = words.Distinct().OrderBy(x => x).ToList();
+                Words = Words.Distinct().OrderBy(x => x).ToList();
             }
+
+            return Words;
         }
     }
 }