diff --git a/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj b/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj index fb050712..02aaf56e 100644 --- a/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj +++ b/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj @@ -13,9 +13,9 @@ - - - + + + diff --git a/BotSharp.MachineLearning.UnitTest/SVM/ClassificationTests.cs b/BotSharp.MachineLearning.UnitTest/SVM/ClassificationTests.cs new file mode 100644 index 00000000..4ffa197e --- /dev/null +++ b/BotSharp.MachineLearning.UnitTest/SVM/ClassificationTests.cs @@ -0,0 +1,126 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SVM; +using SVM.BotSharp.MachineLearning; +using SVM.BotSharp.MachineLearningTest; + +namespace BotSharp.MachineLearning.UnitTest.SVM +{ + [TestClass] + public class ClassificationTests + { + [TestMethod] + public void TestTwoClass() + { + SvmType[] svmTypes = new SvmType[]{SvmType.C_SVC, SvmType.NU_SVC}; + KernelType[] kernelTypes = new KernelType[]{KernelType.LINEAR, KernelType.POLY, KernelType.RBF, KernelType.SIGMOID}; + + foreach (SvmType svm in svmTypes) + { + foreach (KernelType kernel in kernelTypes) + { + double score = testTwoClassModel(100, svm, kernel); + + Assert.AreEqual(1, score, .01, string.Format("SVM {0} with Kernel {1} did not train correctly", svm, kernel)); + } + } + } + + [TestMethod] + public void TestTwoClassProbability() + { + SvmType[] svmTypes = new SvmType[] { SvmType.C_SVC, SvmType.NU_SVC }; + KernelType[] kernelTypes = new KernelType[] { KernelType.LINEAR, KernelType.POLY, KernelType.RBF, KernelType.SIGMOID }; + + foreach (SvmType svm in svmTypes) + { + foreach (KernelType kernel in kernelTypes) + { + double score = testTwoClassModel(100, svm, kernel, true); + + Assert.AreEqual(1, score, .01, string.Format("SVM {0} with Kernel {1} did not train correctly", svm, kernel)); + } + } + } + + [TestMethod] + public void TestMulticlass() + { + SvmType[] svmTypes = new SvmType[] { SvmType.C_SVC, SvmType.NU_SVC }; + KernelType[] kernelTypes = new KernelType[] { KernelType.LINEAR, KernelType.POLY, KernelType.RBF, KernelType.SIGMOID }; + + foreach (SvmType svm in svmTypes) + { + foreach (KernelType kernel in kernelTypes) + { + double score = testMulticlassModel(8, 100, svm, kernel); + + Assert.AreEqual(1, score, .1, string.Format("SVM {0} with Kernel {1} did not train correctly", svm, kernel)); + } + } + } + + [TestMethod] + public void TestMulticlassProbability() + { + SvmType[] svmTypes = new SvmType[] { SvmType.C_SVC, SvmType.NU_SVC }; + KernelType[] kernelTypes = new KernelType[] { KernelType.LINEAR, KernelType.POLY, KernelType.RBF, KernelType.SIGMOID }; + + foreach (SvmType svm in svmTypes) + { + foreach (KernelType kernel in kernelTypes) + { + double score = testMulticlassModel(8, 100, svm, kernel, true); + + Assert.AreEqual(1, score, .1, string.Format("SVM {0} with Kernel {1} did not train correctly", svm, kernel)); + } + } + } + + private double testTwoClassModel(int count, SvmType svm, KernelType kernel, bool probability = false, string outputFile = null) + { + Problem train = SVMUtilities.CreateTwoClassProblem(count); + Parameter param = new Parameter(); + RangeTransform transform = RangeTransform.Compute(train); + Problem scaled = transform.Scale(train); + param.Gamma = .5; + param.SvmType = svm; + param.KernelType = kernel; + param.Probability = probability; + if (svm == SvmType.C_SVC) + { + param.Weights[-1] = 1; + param.Weights[1] = 1; + } + + Model model = Training.Train(scaled, param); + + Problem test = SVMUtilities.CreateTwoClassProblem(count, false); + scaled = transform.Scale(test); + return Prediction.Predict(scaled, outputFile, model, false); + } + + private double testMulticlassModel(int numberOfClasses, int count, SvmType svm, KernelType kernel, bool probability = false, string outputFile = null) + { + Problem train = SVMUtilities.CreateMulticlassProblem(numberOfClasses, count); + Parameter param = new Parameter(); + RangeTransform transform = RangeTransform.Compute(train); + Problem scaled = transform.Scale(train); + param.Gamma = 1.0 / 3; + param.SvmType = svm; + param.KernelType = kernel; + param.Probability = probability; + if (svm == SvmType.C_SVC) + { + for (int i = 0; i < numberOfClasses; i++) + param.Weights[i] = 1; + } + + Model model = Training.Train(scaled, param); + + Problem test = SVMUtilities.CreateMulticlassProblem(numberOfClasses, count, false); + scaled = transform.Scale(test); + return Prediction.Predict(scaled, outputFile, model, false); + } + } +} diff --git a/BotSharp.MachineLearning.UnitTest/SVM/IOTests.cs b/BotSharp.MachineLearning.UnitTest/SVM/IOTests.cs new file mode 100644 index 00000000..09b468b2 --- /dev/null +++ b/BotSharp.MachineLearning.UnitTest/SVM/IOTests.cs @@ -0,0 +1,76 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SVM; +using System.Linq; +using System.IO; +using System.Text; +using SVM.BotSharp.MachineLearning; +using SVM.BotSharp.MachineLearningTest; + +namespace BotSharp.MachineLearning.UnitTest.SVM +{ + [TestClass] + public class IOTests + { + [TestMethod] + public void ReadProblem() + { + Problem expected = SVMUtilities.CreateTwoClassProblem(100); + Problem actual = Problem.Read("train0.problem"); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void WriteProblem() + { + Problem prob = SVMUtilities.CreateTwoClassProblem(100); + using (MemoryStream stream = new MemoryStream()) + using (StreamReader input = new StreamReader("train0.problem")) + { + Problem.Write(stream, prob); + string expected = input.ReadToEnd().Replace("\r\n", "\n"); + string actual = Encoding.ASCII.GetString(stream.ToArray()); + Assert.AreEqual(expected, actual); + } + } + + [TestMethod] + public void ReadModel() + { + Problem train = SVMUtilities.CreateTwoClassProblem(100); + Parameter param = new Parameter(); + RangeTransform transform = RangeTransform.Compute(train); + Problem scaled = transform.Scale(train); + param.KernelType = KernelType.LINEAR; + + Training.SetRandomSeed(SVMUtilities.TRAINING_SEED); + Model expected = Training.Train(scaled, param); + Model actual = Model.Read("svm0.model"); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void WriteModel() + { + Problem train = SVMUtilities.CreateTwoClassProblem(100); + Parameter param = new Parameter(); + RangeTransform transform = RangeTransform.Compute(train); + Problem scaled = transform.Scale(train); + param.KernelType = KernelType.LINEAR; + + Training.SetRandomSeed(SVMUtilities.TRAINING_SEED); + Model model = Training.Train(scaled, param); + + using (MemoryStream stream = new MemoryStream()) + using (StreamReader input = new StreamReader("svm0.model")) + { + Model.Write(stream, model); + string expected = input.ReadToEnd().Replace("\r\n", "\n"); + string actual = Encoding.ASCII.GetString(stream.ToArray()); + Assert.AreEqual(expected, actual); + } + } + } +} diff --git a/BotSharp.MachineLearning.UnitTest/SVM/RegressionTests.cs b/BotSharp.MachineLearning.UnitTest/SVM/RegressionTests.cs new file mode 100644 index 00000000..44e81458 --- /dev/null +++ b/BotSharp.MachineLearning.UnitTest/SVM/RegressionTests.cs @@ -0,0 +1,48 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SVM; +using SVM.BotSharp.MachineLearning; +using SVM.BotSharp.MachineLearningTest; + +namespace BotSharp.MachineLearning.UnitTest.SVM +{ + [TestClass] + public class RegressionTests + { + [TestMethod] + public void TestRegression() + { + SvmType[] svmTypes = new SvmType[] { SvmType.NU_SVR, SvmType.EPSILON_SVR }; + // LINEAR kernel is pretty horrible for regression + KernelType[] kernelTypes = new KernelType[] { KernelType.LINEAR, KernelType.RBF, KernelType.SIGMOID }; + + foreach (SvmType svm in svmTypes) + { + foreach (KernelType kernel in kernelTypes) + { + double error = testRegressionModel(100, svm, kernel); + + Assert.AreEqual(0, error, 2, string.Format("SVM {0} with Kernel {1} did not train correctly", svm, kernel)); + } + } + } + + private double testRegressionModel(int count, SvmType svm, KernelType kernel, string outputFile = null) + { + Problem train = SVMUtilities.CreateRegressionProblem(count); + Parameter param = new Parameter(); + RangeTransform transform = RangeTransform.Compute(train); + Problem scaled = transform.Scale(train); + param.Gamma = 1.0 / 2; + param.SvmType = svm; + param.KernelType = kernel; + param.Degree = 2; + + Model model = Training.Train(scaled, param); + + Problem test = SVMUtilities.CreateRegressionProblem(count, false); + scaled = transform.Scale(test); + return Prediction.Predict(scaled, outputFile, model, false); + } + } +} diff --git a/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs b/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs index 931ba99e..a0633963 100644 --- a/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs +++ b/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs @@ -1,4 +1,22 @@ -using BotSharp.MachineLearning.CRFLite.Decoder; +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.MachineLearning.CRFLite.Decoder; using System; using System.Collections.Generic; using System.IO; @@ -65,7 +83,7 @@ namespace BotSharp.MachineLearning.CRFLite if (inbuf.Count == 0) { //Empty input string - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } ret = tagger.reset(); @@ -94,7 +112,7 @@ namespace BotSharp.MachineLearning.CRFLite return ret; } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } @@ -109,7 +127,7 @@ namespace BotSharp.MachineLearning.CRFLite if (inbuf.Count == 0) { //Empty input string - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } ret = tagger.reset(); @@ -138,7 +156,7 @@ namespace BotSharp.MachineLearning.CRFLite return ret; } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } } } diff --git a/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs b/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs index fb228bb8..6bb93348 100644 --- a/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs +++ b/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs @@ -1,4 +1,22 @@ -using BotSharp.MachineLearning.CRFLite.Encoder; +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.MachineLearning.CRFLite.Encoder; using System; using System.Collections.Generic; using System.Linq; diff --git a/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs b/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs index 581fa235..f5f3fa1b 100644 --- a/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs +++ b/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs @@ -149,7 +149,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder } } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } public int initNbest() @@ -164,10 +164,10 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder eos.next = null; if (BaseUtils.heap_insert(eos, heap_queue) < 0) { - return BaseUtils.ERROR_INSERT_HEAP_FAILED; + return BaseUtils.RETURN_INSERT_HEAP_FAILED; } } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } public int next() @@ -199,7 +199,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder n.next = top; if (BaseUtils.heap_insert(n, heap_queue) < 0) { - return BaseUtils.ERROR_INSERT_HEAP_FAILED; + return BaseUtils.RETURN_INSERT_HEAP_FAILED; } } } @@ -212,7 +212,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder Z_ = cost_ = 0.0; BaseUtils.heap_reset(heap_queue); - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -235,7 +235,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder } } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } public int add(List> row_p) @@ -243,7 +243,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder x_ = row_p; word_num = (short)x_.Count; - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } @@ -270,7 +270,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder break; } } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } //Label input string. The result is saved as result [] @@ -282,7 +282,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder //no word need to be labeled if (word_num == 0) { - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } //building feature set @@ -326,7 +326,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } @@ -334,7 +334,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder { if (word_num <= 0) { - return BaseUtils.ERROR_INVALIDATED_PARAMETER; + return BaseUtils.RETURN_INVALIDATED_PARAMETER; } using (var v = _buildersPool.GetOrCreate()) { @@ -351,7 +351,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder var res = featureIndex.apply_rule(templ, cur, builder, this); if (res == null) { - return BaseUtils.ERROR_EMPTY_FEATURE; + return BaseUtils.RETURN_EMPTY_FEATURE; } id = featureIndex.get_id(res.ToString()); if (id != -1) @@ -373,7 +373,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder var strFeature = featureIndex.apply_rule(templ, cur, builder, this); if (strFeature == null) { - return BaseUtils.ERROR_EMPTY_FEATURE; + return BaseUtils.RETURN_EMPTY_FEATURE; } id = featureIndex.get_id(strFeature.ToString()); @@ -387,7 +387,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder feature_cache_size++; } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } } @@ -465,7 +465,7 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder } } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } } } diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs b/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs index 797d27da..6d234329 100644 --- a/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs +++ b/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs @@ -92,7 +92,7 @@ namespace BotSharp.MachineLearning.CRFLite.Encoder return pair.Key; } - return BaseUtils.ERROR_INVALIDATED_FEATURE; + return BaseUtils.RETURN_INVALIDATED_FEATURE; } public long GetOrAddId(string key) diff --git a/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs b/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs index d623dc97..bddf573b 100644 --- a/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs +++ b/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs @@ -19,7 +19,7 @@ namespace BotSharp.MachineLearning.CRFLite //build raw result at first var iRet = termbuf_build(term_buf); - if (iRet != BaseUtils.ERROR_SUCCESS) + if (iRet != BaseUtils.RETURN_SUCCESS) { return iRet; } @@ -82,7 +82,7 @@ namespace BotSharp.MachineLearning.CRFLite } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } @@ -125,7 +125,7 @@ namespace BotSharp.MachineLearning.CRFLite } } - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } } } diff --git a/BotSharp.MachineLearning/CRFLite/Tagger.cs b/BotSharp.MachineLearning/CRFLite/Tagger.cs index 94f81824..ac5b83f8 100644 --- a/BotSharp.MachineLearning/CRFLite/Tagger.cs +++ b/BotSharp.MachineLearning/CRFLite/Tagger.cs @@ -70,7 +70,7 @@ namespace BotSharp.MachineLearning.CRFLite cost_ = -node_[s, result_[s]].bestCost; - return BaseUtils.ERROR_SUCCESS; + return BaseUtils.RETURN_SUCCESS; } private void calcAlpha(int m, int n) diff --git a/BotSharp.MachineLearning/CRFLite/Utils.cs b/BotSharp.MachineLearning/CRFLite/Utils.cs index 90d85fb8..3376f6d3 100644 --- a/BotSharp.MachineLearning/CRFLite/Utils.cs +++ b/BotSharp.MachineLearning/CRFLite/Utils.cs @@ -32,16 +32,15 @@ namespace BotSharp.MachineLearning.CRFLite public const int MODEL_TYPE_NORM = 100; - - public const int ERROR_INVALIDATED_FEATURE = -8; - public const int ERROR_HEAP_SIZE_TOO_BIG = -7; - public const int ERROR_INSERT_HEAP_FAILED = -6; - public const int ERROR_EMPTY_FEATURE = -5; - public const int ERROR_INVALIDATED_PARAMETER = -4; - public const int ERROR_WRONG_STATUS = -3; - public const int ERROR_TOO_LONG_WORD = -2; - public const int ERROR_UNKNOWN = -1; - public const int ERROR_SUCCESS = 0; + public const int RETURN_INVALIDATED_FEATURE = -8; + public const int RETURN_HEAP_SIZE_TOO_BIG = -7; + public const int RETURN_INSERT_HEAP_FAILED = -6; + public const int RETURN_EMPTY_FEATURE = -5; + public const int RETURN_INVALIDATED_PARAMETER = -4; + public const int RETURN_WRONG_STATUS = -3; + public const int RETURN_TOO_LONG_WORD = -2; + public const int RETURN_UNKNOWN = -1; + public const int RETURN_SUCCESS = 0; public static Heap heap_init(int max_size) { @@ -82,7 +81,7 @@ namespace BotSharp.MachineLearning.CRFLite { if (H.size >= H.capacity) { - return BaseUtils.ERROR_HEAP_SIZE_TOO_BIG; + return BaseUtils.RETURN_HEAP_SIZE_TOO_BIG; } var i = ++H.size; while (i != 1 && H.elem_ptr_list[i / 2].fx > qe.fx) diff --git a/BotSharp.MachineLearning/SVM/Cache.cs b/BotSharp.MachineLearning/SVM/Cache.cs new file mode 100644 index 00000000..9fbf4931 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Cache.cs @@ -0,0 +1,161 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; + +namespace SVM.BotSharp.MachineLearning +{ + internal class Cache + { + private int _count; + private long _size; + + private sealed class head_t + { + public head_t(Cache enclosingInstance) + { + _enclosingInstance = enclosingInstance; + } + private Cache _enclosingInstance; + public Cache EnclosingInstance + { + get + { + return _enclosingInstance; + } + } + internal head_t prev, next; // a cicular list + internal float[] data; + internal int len; // data[0,len) is cached in this entry + } + + private head_t[] head; + private head_t lru_head; + + public Cache(int count, long size) + { + _count = count; + _size = size; + head = new head_t[_count]; + for (int i = 0; i < _count; i++) + head[i] = new head_t(this); + _size /= 4; + _size -= _count * (16 / 4); // sizeof(head_t) == 16 + lru_head = new head_t(this); + lru_head.next = lru_head.prev = lru_head; + } + + private void lru_delete(head_t h) + { + // delete from current location + h.prev.next = h.next; + h.next.prev = h.prev; + } + + private void lru_insert(head_t h) + { + // insert to last position + h.next = lru_head; + h.prev = lru_head.prev; + h.prev.next = h; + h.next.prev = h; + } + + private static void swap(ref T lhs, ref T rhs) + { + T tmp = lhs; + lhs = rhs; + rhs = tmp; + } + + // request data [0,len) + // return some position p where [p,len) need to be filled + // (p >= len if nothing needs to be filled) + // java: simulate pointer using single-element array + public int GetData(int index, out float[] data, int len) + { + head_t h = head[index]; + if (h.len > 0) + lru_delete(h); + int more = len - h.len; + + if (more > 0) + { + // free old space + while (_size < more) + { + head_t old = lru_head.next; + lru_delete(old); + _size += old.len; + old.data = null; + old.len = 0; + } + + // allocate new space + float[] new_data = new float[len]; + if (h.data != null) + Array.Copy(h.data, 0, new_data, 0, h.len); + h.data = new_data; + _size -= more; + swap(ref h.len, ref len); + } + + lru_insert(h); + data = h.data; + return len; + } + + public void SwapIndex(int i, int j) + { + if (i == j) + return; + + if (head[i].len > 0) + lru_delete(head[i]); + if (head[j].len > 0) + lru_delete(head[j]); + swap(ref head[i].data, ref head[j].data); + swap(ref head[i].len, ref head[j].len); + if (head[i].len > 0) + lru_insert(head[i]); + if (head[j].len > 0) + lru_insert(head[j]); + + if (i > j) + swap(ref i, ref j); + + for (head_t h = lru_head.next; h != lru_head; h = h.next) + { + if (h.len > i) + { + if (h.len > j) + swap(ref h.data[i], ref h.data[j]); + else + { + // give up + lru_delete(h); + _size += h.len; + h.data = null; + h.len = 0; + } + } + } + } + } +} diff --git a/BotSharp.MachineLearning/SVM/GaussianTransform.cs b/BotSharp.MachineLearning/SVM/GaussianTransform.cs new file mode 100644 index 00000000..0b9b10b9 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/GaussianTransform.cs @@ -0,0 +1,209 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Collections.Generic; +using System.IO; +using System.Globalization; +using System.Threading; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// A transform which learns the mean and variance of a sample set and uses these to transform new data + /// so that it has zero mean and unit variance. + /// + public class GaussianTransform : IRangeTransform + { + private double[] _means; + private double[] _stddevs; + + /// + /// Determines the Gaussian transform for the provided problem. + /// + /// The Problem to analyze + /// The Gaussian transform for the problem + public static GaussianTransform Compute(Problem prob) + { + int[] counts = new int[prob.MaxIndex]; + double[] means = new double[prob.MaxIndex]; + foreach (Node[] sample in prob.X) + { + for (int i = 0; i < sample.Length; i++) + { + means[sample[i].Index-1] += sample[i].Value; + counts[sample[i].Index-1]++; + } + } + for (int i = 0; i < prob.MaxIndex; i++) + { + if (counts[i] == 0) + counts[i] = 2; + means[i] /= counts[i]; + } + + double[] stddevs = new double[prob.MaxIndex]; + foreach (Node[] sample in prob.X) + { + for (int i = 0; i < sample.Length; i++) + { + double diff = sample[i].Value - means[sample[i].Index - 1]; + stddevs[sample[i].Index - 1] += diff * diff; + } + } + for (int i = 0; i < prob.MaxIndex; i++) + { + if (stddevs[i] == 0) + continue; + stddevs[i] /= (counts[i] - 1); + stddevs[i] = Math.Sqrt(stddevs[i]); + } + + return new GaussianTransform(means, stddevs); + } + + /// + /// Constructor. + /// + /// Means in each dimension + /// Standard deviation in each dimension + public GaussianTransform(double[] means, double[] stddevs) + { + _means = means; + _stddevs = stddevs; + } + + /// + /// Saves the transform to the disk. The samples are not stored, only the + /// statistics. + /// + /// The destination stream + /// The transform + public static void Write(Stream stream, GaussianTransform transform) + { + TemporaryCulture.Start(); + + StreamWriter output = new StreamWriter(stream); + output.WriteLine(transform._means.Length); + for (int i = 0; i < transform._means.Length; i++) + output.WriteLine("{0} {1}", transform._means[i], transform._stddevs[i]); + output.Flush(); + + TemporaryCulture.Stop(); + } + + /// + /// Reads a GaussianTransform from the provided stream. + /// + /// The source stream + /// The transform + public static GaussianTransform Read(Stream stream) + { + TemporaryCulture.Start(); + + StreamReader input = new StreamReader(stream); + int length = int.Parse(input.ReadLine(), CultureInfo.InvariantCulture); + double[] means = new double[length]; + double[] stddevs = new double[length]; + for (int i = 0; i < length; i++) + { + string[] parts = input.ReadLine().Split(); + means[i] = double.Parse(parts[0], CultureInfo.InvariantCulture); + stddevs[i] = double.Parse(parts[1], CultureInfo.InvariantCulture); + } + + TemporaryCulture.Stop(); + + return new GaussianTransform(means, stddevs); + } + + /// + /// Saves the transform to the disk. The samples are not stored, only the + /// statistics. + /// + /// The destination filename + /// The transform + public static void Write(string filename, GaussianTransform transform) + { + FileStream output = File.Open(filename, FileMode.Create); + try + { + Write(output, transform); + } + finally + { + output.Close(); + } + } + + /// + /// Reads a GaussianTransform from the provided stream. + /// + /// The source filename + /// The transform + public static GaussianTransform Read(string filename) + { + FileStream input = File.Open(filename, FileMode.Open); + try + { + return Read(input); + } + finally + { + input.Close(); + } + } + + #region IRangeTransform Members + + /// + /// Transform the input value using the transform stored for the provided index. + /// + /// Input value + /// Index of the transform to use + /// The transformed value + public double Transform(double input, int index) + { + index--; + if (_stddevs[index] == 0) + return 0; + double diff = input - _means[index]; + diff /= _stddevs[index]; + return diff; + } + /// + /// Transforms the input array. + /// + /// The array to transform + /// The transformed array + public Node[] Transform(Node[] input) + { + Node[] output = new Node[input.Length]; + for (int i = 0; i < output.Length; i++) + { + int index = input[i].Index; + double value = input[i].Value; + output[i] = new Node(index, Transform(value, index)); + } + return output; + } + + #endregion + } +} diff --git a/BotSharp.MachineLearning/SVM/IRangeTransform.cs b/BotSharp.MachineLearning/SVM/IRangeTransform.cs new file mode 100644 index 00000000..45f9930e --- /dev/null +++ b/BotSharp.MachineLearning/SVM/IRangeTransform.cs @@ -0,0 +1,41 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Interface implemented by range transforms. + /// + public interface IRangeTransform + { + /// + /// Transform the input value using the transform stored for the provided index. + /// + /// Input value + /// Index of the transform to use + /// The transformed value + double Transform(double input, int index); + /// + /// Transforms the input array. + /// + /// The array to transform + /// The transformed array + Node[] Transform(Node[] input); + } +} diff --git a/BotSharp.MachineLearning/SVM/Kernel.cs b/BotSharp.MachineLearning/SVM/Kernel.cs new file mode 100644 index 00000000..73d05d93 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Kernel.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace SVM.BotSharp.MachineLearning +{ + internal interface IQMatrix + { + float[] GetQ(int column, int len); + double[] GetQD(); + void SwapIndex(int i, int j); + } + + internal abstract class Kernel : IQMatrix + { + private Node[][] _x; + private double[] _xSquare; + + private KernelType _kernelType; + private int _degree; + private double _gamma; + private double _coef0; + + public abstract float[] GetQ(int column, int len); + public abstract double[] GetQD(); + + public virtual void SwapIndex(int i, int j) + { + _x.SwapIndex(i, j); + + if (_xSquare != null) + { + _xSquare.SwapIndex(i, j); + } + } + + private static double powi(double value, int times) + { + double tmp = value, ret = 1.0; + + for (int t = times; t > 0; t /= 2) + { + if (t % 2 == 1) ret *= tmp; + tmp = tmp * tmp; + } + return ret; + } + + public double KernelFunction(int i, int j) + { + switch (_kernelType) + { + case KernelType.LINEAR: + return dot(_x[i], _x[j]); + case KernelType.POLY: + return powi(_gamma * dot(_x[i], _x[j]) + _coef0, _degree); + case KernelType.RBF: + return Math.Exp(-_gamma * (_xSquare[i] + _xSquare[j] - 2 * dot(_x[i], _x[j]))); + case KernelType.SIGMOID: + return Math.Tanh(_gamma * dot(_x[i], _x[j]) + _coef0); + case KernelType.PRECOMPUTED: + return _x[i][(int)(_x[j][0].Value)].Value; + default: + return 0; + } + } + + public Kernel(int l, Node[][] x_, Parameter param) + { + _kernelType = param.KernelType; + _degree = param.Degree; + _gamma = param.Gamma; + _coef0 = param.Coefficient0; + + _x = (Node[][])x_.Clone(); + + if (_kernelType == KernelType.RBF) + { + _xSquare = new double[l]; + for (int i = 0; i < l; i++) + _xSquare[i] = dot(_x[i], _x[i]); + } + else _xSquare = null; + } + + private static double dot(Node[] xNodes, Node[] yNodes) + { + double sum = 0; + int xlen = xNodes.Length; + int ylen = yNodes.Length; + int i = 0; + int j = 0; + Node x = xNodes[0]; + Node y = yNodes[0]; + while (true) + { + if (x._index == y._index) + { + sum += x._value * y._value; + i++; + j++; + if (i < xlen && j < ylen) + { + x = xNodes[i]; + y = yNodes[j]; + } + else if (i < xlen) + { + x = xNodes[i]; + break; + } + else if (j < ylen) + { + y = yNodes[j]; + break; + } + else break; + } + else + { + if (x._index > y._index) + { + ++j; + if (j < ylen) + y = yNodes[j]; + else break; + } + else + { + ++i; + if (i < xlen) + x = xNodes[i]; + else break; + } + } + } + return sum; + } + + private static double computeSquaredDistance(Node[] xNodes, Node[] yNodes) + { + Node x = xNodes[0]; + Node y = yNodes[0]; + int xLength = xNodes.Length; + int yLength = yNodes.Length; + int xIndex = 0; + int yIndex = 0; + double sum = 0; + + while (true) + { + if (x._index == y._index) + { + double d = x._value - y._value; + sum += d * d; + xIndex++; + yIndex++; + if (xIndex < xLength && yIndex < yLength) + { + x = xNodes[xIndex]; + y = yNodes[yIndex]; + } + else if(xIndex < xLength){ + x = xNodes[xIndex]; + break; + } + else if(yIndex < yLength){ + y = yNodes[yIndex]; + break; + }else break; + } + else if (x._index > y._index) + { + sum += y._value * y._value; + if (++yIndex < yLength) + y = yNodes[yIndex]; + else break; + } + else + { + sum += x._value * x._value; + if (++xIndex < xLength) + x = xNodes[xIndex]; + else break; + } + } + + for (; xIndex < xLength; xIndex++) + { + double d = xNodes[xIndex]._value; + sum += d * d; + } + + for (; yIndex < yLength; yIndex++) + { + double d = yNodes[yIndex]._value; + sum += d * d; + } + + return sum; + } + + public static double KernelFunction(Node[] x, Node[] y, Parameter param) + { + switch (param.KernelType) + { + case KernelType.LINEAR: + return dot(x, y); + case KernelType.POLY: + return powi(param.Degree * dot(x, y) + param.Coefficient0, param.Degree); + case KernelType.RBF: + { + double sum = computeSquaredDistance(x, y); + return Math.Exp(-param.Gamma * sum); + } + case KernelType.SIGMOID: + return Math.Tanh(param.Gamma * dot(x, y) + param.Coefficient0); + case KernelType.PRECOMPUTED: + return x[(int)(y[0].Value)].Value; + default: + return 0; + } + } + } +} diff --git a/BotSharp.MachineLearning/SVM/Model.cs b/BotSharp.MachineLearning/SVM/Model.cs new file mode 100644 index 00000000..25600151 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Model.cs @@ -0,0 +1,420 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + + +using System; +using System.IO; +using System.Threading; +using System.Globalization; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Encapsulates an SVM Model. + /// + [Serializable] + public class Model + { + internal Model() + { + } + + /// + /// Parameter object. + /// + public Parameter Parameter{get;set;} + + /// + /// Number of classes in the model. + /// + public int NumberOfClasses{get;set;} + + /// + /// Total number of support vectors. + /// + public int SupportVectorCount { get; set; } + + /// + /// The support vectors. + /// + public Node[][] SupportVectors{get;set;} + + /// + /// The coefficients for the support vectors. + /// + public double[][] SupportVectorCoefficients{get;set;} + + /// + /// Values in [1,...,num_training_data] to indicate SVs in the training set + /// + public int[] SupportVectorIndices { get; set; } + + /// + /// Constants in decision functions + /// + public double[] Rho{get;set;} + + /// + /// First pairwise probability. + /// + public double[] PairwiseProbabilityA{get;set;} + + /// + /// Second pairwise probability. + /// + public double[] PairwiseProbabilityB{get;set;} + + // for classification only + + /// + /// Class labels. + /// + public int[] ClassLabels{get;set;} + + /// + /// Number of support vectors per class. + /// + public int[] NumberOfSVPerClass{get;set;} + + public override bool Equals(object obj) + { + Model test = obj as Model; + if (test == null) + return false; + + bool same = ClassLabels.IsEqual(test.ClassLabels); + same = same && NumberOfClasses == test.NumberOfClasses; + same = same && NumberOfSVPerClass.IsEqual(test.NumberOfSVPerClass); + if(PairwiseProbabilityA != null) + same = same && PairwiseProbabilityA.IsEqual(test.PairwiseProbabilityA); + if(PairwiseProbabilityB != null) + same = same && PairwiseProbabilityB.IsEqual(test.PairwiseProbabilityB); + same = same && Parameter.Equals(test.Parameter); + same = same && Rho.IsEqual(test.Rho); + same = same && SupportVectorCoefficients.IsEqual(test.SupportVectorCoefficients); + same = same && SupportVectorCount == test.SupportVectorCount; + same = same && SupportVectors.IsEqual(test.SupportVectors); + return same; + } + + public override int GetHashCode() + { + return ClassLabels.ComputeHashcode() + + NumberOfClasses.GetHashCode() + + NumberOfSVPerClass.ComputeHashcode() + + PairwiseProbabilityA.ComputeHashcode() + + PairwiseProbabilityB.ComputeHashcode() + + Parameter.GetHashCode() + + Rho.ComputeHashcode() + + SupportVectorCoefficients.ComputeHashcode() + + SupportVectorCount.GetHashCode() + + SupportVectors.ComputeHashcode(); + } + + /// + /// Reads a Model from the provided file. + /// + /// The name of the file containing the Model + /// the Model + public static Model Read(string filename) + { + FileStream input = File.OpenRead(filename); + try + { + return Read(input); + } + finally + { + input.Close(); + } + } + + /// + /// Reads a Model from the provided stream. + /// + /// The stream from which to read the Model. + /// the Model + public static Model Read(Stream stream) + { + TemporaryCulture.Start(); + + StreamReader input = new StreamReader(stream); + + // read parameters + + Model model = new Model(); + Parameter param = new Parameter(); + model.Parameter = param; + model.Rho = null; + model.PairwiseProbabilityA = null; + model.PairwiseProbabilityB = null; + model.ClassLabels = null; + model.NumberOfSVPerClass = null; + + bool headerFinished = false; + while (!headerFinished) + { + string line = input.ReadLine(); + string cmd, arg; + int splitIndex = line.IndexOf(' '); + if (splitIndex >= 0) + { + cmd = line.Substring(0, splitIndex); + arg = line.Substring(splitIndex + 1); + } + else + { + cmd = line; + arg = ""; + } + arg = arg.ToLower(); + + int i,n; + switch(cmd){ + case "svm_type": + param.SvmType = (SvmType)Enum.Parse(typeof(SvmType), arg.ToUpper()); + break; + + case "kernel_type": + if (arg == "polynomial") + arg = "poly"; + param.KernelType = (KernelType)Enum.Parse(typeof(KernelType), arg.ToUpper()); + break; + + case "degree": + param.Degree = int.Parse(arg); + break; + + case "gamma": + param.Gamma = double.Parse(arg); + break; + + case "coef0": + param.Coefficient0 = double.Parse(arg); + break; + + case "nr_class": + model.NumberOfClasses = int.Parse(arg); + break; + + case "total_sv": + model.SupportVectorCount = int.Parse(arg); + break; + + case "rho": + n = model.NumberOfClasses * (model.NumberOfClasses - 1) / 2; + model.Rho = new double[n]; + string[] rhoParts = arg.Split(); + for(i=0; i + /// Writes a model to the provided filename. This will overwrite any previous data in the file. + /// + /// The desired file + /// The Model to write + public static void Write(string filename, Model model) + { + FileStream stream = File.Open(filename, FileMode.Create); + try + { + Write(stream, model); + } + finally + { + stream.Close(); + } + } + + /// + /// Writes a model to the provided stream. + /// + /// The output stream + /// The model to write + public static void Write(Stream stream, Model model) + { + TemporaryCulture.Start(); + + StreamWriter output = new StreamWriter(stream); + + Parameter param = model.Parameter; + + output.Write("svm_type {0}\n", param.SvmType); + output.Write("kernel_type {0}\n", param.KernelType); + + if (param.KernelType == KernelType.POLY) + output.Write("degree {0}\n", param.Degree); + + if (param.KernelType == KernelType.POLY || param.KernelType == KernelType.RBF || param.KernelType == KernelType.SIGMOID) + output.Write("gamma {0:0.000000}\n", param.Gamma); + + if (param.KernelType == KernelType.POLY || param.KernelType == KernelType.SIGMOID) + output.Write("coef0 {0:0.000000}\n", param.Coefficient0); + + int nr_class = model.NumberOfClasses; + int l = model.SupportVectorCount; + output.Write("nr_class {0}\n", nr_class); + output.Write("total_sv {0}\n", l); + + { + output.Write("rho"); + for (int i = 0; i < nr_class * (nr_class - 1) / 2; i++) + output.Write(" {0:0.000000}", model.Rho[i]); + output.Write("\n"); + } + + if (model.ClassLabels != null) + { + output.Write("label"); + for (int i = 0; i < nr_class; i++) + output.Write(" {0}", model.ClassLabels[i]); + output.Write("\n"); + } + + if (model.PairwiseProbabilityA != null) + // regression has probA only + { + output.Write("probA"); + for (int i = 0; i < nr_class * (nr_class - 1) / 2; i++) + output.Write(" {0:0.000000}", model.PairwiseProbabilityA[i]); + output.Write("\n"); + } + if (model.PairwiseProbabilityB != null) + { + output.Write("probB"); + for (int i = 0; i < nr_class * (nr_class - 1) / 2; i++) + output.Write(" {0:0.000000}", model.PairwiseProbabilityB[i]); + output.Write("\n"); + } + + if (model.NumberOfSVPerClass != null) + { + output.Write("nr_sv"); + for (int i = 0; i < nr_class; i++) + output.Write(" {0}", model.NumberOfSVPerClass[i]); + output.Write("\n"); + } + + output.Write("SV\n"); + double[][] sv_coef = model.SupportVectorCoefficients; + Node[][] SV = model.SupportVectors; + + for (int i = 0; i < l; i++) + { + for (int j = 0; j < nr_class - 1; j++) + output.Write("{0:0.000000} ", sv_coef[j][i]); + + Node[] p = SV[i]; + if (p.Length == 0) + { + output.Write("\n"); + continue; + } + if (param.KernelType == KernelType.PRECOMPUTED) + output.Write("0:{0:0.000000}", (int)p[0].Value); + else + { + output.Write("{0}:{1:0.000000}", p[0].Index, p[0].Value); + for (int j = 1; j < p.Length; j++) + output.Write(" {0}:{1:0.000000}", p[j].Index, p[j].Value); + } + output.Write("\n"); + } + + output.Flush(); + + TemporaryCulture.Stop(); + } + } +} \ No newline at end of file diff --git a/BotSharp.MachineLearning/SVM/Node.cs b/BotSharp.MachineLearning/SVM/Node.cs new file mode 100644 index 00000000..763884af --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Node.cs @@ -0,0 +1,117 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Encapsulates a node in a Problem vector, with an index and a value (for more efficient representation + /// of sparse data. + /// + [Serializable] + public class Node : IComparable + { + internal int _index; + internal double _value; + + /// + /// Default Constructor. + /// + public Node() + { + } + /// + /// Constructor. + /// + /// The index of the value. + /// The value to store. + public Node(int index, double value) + { + _index = index; + _value = value; + } + + /// + /// Index of this Node. + /// + public int Index + { + get + { + return _index; + } + set + { + _index = value; + } + } + /// + /// Value at Index. + /// + public double Value + { + get + { + return _value; + } + set + { + _value = value; + } + } + + /// + /// String representation of this Node as {index}:{value}. + /// + /// {index}:{value} + public override string ToString() + { + return string.Format("{0}:{1}", _index, _value.Truncate()); + } + + public override bool Equals(object obj) + { + Node other = obj as Node; + if(other == null) + return false; + + return _index == other._index && _value.Truncate() == other._value.Truncate(); + } + + public override int GetHashCode() + { + return _index.GetHashCode() + _value.GetHashCode(); + } + + #region IComparable Members + + /// + /// Compares this node with another. + /// + /// The node to compare to + /// A positive number if this node is greater, a negative number if it is less than, or 0 if equal + public int CompareTo(Node other) + { + return _index.CompareTo(other._index); + } + + #endregion + } +} \ No newline at end of file diff --git a/BotSharp.MachineLearning/SVM/Parameter.cs b/BotSharp.MachineLearning/SVM/Parameter.cs new file mode 100644 index 00000000..424efb6c --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Parameter.cs @@ -0,0 +1,221 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Linq; +using System.Collections.Generic; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Contains all of the types of SVM this library can model. + /// + public enum SvmType { + /// + /// C-SVC. + /// + C_SVC, + /// + /// nu-SVC. + /// + NU_SVC, + /// + /// one-class SVM + /// + ONE_CLASS, + /// + /// epsilon-SVR + /// + EPSILON_SVR, + /// + /// nu-SVR + /// + NU_SVR + }; + /// + /// Contains the various kernel types this library can use. + /// + public enum KernelType { + /// + /// Linear: u'*v + /// + LINEAR, + /// + /// Polynomial: (gamma*u'*v + coef0)^degree + /// + POLY, + /// + /// Radial basis function: exp(-gamma*|u-v|^2) + /// + RBF, + /// + /// Sigmoid: tanh(gamma*u'*v + coef0) + /// + SIGMOID, + /// + /// Precomputed kernel + /// + PRECOMPUTED, + }; + + /// + /// This class contains the various parameters which can affect the way in which an SVM + /// is learned. Unless you know what you are doing, chances are you are best off using + /// the default values. + /// + [Serializable] + public class Parameter : ICloneable + { + /// + /// Default Constructor. Gives good default values to all parameters. + /// + public Parameter() + { + SvmType = SvmType.C_SVC; + KernelType = KernelType.RBF; + Degree = 3; + Gamma = 0; // 1/k + Coefficient0 = 0; + Nu = 0.5; + CacheSize = 40; + C = 1; + EPS = 1e-3; + P = 0.1; + Shrinking = true; + Probability = false; + Weights = new Dictionary(); + } + + /// + /// Type of SVM (default C-SVC) + /// + public SvmType SvmType{get;set;} + + /// + /// Type of kernel function (default Polynomial) + /// + public KernelType KernelType{get;set;} + + /// + /// Degree in kernel function (default 3). + /// + public int Degree{get;set;} + + /// + /// Gamma in kernel function (default 1/k) + /// + public double Gamma{get;set;} + + /// + /// Zeroeth coefficient in kernel function (default 0) + /// + public double Coefficient0{get;set;} + + /// + /// Cache memory size in MB (default 100) + /// + public double CacheSize{get;set;} + + /// + /// Tolerance of termination criterion (default 0.001) + /// + public double EPS{get;set;} + + /// + /// The parameter C of C-SVC, epsilon-SVR, and nu-SVR (default 1) + /// + public double C{get;set;} + + /// + /// Contains custom weights for class labels. Default weight value is 1. + /// + public Dictionary Weights{get; private set;} + + /// + /// The parameter nu of nu-SVC, one-class SVM, and nu-SVR (default 0.5) + /// + public double Nu{get;set;} + + /// + /// The epsilon in loss function of epsilon-SVR (default 0.1) + /// + public double P{get;set;} + + /// + /// Whether to use the shrinking heuristics, (default True) + /// + public bool Shrinking{get;set;} + + /// + /// Whether to train an SVC or SVR model for probability estimates, (default False) + /// + public bool Probability{get;set;} + + public override bool Equals(object obj) + { + Parameter other = obj as Parameter; + if (other == null) + return false; + + return other.C == C && + other.CacheSize == CacheSize && + other.Coefficient0 == Coefficient0 && + other.Degree == Degree && + other.EPS == EPS && + other.Gamma == Gamma && + other.KernelType == KernelType && + other.Nu == Nu && + other.P == P && + other.Probability == Probability && + other.Shrinking == Shrinking && + other.SvmType == SvmType && + other.Weights.ToArray().IsEqual(Weights.ToArray()); + } + + public override int GetHashCode() + { + return C.GetHashCode() + + CacheSize.GetHashCode() + + Coefficient0.GetHashCode() + + Degree.GetHashCode() + + EPS.GetHashCode() + + Gamma.GetHashCode() + + KernelType.GetHashCode() + + Nu.GetHashCode() + + P.GetHashCode() + + Probability.GetHashCode() + + Shrinking.GetHashCode() + + SvmType.GetHashCode() + + Weights.ToArray().ComputeHashcode(); + } + + + #region ICloneable Members + /// + /// Creates a memberwise clone of this parameters object. + /// + /// The clone (as type Parameter) + public object Clone() + { + return base.MemberwiseClone(); + } + + #endregion + } +} \ No newline at end of file diff --git a/BotSharp.MachineLearning/SVM/ParameterSelection.cs b/BotSharp.MachineLearning/SVM/ParameterSelection.cs new file mode 100644 index 00000000..9904dd36 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/ParameterSelection.cs @@ -0,0 +1,271 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Class representing a grid square result. + /// + public class GridSquare + { + /// + /// The C value + /// + public double C; + /// + /// The Gamma value + /// + public double Gamma; + /// + /// The cross validation score + /// + public double Score; + + public override string ToString() + { + return string.Format("{0} {1} {2}", C, Gamma, Score); + } + } + + /// + /// This class contains routines which perform parameter selection for a model which uses C-SVC and + /// an RBF kernel. + /// + public static class ParameterSelection + { + /// + /// Default number of times to divide the data. + /// + public const int NFOLD = 5; + /// + /// Default minimum power of 2 for the C value (-5) + /// + public const int MIN_C = -5; + /// + /// Default maximum power of 2 for the C value (15) + /// + public const int MAX_C = 15; + /// + /// Default power iteration step for the C value (2) + /// + public const int C_STEP = 2; + /// + /// Default minimum power of 2 for the Gamma value (-15) + /// + public const int MIN_G = -15; + /// + /// Default maximum power of 2 for the Gamma Value (3) + /// + public const int MAX_G = 3; + /// + /// Default power iteration step for the Gamma value (2) + /// + public const int G_STEP = 2; + /// + /// Used to control the degree of parallelism used in grid exploration. Default value is the number of processors. + /// + public static int Threads = Environment.ProcessorCount; + + /// + /// Returns a logarithmic list of values from minimum power of 2 to the maximum power of 2 using the provided iteration size. + /// + /// The minimum power of 2 + /// The maximum power of 2 + /// The iteration size to use in powers + /// + public static List GetList(double minPower, double maxPower, double iteration) + { + List list = new List(); + for (double d = minPower; d <= maxPower; d += iteration) + list.Add(Math.Pow(2, d)); + return list; + } + + /// + /// Performs a Grid parameter selection, trying all possible combinations of the two lists and returning the + /// combination which performed best. The default ranges of C and Gamma values are used. Use this method if there is no validation data available, and it will + /// divide it 5 times to allow 5-fold validation (training on 4/5 and validating on 1/5, 5 times). + /// + /// The training data + /// The parameters to use when optimizing + /// Function used to report results + /// The optimal C value will be put into this variable + /// The optimal Gamma value will be put into this variable + /// A list of grid squares and their results + public static List Grid( + Problem problem, + Func createParams, + Action report, + out double C, + out double Gamma) + { + return Grid(problem, createParams, GetList(MIN_C, MAX_C, C_STEP), GetList(MIN_G, MAX_G, G_STEP), report, NFOLD, out C, out Gamma); + } + /// + /// Performs a Grid parameter selection, trying all possible combinations of the two lists and returning the + /// combination which performed best. Use this method if there is no validation data available, and it will + /// divide it 5 times to allow 5-fold validation (training on 4/5 and validating on 1/5, 5 times). + /// + /// The training data + /// The parameters to use when optimizing + /// The set of C values to use + /// The set of Gamma values to use + /// Function used to report results + /// The optimal C value will be put into this variable + /// The optimal Gamma value will be put into this variable + /// A list of grid squares and their results + public static List Grid( + Problem problem, + Func createParams, + List CValues, + List GammaValues, + Action report, + out double C, + out double Gamma) + { + return Grid(problem, createParams, CValues, GammaValues, report, NFOLD, out C, out Gamma); + } + + /// + /// Performs a Grid parameter selection, trying all possible combinations of the two lists and returning the + /// combination which performed best. Use this method if validation data isn't available, as it will + /// divide the training data and train on a portion of it and test on the rest. + /// + /// The training data + /// The parameters to use when optimizing + /// The set of C values to use + /// The set of Gamma values to use + /// Function used to report results + /// The number of times the data should be divided for validation + /// The optimal C value will be placed in this variable + /// The optimal Gamma value will be placed in this variable + /// A list of grid squares and their results + public static List Grid( + Problem problem, + Func createParams, + List CValues, + List GammaValues, + Action report, + int nrfold, + out double C, + out double Gamma) + { + C = 0; + Gamma = 0; + + List squares = new List(); + foreach (double testC in CValues) + foreach (double testGamma in GammaValues) + squares.Add(new GridSquare { C = testC, Gamma = testGamma }); + + ThreadLocal parameters = new ThreadLocal(() => createParams()); + + Parallel.ForEach(squares, new ParallelOptions{MaxDegreeOfParallelism=Threads}, square => + { + parameters.Value.C = square.C; + parameters.Value.Gamma = square.Gamma; + square.Score = Training.PerformCrossValidation(problem, parameters.Value, nrfold); + if(report != null) + report(square); + }); + + GridSquare best = squares.OrderByDescending(o => o.Score).First(); + C = best.C; + Gamma = best.Gamma; + + return squares.OrderBy(o => o.C).ThenBy(o => o.Gamma).ToList(); + } + /// + /// Performs a Grid parameter selection, trying all possible combinations of the two lists and returning the + /// combination which performed best. Uses the default values of C and Gamma. + /// + /// The training data + /// The validation data + /// The parameters to use when optimizing + /// Function used to report results + /// The optimal C value will be placed in this variable + /// The optimal Gamma value will be placed in this variable + /// A list of grid squares and their results + public static List Grid( + Problem problem, + Problem validation, + Func createParams, + Action report, + out double C, + out double Gamma) + { + return Grid(problem, validation, createParams, GetList(MIN_C, MAX_C, C_STEP), GetList(MIN_G, MAX_G, G_STEP), report, out C, out Gamma); + } + /// + /// Performs a Grid parameter selection, trying all possible combinations of the two lists and returning the + /// combination which performed best. + /// + /// The training data + /// The validation data + /// The parameters to use when optimizing + /// The C values to use + /// The Gamma values to use + /// Function used to report results + /// The optimal C value will be placed in this variable + /// The optimal Gamma value will be placed in this variable + /// A list of grid squares and their results + public static List Grid( + Problem problem, + Problem validation, + Func createParams, + List CValues, + List GammaValues, + Action report, + out double C, + out double Gamma) + { + C = 0; + Gamma = 0; + + List squares = new List(); + foreach (double testC in CValues) + foreach (double testGamma in GammaValues) + squares.Add(new GridSquare { C = testC, Gamma = testGamma }); + ThreadLocal parameters = new ThreadLocal(() => createParams()); + + Parallel.ForEach(squares, new ParallelOptions { MaxDegreeOfParallelism = Threads }, square => + { + parameters.Value.C = square.C; + parameters.Value.Gamma = square.Gamma; + Model model = Training.Train(problem, parameters.Value); + square.Score = Prediction.Predict(validation, null, model, false); + if (report != null) + report(square); + }); + + GridSquare best = squares.OrderByDescending(o => o.Score).First(); + C = best.C; + Gamma = best.Gamma; + + return squares.OrderBy(o=>o.C).ThenBy(o=>o.Gamma).ToList(); + } + } +} diff --git a/BotSharp.MachineLearning/SVM/PerformanceEvaluator.cs b/BotSharp.MachineLearning/SVM/PerformanceEvaluator.cs new file mode 100644 index 00000000..8fd4a4c4 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/PerformanceEvaluator.cs @@ -0,0 +1,415 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Collections.Generic; +using System.IO; +using System.Globalization; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Class encoding a member of a ranked set of labels. + /// + public class RankPair : IComparable + { + private double _score, _label; + + /// + /// Constructor. + /// + /// Score for this pair + /// Label associated with the given score + public RankPair(double score, double label) + { + _score = score; + _label = label; + } + + /// + /// The score for this pair. + /// + public double Score + { + get + { + return _score; + } + } + + /// + /// The Label for this pair. + /// + public double Label + { + get + { + return _label; + } + } + + #region IComparable Members + + /// + /// Compares this pair to another. It will end up in a sorted list in decending score order. + /// + /// The pair to compare to + /// Whether this should come before or after the argument + public int CompareTo(RankPair other) + { + return other.Score.CompareTo(Score); + } + + #endregion + + /// + /// Returns a string representation of this pair. + /// + /// A string in the for Score:Label + public override string ToString() + { + return string.Format("{0}:{1}", Score, Label); + } + } + + /// + /// Class encoding the point on a 2D curve. + /// + public class CurvePoint + { + private float _x, _y; + + /// + /// Constructor. + /// + /// X coordinate + /// Y coordinate + public CurvePoint(float x, float y) + { + _x = x; + _y = y; + } + + /// + /// X coordinate + /// + public float X + { + get + { + return _x; + } + } + + /// + /// Y coordinate + /// + public float Y + { + get + { + return _y; + } + } + + /// + /// Creates a string representation of this point. + /// + /// string in the form (x, y) + public override string ToString() + { + return string.Format("({0}, {1})", _x, _y); + } + } + + /// + /// Class which evaluates an SVM model using several standard techniques. + /// + public class PerformanceEvaluator + { + private class ChangePoint + { + public ChangePoint(int tp, int fp, int tn, int fn) + { + TP = tp; + FP = fp; + TN = tn; + FN = fn; + } + + public int TP, FP, TN, FN; + + public override string ToString() + { + return string.Format("{0}:{1}:{2}:{3}", TP, FP, TN, FN); + } + } + + private List _prCurve; + private double _ap; + + private List _rocCurve; + private double _auc; + + private List _data; + private List _changes; + + /// + /// Constructor. + /// + /// A pre-computed ranked pair set + public PerformanceEvaluator(List set) + { + _data = set; + computeStatistics(); + } + + /// + /// Constructor. + /// + /// Model to evaluate + /// Problem to evaluate + /// Label to be evaluate for + public PerformanceEvaluator(Model model, Problem problem, double category) : this(model, problem, category, "tmp.results") { } + /// + /// Constructor. + /// + /// Model to evaluate + /// Problem to evaluate + /// Results file for output + /// Category to evaluate for + public PerformanceEvaluator(Model model, Problem problem, double category, string resultsFile) + { + Prediction.Predict(problem, resultsFile, model, true); + parseResultsFile(resultsFile, problem.Y, category); + + computeStatistics(); + } + + /// + /// Constructor. + /// + /// Results file + /// The correct labels of each data item + /// The category to evaluate for + public PerformanceEvaluator(string resultsFile, double[] correctLabels, double category) + { + parseResultsFile(resultsFile, correctLabels, category); + computeStatistics(); + } + + private void parseResultsFile(string resultsFile, double[] labels, double category) + { + StreamReader input = new StreamReader(resultsFile); + string[] parts = input.ReadLine().Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + int confidenceIndex = -1; + for (int i = 1; i < parts.Length; i++) + if (double.Parse(parts[i], CultureInfo.InvariantCulture) == category) + { + confidenceIndex = i; + break; + } + _data = new List(); + for (int i = 0; i < labels.Length; i++) + { + parts = input.ReadLine().Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + double confidence = double.Parse(parts[confidenceIndex], CultureInfo.InvariantCulture); + _data.Add(new RankPair(confidence, labels[i] == category ? 1 : 0)); + } + input.Close(); + } + + private void computeStatistics() + { + _data.Sort(); + + findChanges(); + computePR(); + computeRoC(); + } + + private void findChanges() + { + int tp, fp, tn, fn; + tp = fp = tn = fn = 0; + for (int i = 0; i < _data.Count; i++) + { + if (_data[i].Label == 1) + fn++; + else tn++; + } + _changes = new List(); + for (int i = 0; i < _data.Count; i++) + { + if (_data[i].Label == 1) + { + tp++; + fn--; + } + else + { + fp++; + tn--; + } + _changes.Add(new ChangePoint(tp, fp, tn, fn)); + } + } + + private float computePrecision(ChangePoint p) + { + return (float)p.TP / (p.TP + p.FP); + } + + private float computeRecall(ChangePoint p) + { + return (float)p.TP / (p.TP + p.FN); + } + + private void computePR() + { + _prCurve = new List(); + _prCurve.Add(new CurvePoint(0, 1)); + float precision = computePrecision(_changes[0]); + float recall = computeRecall(_changes[0]); + float precisionSum = 0; + if (_changes[0].TP > 0) + { + precisionSum += precision; + _prCurve.Add(new CurvePoint(recall, precision)); + } + for (int i = 1; i < _changes.Count; i++) + { + precision = computePrecision(_changes[i]); + recall = computeRecall(_changes[i]); + if (_changes[i].TP > _changes[i - 1].TP) + { + precisionSum += precision; + _prCurve.Add(new CurvePoint(recall, precision)); + } + } + _prCurve.Add(new CurvePoint(1, (float)(_changes[0].TP + _changes[0].FN) / (_changes[0].FP + _changes[0].TN))); + _ap = precisionSum / (_changes[0].FN + _changes[0].TP); + } + + /// + /// Writes the Precision-Recall curve to a tab-delimited file. + /// + /// Filename for output + public void WritePRCurve(string filename) + { + StreamWriter output = new StreamWriter(filename); + output.WriteLine(_ap); + for (int i = 0; i < _prCurve.Count; i++) + output.WriteLine("{0}\t{1}", _prCurve[i].X, _prCurve[i].Y); + output.Close(); + } + + /// + /// Writes the Receiver Operating Characteristic curve to a tab-delimited file. + /// + /// Filename for output + public void WriteROCCurve(string filename) + { + StreamWriter output = new StreamWriter(filename); + output.WriteLine(_auc); + for (int i = 0; i < _rocCurve.Count; i++) + output.WriteLine("{0}\t{1}", _rocCurve[i].X, _rocCurve[i].Y); + output.Close(); + } + + /// + /// Receiver Operating Characteristic curve + /// + public List ROCCurve + { + get + { + return _rocCurve; + } + } + + /// + /// Returns the area under the ROC Curve + /// + public double AuC + { + get + { + return _auc; + } + } + + /// + /// Precision-Recall curve + /// + public List PRCurve + { + get + { + return _prCurve; + } + } + + /// + /// The average precision + /// + public double AP + { + get + { + return _ap; + } + } + + private float computeTPR(ChangePoint cp) + { + return computeRecall(cp); + } + + private float computeFPR(ChangePoint cp) + { + return (float)cp.FP / (cp.FP + cp.TN); + } + + private void computeRoC() + { + _rocCurve = new List(); + _rocCurve.Add(new CurvePoint(0, 0)); + float tpr = computeTPR(_changes[0]); + float fpr = computeFPR(_changes[0]); + _rocCurve.Add(new CurvePoint(fpr, tpr)); + _auc = 0; + for (int i = 1; i < _changes.Count; i++) + { + float newTPR = computeTPR(_changes[i]); + float newFPR = computeFPR(_changes[i]); + if (_changes[i].TP > _changes[i - 1].TP) + { + _auc += tpr * (newFPR - fpr) + .5 * (newTPR - tpr) * (newFPR - fpr); + tpr = newTPR; + fpr = newFPR; + _rocCurve.Add(new CurvePoint(fpr, tpr)); + } + } + _rocCurve.Add(new CurvePoint(1, 1)); + _auc += tpr * (1 - fpr) + .5 * (1 - tpr) * (1 - fpr); + } + + } +} diff --git a/BotSharp.MachineLearning/SVM/PrecomputedKernel.cs b/BotSharp.MachineLearning/SVM/PrecomputedKernel.cs new file mode 100644 index 00000000..2d348d3e --- /dev/null +++ b/BotSharp.MachineLearning/SVM/PrecomputedKernel.cs @@ -0,0 +1,117 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Collections.Generic; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Class encapsulating a precomputed kernel, where each position indicates the similarity score for two items in the training data. + /// + [Serializable] + public class PrecomputedKernel + { + private float[,] _similarities; + private int _rows; + private int _columns; + + /// + /// Constructor. + /// + /// The similarity scores between all items in the training data + public PrecomputedKernel(float[,] similarities) + { + _similarities = similarities; + _rows = _similarities.GetLength(0); + _columns = _similarities.GetLength(1); + } + + /// + /// Constructor. + /// + /// Nodes for self-similarity analysis + /// Parameters to use when computing similarities + public PrecomputedKernel(List nodes, Parameter param) + { + _rows = nodes.Count; + _columns = _rows; + _similarities = new float[_rows, _columns]; + for (int r = 0; r < _rows; r++) + { + for (int c = 0; c < r; c++) + _similarities[r, c] = _similarities[c, r]; + _similarities[r, r] = 1; + for (int c = r + 1; c < _columns; c++) + _similarities[r, c] = (float)Kernel.KernelFunction(nodes[r], nodes[c], param); + } + } + + /// + /// Constructor. + /// + /// Nodes to use as the rows of the matrix + /// Nodes to use as the columns of the matrix + /// Parameters to use when compute similarities + public PrecomputedKernel(List rows, List columns, Parameter param) + { + _rows = rows.Count; + _columns = columns.Count; + _similarities = new float[_rows, _columns]; + for (int r = 0; r < _rows; r++) + for (int c = 0; c < _columns; c++) + _similarities[r, c] = (float)Kernel.KernelFunction(rows[r], columns[c], param); + } + + /// + /// Constructs a object using the labels provided. If a label is set to "0" that item is ignored. + /// + /// The labels for the row items + /// The labels for the column items + /// A object + public Problem Compute(double[] rowLabels, double[] columnLabels) + { + List X = new List(); + List Y = new List(); + int maxIndex = 0; + for (int i = 0; i < columnLabels.Length; i++) + if (columnLabels[i] != 0) + maxIndex++; + maxIndex++; + for (int r = 0; r < _rows; r++) + { + if (rowLabels[r] == 0) + continue; + List nodes = new List(); + nodes.Add(new Node(0, X.Count + 1)); + for (int c = 0; c < _columns; c++) + { + if (columnLabels[c] == 0) + continue; + double value = _similarities[r, c]; + nodes.Add(new Node(nodes.Count, value)); + } + X.Add(nodes.ToArray()); + Y.Add(rowLabels[r]); + } + return new Problem(X.Count, Y.ToArray(), X.ToArray(), maxIndex); + } + + } +} diff --git a/BotSharp.MachineLearning/SVM/Prediction.cs b/BotSharp.MachineLearning/SVM/Prediction.cs new file mode 100644 index 00000000..f6293246 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Prediction.cs @@ -0,0 +1,214 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Linq; +using System.IO; +using System.Diagnostics; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Class containing the routines to perform class membership prediction using a trained SVM. + /// + public static class Prediction + { + /// + /// Predicts the class memberships of all the vectors in the problem. + /// + /// The SVM Problem to solve + /// File for result output + /// The Model to use + /// Whether to output a distribution over the classes + /// Percentage correctly labelled + public static double Predict( + Problem problem, + string outputFile, + Model model, + bool predict_probability) + { + int correct = 0; + int total = 0; + double error = 0; + double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0; + StreamWriter output = outputFile != null ? new StreamWriter(outputFile) : null; + + SvmType svm_type = Procedures.svm_get_svm_type(model); + int nr_class = Procedures.svm_get_nr_class(model); + int[] labels = new int[nr_class]; + double[] prob_estimates = null; + + if (predict_probability) + { + if (svm_type == SvmType.EPSILON_SVR || svm_type == SvmType.NU_SVR) + { + Console.WriteLine("Prob. model for test data: target value = predicted value + z,\nz: Laplace distribution e^(-|z|/sigma)/(2sigma),sigma=" + Procedures.svm_get_svr_probability(model)); + } + else + { + Procedures.svm_get_labels(model, labels); + prob_estimates = new double[nr_class]; + if (output != null) + { + output.Write("labels"); + for (int j = 0; j < nr_class; j++) + { + output.Write(" " + labels[j]); + } + output.Write("\n"); + } + } + } + for (int i = 0; i < problem.Count; i++) + { + double target = problem.Y[i]; + Node[] x = problem.X[i]; + + double v; + if (predict_probability && (svm_type == SvmType.C_SVC || svm_type == SvmType.NU_SVC)) + { + v = Procedures.svm_predict_probability(model, x, prob_estimates); + if (output != null) + { + output.Write(v + " "); + for (int j = 0; j < nr_class; j++) + { + output.Write(prob_estimates[j] + " "); + } + output.Write("\n"); + } + } + else + { + v = Procedures.svm_predict(model, x); + if(output != null) + output.Write(v + "\n"); + } + + if (v == target) + ++correct; + error += (v - target) * (v - target); + sumv += v; + sumy += target; + sumvv += v * v; + sumyy += target * target; + sumvy += v * target; + ++total; + } + if(output != null) + output.Close(); + + if (model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) + return error / total; + else return (double)correct / total; + } + + /// + /// Predict the labels for all the data points in a problem. + /// + /// The model to use + /// The problem to solve + /// The predicted labels + public static double[] PredictLabels(this Model model, Problem problem) + { + return problem.X.Select(o => model.Predict(o)).ToArray(); + } + + /// + /// Predict the probability distributions over all labels for each data point in a problem. + /// + /// The model to use + /// The problem to solve + /// A distribution over labels for each data point + public static double[][] PredictLabelsProbability(this Model model, Problem problem) + { + return problem.X.Select(o => model.PredictProbability(o)).ToArray(); + } + + /// + /// Predict the class for a single input vector. + /// + /// The Model to use for prediction + /// The vector for which to predict class + /// The result + public static double Predict(this Model model, Node[] x) + { + return Procedures.svm_predict(model, x); + } + + /// + /// Predicts a class distribution for the single input vector. + /// + /// Model to use for prediction + /// The vector for which to predict the class distribution + /// A probability distribtion over classes + public static double[] PredictProbability(this Model model, Node[] x) + { + SvmType svm_type = Procedures.svm_get_svm_type(model); + if (svm_type != SvmType.C_SVC && svm_type != SvmType.NU_SVC) + throw new Exception("Model type " + svm_type + " unable to predict probabilities."); + int nr_class = Procedures.svm_get_nr_class(model); + double[] probEstimates = new double[nr_class]; + Procedures.svm_predict_probability(model, x, probEstimates); + return probEstimates; + } + + private static void exit_with_help() + { + Debug.Write("usage: svm_predict [options] test_file model_file output_file\n" + "options:\n" + "-b probability_estimates: whether to predict probability estimates, 0 or 1 (default 0); one-class SVM not supported yet\n"); + Environment.Exit(1); + } + + /// + /// Legacy method, provided to allow usage as though this were the command line version of libsvm. + /// + /// Standard arguments passed to the svm_predict exectutable. See libsvm documentation for details. + [Obsolete("Use the other version of Predict() instead")] + public static void Predict(params string[] args) + { + int i = 0; + bool predictProbability = false; + + // parse options + for (i = 0; i < args.Length; i++) + { + if (args[i][0] != '-') + break; + ++i; + switch (args[i - 1][1]) + { + + case 'b': + predictProbability = int.Parse(args[i]) == 1; + break; + + default: + throw new ArgumentException("Unknown option"); + + } + } + if (i >= args.Length) + throw new ArgumentException("No input, model and output files provided"); + + Problem problem = Problem.Read(args[i]); + Model model = Model.Read(args[i + 1]); + Predict(problem, args[i + 2], model, predictProbability); + } + } +} \ No newline at end of file diff --git a/BotSharp.MachineLearning/SVM/Problem.cs b/BotSharp.MachineLearning/SVM/Problem.cs new file mode 100644 index 00000000..4d1eb783 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Problem.cs @@ -0,0 +1,190 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.IO; +using System.Collections.Generic; +using System.Threading; +using System.Globalization; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Encapsulates a problem, or set of vectors which must be classified. + /// + [Serializable] + public class Problem + { + /// + /// Constructor. + /// + /// Number of vectors + /// The class labels + /// Vector data. + /// Maximum index for a vector + public Problem(int count, double[] y, Node[][] x, int maxIndex) + { + Count = count; + Y = y; + X = x; + MaxIndex = maxIndex; + } + /// + /// Empty Constructor. Nothing is initialized. + /// + public Problem() + { + } + /// + /// Number of vectors. + /// + public int Count { get; set; } + + /// + /// Class labels. + /// + public double[] Y { get; set; } + + /// + /// Vector data. + /// + public Node[][] X { get; set; } + + /// + /// Maximum index for a vector. + /// + public int MaxIndex { get; set; } + + public override bool Equals(object obj) + { + Problem other = obj as Problem; + if (other == null) + return false; + + return other.Count == Count && + other.MaxIndex == MaxIndex && + other.X.IsEqual(X) && + other.Y.IsEqual(Y); + } + + public override int GetHashCode() + { + return Count.GetHashCode() + + MaxIndex.GetHashCode() + + X.ComputeHashcode() + + Y.ComputeHashcode(); + } + + /// + /// Reads a problem from a stream. + /// + /// Stream to read from + /// The problem + public static Problem Read(Stream stream) + { + TemporaryCulture.Start(); + + StreamReader input = new StreamReader(stream); + List vy = new List(); + List vx = new List(); + int max_index = 0; + + while (input.Peek() > -1) + { + string[] parts = input.ReadLine().Trim().Split(); + + vy.Add(double.Parse(parts[0])); + int m = parts.Length - 1; + Node[] x = new Node[m]; + for (int j = 0; j < m; j++) + { + x[j] = new Node(); + string[] nodeParts = parts[j + 1].Split(':'); + x[j].Index = int.Parse(nodeParts[0]); + x[j].Value = double.Parse(nodeParts[1]); + } + if (m > 0) + max_index = Math.Max(max_index, x[m - 1].Index); + vx.Add(x); + } + + TemporaryCulture.Stop(); + + return new Problem(vy.Count, vy.ToArray(), vx.ToArray(), max_index); + } + + /// + /// Writes a problem to a stream. + /// + /// The stream to write the problem to. + /// The problem to write. + public static void Write(Stream stream, Problem problem) + { + TemporaryCulture.Start(); + + StreamWriter output = new StreamWriter(stream); + for (int i = 0; i < problem.Count; i++) + { + output.Write(problem.Y[i]); + for (int j = 0; j < problem.X[i].Length; j++) + output.Write(" {0}:{1:0.000000}", problem.X[i][j].Index, problem.X[i][j].Value); + output.Write("\n"); + } + output.Flush(); + + TemporaryCulture.Stop(); + } + + /// + /// Reads a Problem from a file. + /// + /// The file to read from. + /// the Probem + public static Problem Read(string filename) + { + FileStream input = File.OpenRead(filename); + try + { + return Read(input); + } + finally + { + input.Close(); + } + } + + /// + /// Writes a problem to a file. This will overwrite any previous data in the file. + /// + /// The file to write to + /// The problem to write + public static void Write(string filename, Problem problem) + { + FileStream output = File.Open(filename, FileMode.Create); + try + { + Write(output, problem); + } + finally + { + output.Close(); + } + } + } +} \ No newline at end of file diff --git a/BotSharp.MachineLearning/SVM/RangeTransform.cs b/BotSharp.MachineLearning/SVM/RangeTransform.cs new file mode 100644 index 00000000..5e3a2e80 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/RangeTransform.cs @@ -0,0 +1,245 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.IO; +using System.Threading; +using System.Globalization; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Class which encapsulates a range transformation. + /// + public class RangeTransform : IRangeTransform + { + /// + /// Default lower bound for scaling (-1). + /// + public const int DEFAULT_LOWER_BOUND = -1; + /// + /// Default upper bound for scaling (1). + /// + public const int DEFAULT_UPPER_BOUND = 1; + + /// + /// Determines the Range transform for the provided problem. Uses the default lower and upper bounds. + /// + /// The Problem to analyze + /// The Range transform for the problem + public static RangeTransform Compute(Problem prob) + { + return Compute(prob, DEFAULT_LOWER_BOUND, DEFAULT_UPPER_BOUND); + } + /// + /// Determines the Range transform for the provided problem. + /// + /// The Problem to analyze + /// The lower bound for scaling + /// The upper bound for scaling + /// The Range transform for the problem + public static RangeTransform Compute(Problem prob, double lowerBound, double upperBound) + { + double[] minVals = new double[prob.MaxIndex]; + double[] maxVals = new double[prob.MaxIndex]; + for (int i = 0; i < prob.MaxIndex; i++) + { + minVals[i] = double.MaxValue; + maxVals[i] = double.MinValue; + } + for (int i = 0; i < prob.Count; i++) + { + for (int j = 0; j < prob.X[i].Length; j++) + { + int index = prob.X[i][j].Index - 1; + double value = prob.X[i][j].Value; + minVals[index] = Math.Min(minVals[index], value); + maxVals[index] = Math.Max(maxVals[index], value); + } + } + for (int i = 0; i < prob.MaxIndex; i++) + { + if (minVals[i] == double.MaxValue || maxVals[i] == double.MinValue) + { + minVals[i] = 0; + maxVals[i] = 0; + } + } + return new RangeTransform(minVals, maxVals, lowerBound, upperBound); + } + + private double[] _inputStart; + private double[] _inputScale; + private double _outputStart; + private double _outputScale; + private int _length; + + /// + /// Constructor. + /// + /// The minimum values in each dimension. + /// The maximum values in each dimension. + /// The desired lower bound for all dimensions. + /// The desired upper bound for all dimensions. + public RangeTransform(double[] minValues, double[] maxValues, double lowerBound, double upperBound) + { + _length = minValues.Length; + if(maxValues.Length != _length) + throw new Exception("Number of max and min values must be equal."); + _inputStart = new double[_length]; + _inputScale = new double[_length]; + for (int i = 0; i < _length; i++) + { + _inputStart[i] = minValues[i]; + _inputScale[i] = maxValues[i] - minValues[i]; + } + _outputStart = lowerBound; + _outputScale = upperBound - lowerBound; + } + private RangeTransform(double[] inputStart, double[] inputScale, double outputStart, double outputScale, int length) + { + _inputStart = inputStart; + _inputScale = inputScale; + _outputStart = outputStart; + _outputScale = outputScale; + _length = length; + } + /// + /// Transforms the input array based upon the values provided. + /// + /// The input array + /// A scaled array + public Node[] Transform(Node[] input) + { + Node[] output = new Node[input.Length]; + for (int i = 0; i < output.Length; i++) + { + int index = input[i].Index; + double value = input[i].Value; + output[i] = new Node(index, Transform(value, index)); + } + return output; + } + + /// + /// Transforms this an input value using the scaling transform for the provided dimension. + /// + /// The input value to transform + /// The dimension whose scaling transform should be used + /// The scaled value + public double Transform(double input, int index) + { + index--; + double tmp = input - _inputStart[index]; + if (_inputScale[index] == 0) + return 0; + tmp /= _inputScale[index]; + tmp *= _outputScale; + return tmp + _outputStart; + } + /// + /// Writes this Range transform to a stream. + /// + /// The stream to write to + /// The range to write + public static void Write(Stream stream, RangeTransform r) + { + TemporaryCulture.Start(); + + StreamWriter output = new StreamWriter(stream); + output.WriteLine(r._length); + output.Write(r._inputStart[0]); + for(int i=1; i + /// Writes this Range transform to a file. This will overwrite any previous data in the file. + /// + /// The file to write to + /// The Range to write + public static void Write(string outputFile, RangeTransform r) + { + FileStream s = File.Open(outputFile, FileMode.Create); + try + { + Write(s, r); + } + finally + { + s.Close(); + } + } + + /// + /// Reads a Range transform from a file. + /// + /// The file to read from + /// The Range transform + public static RangeTransform Read(string inputFile) + { + FileStream s = File.OpenRead(inputFile); + try + { + return Read(s); + } + finally + { + s.Close(); + } + } + + /// + /// Reads a Range transform from a stream. + /// + /// The stream to read from + /// The Range transform + public static RangeTransform Read(Stream stream) + { + TemporaryCulture.Start(); + + StreamReader input = new StreamReader(stream); + int length = int.Parse(input.ReadLine()); + double[] inputStart = new double[length]; + double[] inputScale = new double[length]; + string[] parts = input.ReadLine().Split(); + for (int i = 0; i < length; i++) + inputStart[i] = double.Parse(parts[i]); + parts = input.ReadLine().Split(); + for (int i = 0; i < length; i++) + inputScale[i] = double.Parse(parts[i]); + parts = input.ReadLine().Split(); + double outputStart = double.Parse(parts[0]); + double outputScale = double.Parse(parts[1]); + + TemporaryCulture.Stop(); + + return new RangeTransform(inputStart, inputScale, outputStart, outputScale, length); + } + } +} diff --git a/BotSharp.MachineLearning/SVM/SVMExtensions.cs b/BotSharp.MachineLearning/SVM/SVMExtensions.cs new file mode 100644 index 00000000..fddce8bb --- /dev/null +++ b/BotSharp.MachineLearning/SVM/SVMExtensions.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace SVM.BotSharp.MachineLearning +{ + internal static class SVMExtensions + { + private const double PRECISION = 1000000.0; + + public static double Truncate(this double x) + { + return Math.Round(x * PRECISION) / PRECISION; + } + + public static void SwapIndex(this T[] list, int i, int j) + { + T tmp = list[i]; + list[i] = list[j]; + list[j] = tmp; + } + + public static bool IsEqual(this T[][] lhs, T[][] rhs) + { + if (lhs.Length != rhs.Length) + return false; + + for (int i = 0; i < lhs.Length; i++) + { + if (!lhs[i].IsEqual(rhs[i])) + return false; + } + + return true; + } + + public static bool IsEqual(this T[] lhs, T[] rhs) + { + if (lhs.Length != rhs.Length) + return false; + for (int i = 0; i < lhs.Length; i++) + if (!lhs[i].Equals(rhs[i])) + return false; + return true; + } + + public static bool IsEqual(this double[] lhs, double[] rhs) + { + if (lhs.Length != rhs.Length) + return false; + for (int i = 0; i < lhs.Length; i++) + { + double x = lhs[i].Truncate(); + double y = rhs[i].Truncate(); + if (x != y) + return false; + } + return true; + } + + public static bool IsEqual(this double[][] lhs, double[][] rhs) + { + if (lhs.Length != rhs.Length) + return false; + for (int i = 0; i < lhs.Length; i++) + if (!lhs[i].IsEqual(rhs[i])) + return false; + return true; + } + + public static int ComputeHashcode(this T[] array) + { + return array.Sum(o => o.GetHashCode()); + } + + public static int ComputeHashcode(this T[][] array) + { + return array.Sum(o => o.ComputeHashcode()); + } + } +} diff --git a/BotSharp.MachineLearning/SVM/SVMUtilities.cs b/BotSharp.MachineLearning/SVM/SVMUtilities.cs new file mode 100644 index 00000000..57e62674 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/SVMUtilities.cs @@ -0,0 +1,116 @@ +using SVM; +using SVM.BotSharp.MachineLearning; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SVM.BotSharp.MachineLearningTest +{ + public static class SVMUtilities + { + private const double SCALE = 100; + public const int TRAINING_SEED = 20080524; + public const int TESTING_SEED = 20140407; + + public static Problem CreateTwoClassProblem(int count, bool isTraining = true) + { + Problem prob = new Problem(); + prob.Count = count; + prob.MaxIndex = 2; + + Random rand = new Random(isTraining ? TRAINING_SEED : TESTING_SEED); + // create points on either side of the vertical axis + int positive = count / 2; + List labels = new List(); + List data = new List(); + for (int i = 0; i < count; i++) + { + double x = rand.NextDouble() * SCALE + 10; + double y = rand.NextDouble() * SCALE - (SCALE * .5); + x = i < positive ? x : -x; + data.Add(new Node[] { new Node(1, x), new Node(2, y) }); + labels.Add(i < positive ? 1 : -1); + } + prob.X = data.ToArray(); + prob.Y = labels.ToArray(); + + return prob; + } + + public static Problem CreateMulticlassProblem(int numberOfClasses, int count, bool isTraining = true) + { + if (numberOfClasses > 8) + throw new ArgumentException("Number of classes must be < 8"); + + Problem prob = new Problem(); + prob.Count = count; + prob.MaxIndex = 3; + + int[] samplesPerClass = new int[numberOfClasses]; + double countPerClass = (double)count / numberOfClasses; + double current = countPerClass; + for (int i = 1; i < samplesPerClass.Length; i++) + { + samplesPerClass[i] = (int)current; + current += countPerClass; + samplesPerClass[i - 1] = samplesPerClass[i] - samplesPerClass[i - 1]; + } + samplesPerClass[samplesPerClass.Length - 1] = count - samplesPerClass.Last(); + + int[] xSigns = new int[8] { -1, 1, 1, -1, -1, 1, 1, -1 }; + int[] ySigns = new int[8] { 1, 1, -1, -1, 1, 1, -1, -1 }; + int[] zSigns = new int[8] { 1, 1, 1, 1, -1, -1, -1, -1 }; + + Random rand = new Random(isTraining ? TRAINING_SEED : TESTING_SEED); + + List labels = new List(); + List data = new List(); + for (int i = 0; i < numberOfClasses; i++) + { + for (int j = 0; j < samplesPerClass[i]; j++) + { + double x = rand.NextDouble() * SCALE + 10; + double y = rand.NextDouble() * SCALE + 10; + double z = rand.NextDouble() * SCALE + 10; + x *= xSigns[i]; + y *= ySigns[i]; + z *= zSigns[i]; + + data.Add(new Node[] { new Node(1, x), new Node(2, y), new Node(3, z) }); + labels.Add(i); + } + } + + prob.X = data.ToArray(); + prob.Y = labels.ToArray(); + + return prob; + } + + public static Problem CreateRegressionProblem(int count, bool isTraining = true) + { + Problem prob = new Problem(); + prob.Count = count; + prob.MaxIndex = 2; + + Random rand = new Random(isTraining ? TRAINING_SEED : TESTING_SEED); + + List labels = new List(); + List data = new List(); + for (int i = 0; i < count; i++) + { + double y = rand.NextDouble() * 10 - 5; + double z = rand.NextDouble() * 10 - 5; + double x = 2 * y + z; + data.Add(new Node[] { new Node(1, y), new Node(2, z) }); + labels.Add(x); + } + prob.X = data.ToArray(); + prob.Y = labels.ToArray(); + + return prob; + } + } +} diff --git a/BotSharp.MachineLearning/SVM/Scaling.cs b/BotSharp.MachineLearning/SVM/Scaling.cs new file mode 100644 index 00000000..b0922cb7 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Scaling.cs @@ -0,0 +1,49 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Deals with the scaling of Problems so they have uniform ranges across all dimensions in order to + /// result in better SVM performance. + /// + public static class Scaling + { + /// + /// Scales a problem using the provided range. This will not affect the parameter. + /// + /// The problem to scale + /// The Range transform to use in scaling + /// The Scaled problem + public static Problem Scale(this IRangeTransform range, Problem prob) + { + Problem scaledProblem = new Problem(prob.Count, new double[prob.Count], new Node[prob.Count][], prob.MaxIndex); + for (int i = 0; i < scaledProblem.Count; i++) + { + scaledProblem.X[i] = new Node[prob.X[i].Length]; + for (int j = 0; j < scaledProblem.X[i].Length; j++) + scaledProblem.X[i][j] = new Node(prob.X[i][j].Index, range.Transform(prob.X[i][j].Value, prob.X[i][j].Index)); + scaledProblem.Y[i] = prob.Y[i]; + } + return scaledProblem; + } + } +} diff --git a/BotSharp.MachineLearning/SVM/Solver.cs b/BotSharp.MachineLearning/SVM/Solver.cs new file mode 100644 index 00000000..5faada66 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Solver.cs @@ -0,0 +1,2298 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Linq; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace SVM.BotSharp.MachineLearning +{ + + + // An SMO algorithm in Fan et al., JMLR 6(2005), p. 1889--1918 + // Solves: + // + // min 0.5(\alpha^T Q \alpha) + p^T \alpha + // + // y^T \alpha = \delta + // y_i = +1 or -1 + // 0 <= alpha_i <= Cp for y_i = 1 + // 0 <= alpha_i <= Cn for y_i = -1 + // + // Given: + // + // Q, p, y, Cp, Cn, and an initial feasible point \alpha + // l is the size of vectors and matrices + // eps is the stopping tolerance + // + // solution will be put in \alpha, objective value will be put in obj + // + class Solver + { + protected int active_size; + protected sbyte[] y; + protected double[] G; // gradient of objective function + protected const byte LOWER_BOUND = 0; + protected const byte UPPER_BOUND = 1; + protected const byte FREE = 2; + protected byte[] alpha_status; // LOWER_BOUND, UPPER_BOUND, FREE + protected double[] alpha; + protected IQMatrix Q; + protected double[] QD; + protected double eps; + protected double Cp, Cn; + protected double[] p; + protected int[] active_set; + protected double[] G_bar; // gradient, if we treat free variables as 0 + protected int l; + protected bool unshrink; // XXX + + protected const double INF = Double.PositiveInfinity; + + double get_C(int i) + { + return (y[i] > 0) ? Cp : Cn; + } + void update_alpha_status(int i) + { + if (alpha[i] >= get_C(i)) + alpha_status[i] = UPPER_BOUND; + else if (alpha[i] <= 0) + alpha_status[i] = LOWER_BOUND; + else alpha_status[i] = FREE; + } + protected bool is_upper_bound(int i) { return alpha_status[i] == UPPER_BOUND; } + protected bool is_lower_bound(int i) { return alpha_status[i] == LOWER_BOUND; } + protected bool is_free(int i) { return alpha_status[i] == FREE; } + + // java: information about solution except alpha, + // because we cannot return multiple values otherwise... + public class SolutionInfo + { + public double obj { get; set; } + public double rho { get; set; } + public double upper_bound_p { get; set; } + public double upper_bound_n { get; set; } + public double r { get; set; } // for Solver_NU + } + + protected void swap_index(int i, int j) + { + Q.SwapIndex(i, j); + do { sbyte _ = y[i]; y[i] = y[j]; y[j] = _; } while (false); + do { double _ = G[i]; G[i] = G[j]; G[j] = _; } while (false); + do { byte _ = alpha_status[i]; alpha_status[i] = alpha_status[j]; alpha_status[j] = _; } while (false); + do { double _ = alpha[i]; alpha[i] = alpha[j]; alpha[j] = _; } while (false); + do { double _ = p[i]; p[i] = p[j]; p[j] = _; } while (false); + do { int _ = active_set[i]; active_set[i] = active_set[j]; active_set[j] = _; } while (false); + do { double _ = G_bar[i]; G_bar[i] = G_bar[j]; G_bar[j] = _; } while (false); + } + + protected void reconstruct_gradient() + { + // reconstruct inactive elements of G from G_bar and free variables + + if (active_size == l) return; + + int i, j; + int nr_free = 0; + + for (j = active_size; j < l; j++) + G[j] = G_bar[j] + p[j]; + + for (j = 0; j < active_size; j++) + if (is_free(j)) + nr_free++; + + if (2 * nr_free < active_size) + Procedures.info("\nWARNING: using -h 0 may be faster\n"); + + if (nr_free * l > 2 * active_size * (l - active_size)) + { + for (i = active_size; i < l; i++) + { + float[] Q_i = Q.GetQ(i, active_size); + for (j = 0; j < active_size; j++) + if (is_free(j)) + G[i] += alpha[j] * Q_i[j]; + } + } + else + { + for (i = 0; i < active_size; i++) + if (is_free(i)) + { + float[] Q_i = Q.GetQ(i, l); + double alpha_i = alpha[i]; + for (j = active_size; j < l; j++) + G[j] += alpha_i * Q_i[j]; + } + } + } + + public virtual void Solve(int l, IQMatrix Q, double[] p_, sbyte[] y_, + double[] alpha_, double Cp, double Cn, double eps, SolutionInfo si, bool shrinking) + { + this.l = l; + this.Q = Q; + QD = Q.GetQD(); + p = (double[])p_.Clone(); + y = (sbyte[])y_.Clone(); + alpha = (double[])alpha_.Clone(); + this.Cp = Cp; + this.Cn = Cn; + this.eps = eps; + this.unshrink = false; + + // initialize alpha_status + { + alpha_status = new byte[l]; + for (int i = 0; i < l; i++) + update_alpha_status(i); + } + + // initialize active set (for shrinking) + { + active_set = new int[l]; + for (int i = 0; i < l; i++) + active_set[i] = i; + active_size = l; + } + + // initialize gradient + { + G = new double[l]; + G_bar = new double[l]; + int i; + for (i = 0; i < l; i++) + { + G[i] = p[i]; + G_bar[i] = 0; + } + for (i = 0; i < l; i++) + if (!is_lower_bound(i)) + { + float[] Q_i = Q.GetQ(i, l); + double alpha_i = alpha[i]; + int j; + for (j = 0; j < l; j++) + G[j] += alpha_i * Q_i[j]; + if (is_upper_bound(i)) + for (j = 0; j < l; j++) + G_bar[j] += get_C(i) * Q_i[j]; + } + } + + // optimization step + + int iter = 0; + int max_iter = Math.Max(10000000, l > int.MaxValue / 100 ? int.MaxValue : 100 * l); + int counter = Math.Min(l, 1000) + 1; + int[] working_set = new int[2]; + + while (iter < max_iter) + { + // show progress and do shrinking + + if (--counter == 0) + { + counter = Math.Min(l, 1000); + if (shrinking) do_shrinking(); + Procedures.info("."); + } + + if (select_working_set(working_set) != 0) + { + // reconstruct the whole gradient + reconstruct_gradient(); + // reset active set size and check + active_size = l; + Procedures.info("*"); + if (select_working_set(working_set) != 0) + break; + else + counter = 1; // do shrinking next iteration + } + + int i = working_set[0]; + int j = working_set[1]; + + ++iter; + + // update alpha[i] and alpha[j], handle bounds carefully + + float[] Q_i = Q.GetQ(i, active_size); + float[] Q_j = Q.GetQ(j, active_size); + + double C_i = get_C(i); + double C_j = get_C(j); + + double old_alpha_i = alpha[i]; + double old_alpha_j = alpha[j]; + + if (y[i] != y[j]) + { + double quad_coef = QD[i] + QD[j] + 2 * Q_i[j]; + if (quad_coef <= 0) + quad_coef = 1e-12; + double delta = (-G[i] - G[j]) / quad_coef; + double diff = alpha[i] - alpha[j]; + alpha[i] += delta; + alpha[j] += delta; + + if (diff > 0) + { + if (alpha[j] < 0) + { + alpha[j] = 0; + alpha[i] = diff; + } + } + else + { + if (alpha[i] < 0) + { + alpha[i] = 0; + alpha[j] = -diff; + } + } + if (diff > C_i - C_j) + { + if (alpha[i] > C_i) + { + alpha[i] = C_i; + alpha[j] = C_i - diff; + } + } + else + { + if (alpha[j] > C_j) + { + alpha[j] = C_j; + alpha[i] = C_j + diff; + } + } + } + else + { + double quad_coef = QD[i] + QD[j] - 2 * Q_i[j]; + if (quad_coef <= 0) + quad_coef = 1e-12; + double delta = (G[i] - G[j]) / quad_coef; + double sum = alpha[i] + alpha[j]; + alpha[i] -= delta; + alpha[j] += delta; + + if (sum > C_i) + { + if (alpha[i] > C_i) + { + alpha[i] = C_i; + alpha[j] = sum - C_i; + } + } + else + { + if (alpha[j] < 0) + { + alpha[j] = 0; + alpha[i] = sum; + } + } + if (sum > C_j) + { + if (alpha[j] > C_j) + { + alpha[j] = C_j; + alpha[i] = sum - C_j; + } + } + else + { + if (alpha[i] < 0) + { + alpha[i] = 0; + alpha[j] = sum; + } + } + } + + // update G + + double delta_alpha_i = alpha[i] - old_alpha_i; + double delta_alpha_j = alpha[j] - old_alpha_j; + + for (int k = 0; k < active_size; k++) + { + G[k] += Q_i[k] * delta_alpha_i + Q_j[k] * delta_alpha_j; + } + + // update alpha_status and G_bar + + { + bool ui = is_upper_bound(i); + bool uj = is_upper_bound(j); + update_alpha_status(i); + update_alpha_status(j); + int k; + if (ui != is_upper_bound(i)) + { + Q_i = Q.GetQ(i, l); + if (ui) + for (k = 0; k < l; k++) + G_bar[k] -= C_i * Q_i[k]; + else + for (k = 0; k < l; k++) + G_bar[k] += C_i * Q_i[k]; + } + + if (uj != is_upper_bound(j)) + { + Q_j = Q.GetQ(j, l); + if (uj) + for (k = 0; k < l; k++) + G_bar[k] -= C_j * Q_j[k]; + else + for (k = 0; k < l; k++) + G_bar[k] += C_j * Q_j[k]; + } + } + + } + + if (iter >= max_iter) + { + if (active_size < l) + { + // reconstruct the whole gradient to calculate objective value + reconstruct_gradient(); + active_size = l; + Procedures.info("*"); + } + Console.Error.Write("\nWARNING: reaching max number of iterations\n"); + } + + // calculate rho + + si.rho = calculate_rho(); + + // calculate objective value + { + double v = 0; + int i; + for (i = 0; i < l; i++) + v += alpha[i] * (G[i] + p[i]); + + si.obj = v / 2; + } + + // put back the solution + { + for (int i = 0; i < l; i++) + alpha_[active_set[i]] = alpha[i]; + } + + si.upper_bound_p = Cp; + si.upper_bound_n = Cn; + + Procedures.info("\noptimization finished, #iter = " + iter + "\n"); + } + + // return 1 if already optimal, return 0 otherwise + protected virtual int select_working_set(int[] working_set) + { + // return i,j such that + // i: maximizes -y_i * grad(f)_i, i in I_up(\alpha) + // j: mimimizes the decrease of obj value + // (if quadratic coefficeint <= 0, replace it with tau) + // -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha) + + double Gmax = -INF; + double Gmax2 = -INF; + int Gmax_idx = -1; + int Gmin_idx = -1; + double obj_diff_min = INF; + + for (int t = 0; t < active_size; t++) + if (y[t] == +1) + { + if (!is_upper_bound(t)) + if (-G[t] >= Gmax) + { + Gmax = -G[t]; + Gmax_idx = t; + } + } + else + { + if (!is_lower_bound(t)) + if (G[t] >= Gmax) + { + Gmax = G[t]; + Gmax_idx = t; + } + } + + int i = Gmax_idx; + float[] Q_i = null; + if (i != -1) // null Q_i not accessed: Gmax=-INF if i=-1 + Q_i = Q.GetQ(i, active_size); + + for (int j = 0; j < active_size; j++) + { + if (y[j] == +1) + { + if (!is_lower_bound(j)) + { + double grad_diff = Gmax + G[j]; + if (G[j] >= Gmax2) + Gmax2 = G[j]; + if (grad_diff > 0) + { + double obj_diff; + double quad_coef = QD[i] + QD[j] - 2.0 * y[i] * Q_i[j]; + if (quad_coef > 0) + obj_diff = -(grad_diff * grad_diff) / quad_coef; + else + obj_diff = -(grad_diff * grad_diff) / 1e-12; + + if (obj_diff <= obj_diff_min) + { + Gmin_idx = j; + obj_diff_min = obj_diff; + } + } + } + } + else + { + if (!is_upper_bound(j)) + { + double grad_diff = Gmax - G[j]; + if (-G[j] >= Gmax2) + Gmax2 = -G[j]; + if (grad_diff > 0) + { + double obj_diff; + double quad_coef = QD[i] + QD[j] + 2.0 * y[i] * Q_i[j]; + if (quad_coef > 0) + obj_diff = -(grad_diff * grad_diff) / quad_coef; + else + obj_diff = -(grad_diff * grad_diff) / 1e-12; + + if (obj_diff <= obj_diff_min) + { + Gmin_idx = j; + obj_diff_min = obj_diff; + } + } + } + } + } + + if (Gmax + Gmax2 < eps) + return 1; + + working_set[0] = Gmax_idx; + working_set[1] = Gmin_idx; + return 0; + } + + private bool be_shrunk(int i, double Gmax1, double Gmax2) + { + if (is_upper_bound(i)) + { + if (y[i] == +1) + return (-G[i] > Gmax1); + else + return (-G[i] > Gmax2); + } + else if (is_lower_bound(i)) + { + if (y[i] == +1) + return (G[i] > Gmax2); + else + return (G[i] > Gmax1); + } + else + return (false); + } + + protected virtual void do_shrinking() + { + int i; + double Gmax1 = -INF; // max { -y_i * grad(f)_i | i in I_up(\alpha) } + double Gmax2 = -INF; // max { y_i * grad(f)_i | i in I_low(\alpha) } + + // find maximal violating pair first + for (i = 0; i < active_size; i++) + { + if (y[i] == +1) + { + if (!is_upper_bound(i)) + { + if (-G[i] >= Gmax1) + Gmax1 = -G[i]; + } + if (!is_lower_bound(i)) + { + if (G[i] >= Gmax2) + Gmax2 = G[i]; + } + } + else + { + if (!is_upper_bound(i)) + { + if (-G[i] >= Gmax2) + Gmax2 = -G[i]; + } + if (!is_lower_bound(i)) + { + if (G[i] >= Gmax1) + Gmax1 = G[i]; + } + } + } + + if (unshrink == false && Gmax1 + Gmax2 <= eps * 10) + { + unshrink = true; + reconstruct_gradient(); + active_size = l; + } + + for (i = 0; i < active_size; i++) + if (be_shrunk(i, Gmax1, Gmax2)) + { + active_size--; + while (active_size > i) + { + if (!be_shrunk(active_size, Gmax1, Gmax2)) + { + swap_index(i, active_size); + break; + } + active_size--; + } + } + } + + protected virtual double calculate_rho() + { + double r; + int nr_free = 0; + double ub = INF, lb = -INF, sum_free = 0; + for (int i = 0; i < active_size; i++) + { + double yG = y[i] * G[i]; + + if (is_lower_bound(i)) + { + if (y[i] > 0) + ub = Math.Min(ub, yG); + else + lb = Math.Max(lb, yG); + } + else if (is_upper_bound(i)) + { + if (y[i] < 0) + ub = Math.Min(ub, yG); + else + lb = Math.Max(lb, yG); + } + else + { + ++nr_free; + sum_free += yG; + } + } + + if (nr_free > 0) + r = sum_free / nr_free; + else + r = (ub + lb) / 2; + + return r; + } + + } + + // + // Solver for nu-svm classification and regression + // + // additional constraint: e^T \alpha = constant + // + sealed class Solver_NU : Solver + { + private SolutionInfo si; + + public override void Solve(int l, IQMatrix Q, double[] p, sbyte[] y, + double[] alpha, double Cp, double Cn, double eps, + SolutionInfo si, bool shrinking) + { + this.si = si; + base.Solve(l, Q, p, y, alpha, Cp, Cn, eps, si, shrinking); + } + + // return 1 if already optimal, return 0 otherwise + protected override int select_working_set(int[] working_set) + { + // return i,j such that y_i = y_j and + // i: maximizes -y_i * grad(f)_i, i in I_up(\alpha) + // j: minimizes the decrease of obj value + // (if quadratic coefficeint <= 0, replace it with tau) + // -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha) + + double Gmaxp = -INF; + double Gmaxp2 = -INF; + int Gmaxp_idx = -1; + + double Gmaxn = -INF; + double Gmaxn2 = -INF; + int Gmaxn_idx = -1; + + int Gmin_idx = -1; + double obj_diff_min = INF; + + for (int t = 0; t < active_size; t++) + if (y[t] == +1) + { + if (!is_upper_bound(t)) + if (-G[t] >= Gmaxp) + { + Gmaxp = -G[t]; + Gmaxp_idx = t; + } + } + else + { + if (!is_lower_bound(t)) + if (G[t] >= Gmaxn) + { + Gmaxn = G[t]; + Gmaxn_idx = t; + } + } + + int ip = Gmaxp_idx; + int iN = Gmaxn_idx; + float[] Q_ip = null; + float[] Q_in = null; + if (ip != -1) // null Q_ip not accessed: Gmaxp=-INF if ip=-1 + Q_ip = Q.GetQ(ip, active_size); + if (iN != -1) + Q_in = Q.GetQ(iN, active_size); + + for (int j = 0; j < active_size; j++) + { + if (y[j] == +1) + { + if (!is_lower_bound(j)) + { + double grad_diff = Gmaxp + G[j]; + if (G[j] >= Gmaxp2) + Gmaxp2 = G[j]; + if (grad_diff > 0) + { + double obj_diff; + double quad_coef = QD[ip] + QD[j] - 2 * Q_ip[j]; + if (quad_coef > 0) + obj_diff = -(grad_diff * grad_diff) / quad_coef; + else + obj_diff = -(grad_diff * grad_diff) / 1e-12; + + if (obj_diff <= obj_diff_min) + { + Gmin_idx = j; + obj_diff_min = obj_diff; + } + } + } + } + else + { + if (!is_upper_bound(j)) + { + double grad_diff = Gmaxn - G[j]; + if (-G[j] >= Gmaxn2) + Gmaxn2 = -G[j]; + if (grad_diff > 0) + { + double obj_diff; + double quad_coef = QD[iN] + QD[j] - 2 * Q_in[j]; + if (quad_coef > 0) + obj_diff = -(grad_diff * grad_diff) / quad_coef; + else + obj_diff = -(grad_diff * grad_diff) / 1e-12; + + if (obj_diff <= obj_diff_min) + { + Gmin_idx = j; + obj_diff_min = obj_diff; + } + } + } + } + } + + if (Math.Max(Gmaxp + Gmaxp2, Gmaxn + Gmaxn2) < eps) + return 1; + + if (y[Gmin_idx] == +1) + working_set[0] = Gmaxp_idx; + else + working_set[0] = Gmaxn_idx; + working_set[1] = Gmin_idx; + + return 0; + } + + private bool be_shrunk(int i, double Gmax1, double Gmax2, double Gmax3, double Gmax4) + { + if (is_upper_bound(i)) + { + if (y[i] == +1) + return (-G[i] > Gmax1); + else + return (-G[i] > Gmax4); + } + else if (is_lower_bound(i)) + { + if (y[i] == +1) + return (G[i] > Gmax2); + else + return (G[i] > Gmax3); + } + else + return (false); + } + + protected override void do_shrinking() + { + double Gmax1 = -INF; // max { -y_i * grad(f)_i | y_i = +1, i in I_up(\alpha) } + double Gmax2 = -INF; // max { y_i * grad(f)_i | y_i = +1, i in I_low(\alpha) } + double Gmax3 = -INF; // max { -y_i * grad(f)_i | y_i = -1, i in I_up(\alpha) } + double Gmax4 = -INF; // max { y_i * grad(f)_i | y_i = -1, i in I_low(\alpha) } + + // find maximal violating pair first + int i; + for (i = 0; i < active_size; i++) + { + if (!is_upper_bound(i)) + { + if (y[i] == +1) + { + if (-G[i] > Gmax1) Gmax1 = -G[i]; + } + else if (-G[i] > Gmax4) Gmax4 = -G[i]; + } + if (!is_lower_bound(i)) + { + if (y[i] == +1) + { + if (G[i] > Gmax2) Gmax2 = G[i]; + } + else if (G[i] > Gmax3) Gmax3 = G[i]; + } + } + + if (unshrink == false && Math.Max(Gmax1 + Gmax2, Gmax3 + Gmax4) <= eps * 10) + { + unshrink = true; + reconstruct_gradient(); + active_size = l; + } + + for (i = 0; i < active_size; i++) + if (be_shrunk(i, Gmax1, Gmax2, Gmax3, Gmax4)) + { + active_size--; + while (active_size > i) + { + if (!be_shrunk(active_size, Gmax1, Gmax2, Gmax3, Gmax4)) + { + swap_index(i, active_size); + break; + } + active_size--; + } + } + } + + protected override double calculate_rho() + { + int nr_free1 = 0, nr_free2 = 0; + double ub1 = INF, ub2 = INF; + double lb1 = -INF, lb2 = -INF; + double sum_free1 = 0, sum_free2 = 0; + + for (int i = 0; i < active_size; i++) + { + if (y[i] == +1) + { + if (is_lower_bound(i)) + ub1 = Math.Min(ub1, G[i]); + else if (is_upper_bound(i)) + lb1 = Math.Max(lb1, G[i]); + else + { + ++nr_free1; + sum_free1 += G[i]; + } + } + else + { + if (is_lower_bound(i)) + ub2 = Math.Min(ub2, G[i]); + else if (is_upper_bound(i)) + lb2 = Math.Max(lb2, G[i]); + else + { + ++nr_free2; + sum_free2 += G[i]; + } + } + } + + double r1, r2; + if (nr_free1 > 0) + r1 = sum_free1 / nr_free1; + else + r1 = (ub1 + lb1) / 2; + + if (nr_free2 > 0) + r2 = sum_free2 / nr_free2; + else + r2 = (ub2 + lb2) / 2; + + si.r = (r1 + r2) / 2; + return (r1 - r2) / 2; + } + } + + // + // Q matrices for various formulations + // + class SVC_Q : Kernel + { + private readonly sbyte[] y; + private readonly Cache cache; + private readonly double[] QD; + + public SVC_Q(Problem prob, Parameter param, sbyte[] y_) + : base(prob.Count, prob.X, param) + { + y = (sbyte[])y_.Clone(); + cache = new Cache(prob.Count, (long)(param.CacheSize * (1 << 20))); + QD = new double[prob.Count]; + for (int i = 0; i < prob.Count; i++) + QD[i] = KernelFunction(i, i); + } + + public override float[] GetQ(int i, int len) + { + float[] data; + int start, j; + if ((start = cache.GetData(i, out data, len)) < len) + { + for (j = start; j < len; j++) + data[j] = (float)(y[i] * y[j] * KernelFunction(i, j)); + } + return data; + } + + public override double[] GetQD() + { + return QD; + } + + public override void SwapIndex(int i, int j) + { + cache.SwapIndex(i, j); + base.SwapIndex(i, j); + do { sbyte _ = y[i]; y[i] = y[j]; y[j] = _; } while (false); + do { double _ = QD[i]; QD[i] = QD[j]; QD[j] = _; } while (false); + } + } + + class ONE_CLASS_Q : Kernel + { + private readonly Cache cache; + private readonly double[] QD; + + public ONE_CLASS_Q(Problem prob, Parameter param) + : base(prob.Count, prob.X, param) + { + cache = new Cache(prob.Count, (long)(param.CacheSize * (1 << 20))); + QD = new double[prob.Count]; + for (int i = 0; i < prob.Count; i++) + QD[i] = KernelFunction(i, i); + } + + public override float[] GetQ(int i, int len) + { + float[] data; + int start, j; + if ((start = cache.GetData(i, out data, len)) < len) + { + for (j = start; j < len; j++) + data[j] = (float)KernelFunction(i, j); + } + return data; + } + + public override double[] GetQD() + { + return QD; + } + + public override void SwapIndex(int i, int j) + { + cache.SwapIndex(i, j); + base.SwapIndex(i, j); + do { double _ = QD[i]; QD[i] = QD[j]; QD[j] = _; } while (false); + } + } + + class SVR_Q : Kernel + { + private readonly int l; + private readonly Cache cache; + private readonly sbyte[] sign; + private readonly int[] index; + private int next_buffer; + private float[][] buffer; + private readonly double[] QD; + + public SVR_Q(Problem prob, Parameter param) + : base(prob.Count, prob.X, param) + { + l = prob.Count; + cache = new Cache(l, (long)(param.CacheSize * (1 << 20))); + QD = new double[2 * l]; + sign = new sbyte[2 * l]; + index = new int[2 * l]; + for (int k = 0; k < l; k++) + { + sign[k] = 1; + sign[k + l] = -1; + index[k] = k; + index[k + l] = k; + QD[k] = KernelFunction(k, k); + QD[k + l] = QD[k]; + } + buffer = new float[][] { new float[2 * l], new float[2 * l] }; + next_buffer = 0; + } + + public override void SwapIndex(int i, int j) + { + do { sbyte _ = sign[i]; sign[i] = sign[j]; sign[j] = _; } while (false); + do { int _ = index[i]; index[i] = index[j]; index[j] = _; } while (false); + do { double _ = QD[i]; QD[i] = QD[j]; QD[j] = _; } while (false); + } + + public override float[] GetQ(int i, int len) + { + float[] data; + int j, real_i = index[i]; + if (cache.GetData(real_i, out data, l) < l) + { + for (j = 0; j < l; j++) + data[j] = (float)KernelFunction(real_i, j); + } + + // reorder and copy + float[] buf = buffer[next_buffer]; + next_buffer = 1 - next_buffer; + sbyte si = sign[i]; + for (j = 0; j < len; j++) + buf[j] = (float)si * sign[j] * data[index[j]]; + return buf; + } + + public override double[] GetQD() + { + return QD; + } + } + + + internal static class Procedures + { + private static bool _verbose; + public static bool IsVerbose + { + get + { + return _verbose; + } + set + { + _verbose = value; + } + } + + // + // construct and solve various formulations + // + public const int LIBSVM_VERSION = 318; + private static Random rand = new Random(); + + private static TextWriter svm_print_stdout = Console.Out; + + public static void setRandomSeed(int seed) + { + rand = new Random(seed); + } + + public static void info(String s) + { + if (IsVerbose) + svm_print_stdout.Write(s); + } + + private static void solve_c_svc(Problem prob, Parameter param, + double[] alpha, Solver.SolutionInfo si, + double Cp, double Cn) + { + int l = prob.Count; + double[] minus_ones = new double[l]; + sbyte[] y = new sbyte[l]; + + int i; + + for (i = 0; i < l; i++) + { + alpha[i] = 0; + minus_ones[i] = -1; + if (prob.Y[i] > 0) y[i] = +1; else y[i] = -1; + } + + Solver s = new Solver(); + s.Solve(l, new SVC_Q(prob, param, y), minus_ones, y, + alpha, Cp, Cn, param.EPS, si, param.Shrinking); + + double sum_alpha = 0; + for (i = 0; i < l; i++) + sum_alpha += alpha[i]; + + if (Cp == Cn) + info("nu = " + sum_alpha / (Cp * prob.Count) + "\n"); + + for (i = 0; i < l; i++) + alpha[i] *= y[i]; + } + + private static void solve_nu_svc(Problem prob, Parameter param, + double[] alpha, Solver.SolutionInfo si) + { + int i; + int l = prob.Count; + double nu = param.Nu; + + sbyte[] y = new sbyte[l]; + + for (i = 0; i < l; i++) + if (prob.Y[i] > 0) + y[i] = +1; + else + y[i] = -1; + + double sum_pos = nu * l / 2; + double sum_neg = nu * l / 2; + + for (i = 0; i < l; i++) + if (y[i] == +1) + { + alpha[i] = Math.Min(1.0, sum_pos); + sum_pos -= alpha[i]; + } + else + { + alpha[i] = Math.Min(1.0, sum_neg); + sum_neg -= alpha[i]; + } + + double[] zeros = new double[l]; + + for (i = 0; i < l; i++) + zeros[i] = 0; + + Solver_NU s = new Solver_NU(); + s.Solve(l, new SVC_Q(prob, param, y), zeros, y, + alpha, 1.0, 1.0, param.EPS, si, param.Shrinking); + double r = si.r; + + info("C = " + 1 / r + "\n"); + + for (i = 0; i < l; i++) + alpha[i] *= y[i] / r; + + si.rho /= r; + si.obj /= (r * r); + si.upper_bound_p = 1 / r; + si.upper_bound_n = 1 / r; + } + + private static void solve_one_class(Problem prob, Parameter param, + double[] alpha, Solver.SolutionInfo si) + { + int l = prob.Count; + double[] zeros = new double[l]; + sbyte[] ones = new sbyte[l]; + int i; + + int n = (int)(param.Nu * prob.Count); // # of alpha's at upper bound + + for (i = 0; i < n; i++) + alpha[i] = 1; + if (n < prob.Count) + alpha[n] = param.Nu * prob.Count - n; + for (i = n + 1; i < l; i++) + alpha[i] = 0; + + for (i = 0; i < l; i++) + { + zeros[i] = 0; + ones[i] = 1; + } + + Solver s = new Solver(); + s.Solve(l, new ONE_CLASS_Q(prob, param), zeros, ones, + alpha, 1.0, 1.0, param.EPS, si, param.Shrinking); + } + + private static void solve_epsilon_svr(Problem prob, Parameter param, + double[] alpha, Solver.SolutionInfo si) + { + int l = prob.Count; + double[] alpha2 = new double[2 * l]; + double[] linear_term = new double[2 * l]; + sbyte[] y = new sbyte[2 * l]; + int i; + + for (i = 0; i < l; i++) + { + alpha2[i] = 0; + linear_term[i] = param.P - prob.Y[i]; + y[i] = 1; + + alpha2[i + l] = 0; + linear_term[i + l] = param.P + prob.Y[i]; + y[i + l] = -1; + } + + Solver s = new Solver(); + s.Solve(2 * l, new SVR_Q(prob, param), linear_term, y, + alpha2, param.C, param.C, param.EPS, si, param.Shrinking); + + double sum_alpha = 0; + for (i = 0; i < l; i++) + { + alpha[i] = alpha2[i] - alpha2[i + l]; + sum_alpha += Math.Abs(alpha[i]); + } + info("nu = " + sum_alpha / (param.C * l) + "\n"); + } + + private static void solve_nu_svr(Problem prob, Parameter param, + double[] alpha, Solver.SolutionInfo si) + { + int l = prob.Count; + double C = param.C; + double[] alpha2 = new double[2 * l]; + double[] linear_term = new double[2 * l]; + sbyte[] y = new sbyte[2 * l]; + int i; + + double sum = C * param.Nu * l / 2; + for (i = 0; i < l; i++) + { + alpha2[i] = alpha2[i + l] = Math.Min(sum, C); + sum -= alpha2[i]; + + linear_term[i] = -prob.Y[i]; + y[i] = 1; + + linear_term[i + l] = prob.Y[i]; + y[i + l] = -1; + } + + Solver_NU s = new Solver_NU(); + s.Solve(2 * l, new SVR_Q(prob, param), linear_term, y, + alpha2, C, C, param.EPS, si, param.Shrinking); + + info("epsilon = " + (-si.r) + "\n"); + + for (i = 0; i < l; i++) + alpha[i] = alpha2[i] - alpha2[i + l]; + } + + // + // decision_function + // + private class decision_function + { + public double[] alpha { get; set; } + public double rho { get; set; } + }; + + static decision_function svm_train_one( + Problem prob, Parameter param, + double Cp, double Cn) + { + double[] alpha = new double[prob.Count]; + Solver.SolutionInfo si = new Solver.SolutionInfo(); + switch (param.SvmType) + { + case SvmType.C_SVC: + solve_c_svc(prob, param, alpha, si, Cp, Cn); + break; + case SvmType.NU_SVC: + solve_nu_svc(prob, param, alpha, si); + break; + case SvmType.ONE_CLASS: + solve_one_class(prob, param, alpha, si); + break; + case SvmType.EPSILON_SVR: + solve_epsilon_svr(prob, param, alpha, si); + break; + case SvmType.NU_SVR: + solve_nu_svr(prob, param, alpha, si); + break; + } + + info("obj = " + si.obj + ", rho = " + si.rho + "\n"); + + // output SVs + + int nSV = 0; + int nBSV = 0; + for (int i = 0; i < prob.Count; i++) + { + if (Math.Abs(alpha[i]) > 0) + { + ++nSV; + if (prob.Y[i] > 0) + { + if (Math.Abs(alpha[i]) >= si.upper_bound_p) + ++nBSV; + } + else + { + if (Math.Abs(alpha[i]) >= si.upper_bound_n) + ++nBSV; + } + } + } + + info("nSV = " + nSV + ", nBSV = " + nBSV + "\n"); + + decision_function f = new decision_function(); + f.alpha = alpha; + f.rho = si.rho; + return f; + } + + // Platt's binary SVM Probablistic Output: an improvement from Lin et al. + private static void sigmoid_train(int l, double[] dec_values, double[] labels, + double[] probAB) + { + double A, B; + double prior1 = 0, prior0 = 0; + int i; + + for (i = 0; i < l; i++) + if (labels[i] > 0) prior1 += 1; + else prior0 += 1; + + int max_iter = 100; // Maximal number of iterations + double min_step = 1e-10; // Minimal step taken in line search + double sigma = 1e-12; // For numerically strict PD of Hessian + double eps = 1e-5; + double hiTarget = (prior1 + 1.0) / (prior1 + 2.0); + double loTarget = 1 / (prior0 + 2.0); + double[] t = new double[l]; + double fApB, p, q, h11, h22, h21, g1, g2, det, dA, dB, gd, stepsize; + double newA, newB, newf, d1, d2; + int iter; + + // Initial Point and Initial Fun Value + A = 0.0; B = Math.Log((prior0 + 1.0) / (prior1 + 1.0)); + double fval = 0.0; + + for (i = 0; i < l; i++) + { + if (labels[i] > 0) t[i] = hiTarget; + else t[i] = loTarget; + fApB = dec_values[i] * A + B; + if (fApB >= 0) + fval += t[i] * fApB + Math.Log(1 + Math.Exp(-fApB)); + else + fval += (t[i] - 1) * fApB + Math.Log(1 + Math.Exp(fApB)); + } + for (iter = 0; iter < max_iter; iter++) + { + // Update Gradient and Hessian (use H' = H + sigma I) + h11 = sigma; // numerically ensures strict PD + h22 = sigma; + h21 = 0.0; g1 = 0.0; g2 = 0.0; + for (i = 0; i < l; i++) + { + fApB = dec_values[i] * A + B; + if (fApB >= 0) + { + p = Math.Exp(-fApB) / (1.0 + Math.Exp(-fApB)); + q = 1.0 / (1.0 + Math.Exp(-fApB)); + } + else + { + p = 1.0 / (1.0 + Math.Exp(fApB)); + q = Math.Exp(fApB) / (1.0 + Math.Exp(fApB)); + } + d2 = p * q; + h11 += dec_values[i] * dec_values[i] * d2; + h22 += d2; + h21 += dec_values[i] * d2; + d1 = t[i] - p; + g1 += dec_values[i] * d1; + g2 += d1; + } + + // Stopping Criteria + if (Math.Abs(g1) < eps && Math.Abs(g2) < eps) + break; + + // Finding Newton direction: -inv(H') * g + det = h11 * h22 - h21 * h21; + dA = -(h22 * g1 - h21 * g2) / det; + dB = -(-h21 * g1 + h11 * g2) / det; + gd = g1 * dA + g2 * dB; + + + stepsize = 1; // Line Search + while (stepsize >= min_step) + { + newA = A + stepsize * dA; + newB = B + stepsize * dB; + + // New function value + newf = 0.0; + for (i = 0; i < l; i++) + { + fApB = dec_values[i] * newA + newB; + if (fApB >= 0) + newf += t[i] * fApB + Math.Log(1 + Math.Exp(-fApB)); + else + newf += (t[i] - 1) * fApB + Math.Log(1 + Math.Exp(fApB)); + } + // Check sufficient decrease + if (newf < fval + 0.0001 * stepsize * gd) + { + A = newA; B = newB; fval = newf; + break; + } + else + stepsize = stepsize / 2.0; + } + + if (stepsize < min_step) + { + info("Line search fails in two-class probability estimates\n"); + break; + } + } + + if (iter >= max_iter) + info("Reaching maximal iterations in two-class probability estimates\n"); + probAB[0] = A; probAB[1] = B; + } + + private static double sigmoid_predict(double decision_value, double A, double B) + { + double fApB = decision_value * A + B; + if (fApB >= 0) + return Math.Exp(-fApB) / (1.0 + Math.Exp(-fApB)); + else + return 1.0 / (1 + Math.Exp(fApB)); + } + + // Method 2 from the multiclass_prob paper by Wu, Lin, and Weng + private static void multiclass_probability(int k, double[,] r, double[] p) + { + int t, j; + int iter = 0, max_iter = Math.Max(100, k); + double[,] Q = new double[k, k]; + double[] Qp = new double[k]; + double pQp, eps = 0.005 / k; + + for (t = 0; t < k; t++) + { + p[t] = 1.0 / k; // Valid if k = 1 + Q[t, t] = 0; + for (j = 0; j < t; j++) + { + Q[t, t] += r[j, t] * r[j, t]; + Q[t, j] = Q[j, t]; + } + for (j = t + 1; j < k; j++) + { + Q[t, t] += r[j, t] * r[j, t]; + Q[t, j] = -r[j, t] * r[t, j]; + } + } + for (iter = 0; iter < max_iter; iter++) + { + // stopping condition, recalculate QP,pQP for numerical accuracy + pQp = 0; + for (t = 0; t < k; t++) + { + Qp[t] = 0; + for (j = 0; j < k; j++) + Qp[t] += Q[t, j] * p[j]; + pQp += p[t] * Qp[t]; + } + double max_error = 0; + for (t = 0; t < k; t++) + { + double error = Math.Abs(Qp[t] - pQp); + if (error > max_error) + max_error = error; + } + if (max_error < eps) break; + + for (t = 0; t < k; t++) + { + double diff = (-Qp[t] + pQp) / Q[t, t]; + p[t] += diff; + pQp = (pQp + diff * (diff * Q[t, t] + 2 * Qp[t])) / (1 + diff) / (1 + diff); + for (j = 0; j < k; j++) + { + Qp[j] = (Qp[j] + diff * Q[t, j]) / (1 + diff); + p[j] /= (1 + diff); + } + } + } + if (iter >= max_iter) + info("Exceeds max_iter in multiclass_prob\n"); + } + + // Cross-validation decision values for probability estimates + private static void svm_binary_svc_probability(Problem prob, Parameter param, double Cp, double Cn, double[] probAB) + { + int i; + int nr_fold = 5; + int[] perm = new int[prob.Count]; + double[] dec_values = new double[prob.Count]; + + // random shuffle + for (i = 0; i < prob.Count; i++) perm[i] = i; + for (i = 0; i < prob.Count; i++) + { + int j = i + rand.Next(prob.Count - i); + do { int _ = perm[i]; perm[i] = perm[j]; perm[j] = _; } while (false); + } + for (i = 0; i < nr_fold; i++) + { + int begin = i * prob.Count / nr_fold; + int end = (i + 1) * prob.Count / nr_fold; + int j, k; + Problem subprob = new Problem(); + + subprob.Count = prob.Count - (end - begin); + subprob.X = new Node[subprob.Count][]; + subprob.Y = new double[subprob.Count]; + + k = 0; + for (j = 0; j < begin; j++) + { + subprob.X[k] = prob.X[perm[j]]; + subprob.Y[k] = prob.Y[perm[j]]; + ++k; + } + for (j = end; j < prob.Count; j++) + { + subprob.X[k] = prob.X[perm[j]]; + subprob.Y[k] = prob.Y[perm[j]]; + ++k; + } + int p_count = 0, n_count = 0; + for (j = 0; j < k; j++) + if (subprob.Y[j] > 0) + p_count++; + else + n_count++; + + if (p_count == 0 && n_count == 0) + for (j = begin; j < end; j++) + dec_values[perm[j]] = 0; + else if (p_count > 0 && n_count == 0) + for (j = begin; j < end; j++) + dec_values[perm[j]] = 1; + else if (p_count == 0 && n_count > 0) + for (j = begin; j < end; j++) + dec_values[perm[j]] = -1; + else + { + Parameter subparam = (Parameter)param.Clone(); + subparam.Probability = false; + subparam.C = 1.0; + subparam.Weights[1] = Cp; + subparam.Weights[-1] = Cn; + Model submodel = svm_train(subprob, subparam); + for (j = begin; j < end; j++) + { + double[] dec_value = new double[1]; + svm_predict_values(submodel, prob.X[perm[j]], dec_value); + dec_values[perm[j]] = dec_value[0]; + // ensure +1 -1 order; reason not using CV subroutine + dec_values[perm[j]] *= submodel.ClassLabels[0]; + } + } + } + sigmoid_train(prob.Count, dec_values, prob.Y, probAB); + } + + // Return parameter of a Laplace distribution + private static double svm_svr_probability(Problem prob, Parameter param) + { + int i; + int nr_fold = 5; + double[] ymv = new double[prob.Count]; + double mae = 0; + + Parameter newparam = (Parameter)param.Clone(); + newparam.Probability = false; + svm_cross_validation(prob, newparam, nr_fold, ymv); + for (i = 0; i < prob.Count; i++) + { + ymv[i] = prob.Y[i] - ymv[i]; + mae += Math.Abs(ymv[i]); + } + mae /= prob.Count; + double std = Math.Sqrt(2 * mae * mae); + int count = 0; + mae = 0; + for (i = 0; i < prob.Count; i++) + if (Math.Abs(ymv[i]) > 5 * std) + count = count + 1; + else + mae += Math.Abs(ymv[i]); + mae /= (prob.Count - count); + info("Prob. model for test data: target value = predicted value + z,\nz: Laplace distribution e^(-|z|/sigma)/(2sigma),sigma=" + mae + "\n"); + return mae; + } + + // label: label name, start: begin of each class, count: #data of classes, perm: indices to the original data + // perm, length l, must be allocated before calling this subroutine + private static void svm_group_classes(Problem prob, out int nr_class_ret, out int[] label_ret, out int[] start_ret, out int[] count_ret, int[] perm) + { + int l = prob.Count; + int max_nr_class = 16; + int nr_class = 0; + int[] label = new int[max_nr_class]; + int[] count = new int[max_nr_class]; + int[] data_label = new int[l]; + int i; + + for (i = 0; i < l; i++) + { + int this_label = (int)(prob.Y[i]); + int j; + for (j = 0; j < nr_class; j++) + { + if (this_label == label[j]) + { + ++count[j]; + break; + } + } + data_label[i] = j; + if (j == nr_class) + { + if (nr_class == max_nr_class) + { + max_nr_class *= 2; + int[] new_data = new int[max_nr_class]; + Array.Copy(label, 0, new_data, 0, label.Length); + label = new_data; + new_data = new int[max_nr_class]; + Array.Copy(count, 0, new_data, 0, count.Length); + count = new_data; + } + label[nr_class] = this_label; + count[nr_class] = 1; + ++nr_class; + } + } + + // + // Labels are ordered by their first occurrence in the training set. + // However, for two-class sets with -1/+1 labels and -1 appears first, + // we swap labels to ensure that internally the binary SVM has positive data corresponding to the +1 instances. + // + if (nr_class == 2 && label[0] == -1 && label[1] == +1) + { + do { int _ = label[0]; label[0] = label[1]; label[1] = _; } while (false); + do { int _ = count[0]; count[0] = count[1]; count[1] = _; } while (false); + for (i = 0; i < l; i++) + { + if (data_label[i] == 0) + data_label[i] = 1; + else + data_label[i] = 0; + } + } + + int[] start = new int[nr_class]; + start[0] = 0; + for (i = 1; i < nr_class; i++) + start[i] = start[i - 1] + count[i - 1]; + for (i = 0; i < l; i++) + { + perm[start[data_label[i]]] = i; + ++start[data_label[i]]; + } + start[0] = 0; + for (i = 1; i < nr_class; i++) + start[i] = start[i - 1] + count[i - 1]; + + nr_class_ret = nr_class; + label_ret = label; + start_ret = start; + count_ret = count; + } + + // + // Interface functions + // + public static Model svm_train(Problem prob, Parameter param) + { + Model model = new Model(); + model.Parameter = param; + + if (param.SvmType == SvmType.ONE_CLASS || + param.SvmType == SvmType.EPSILON_SVR || + param.SvmType == SvmType.NU_SVR) + { + // regression or one-class-svm + model.NumberOfClasses = 2; + model.ClassLabels = null; + model.NumberOfSVPerClass = null; + model.PairwiseProbabilityA = null; model.PairwiseProbabilityB = null; + model.SupportVectorCoefficients = new double[1][]; + + if (param.Probability && + (param.SvmType == SvmType.EPSILON_SVR || + param.SvmType == SvmType.NU_SVR)) + { + model.PairwiseProbabilityA = new double[1]; + model.PairwiseProbabilityA[0] = svm_svr_probability(prob, param); + } + + decision_function f = svm_train_one(prob, param, 0, 0); + model.Rho = new double[1]; + model.Rho[0] = f.rho; + + int nSV = 0; + int i; + for (i = 0; i < prob.Count; i++) + if (Math.Abs(f.alpha[i]) > 0) ++nSV; + model.SupportVectorCount = nSV; + model.SupportVectors = new Node[nSV][]; + model.SupportVectorCoefficients[0] = new double[nSV]; + model.SupportVectorIndices = new int[nSV]; + int j = 0; + for (i = 0; i < prob.Count; i++) + if (Math.Abs(f.alpha[i]) > 0) + { + model.SupportVectors[j] = prob.X[i]; + model.SupportVectorCoefficients[0][j] = f.alpha[i]; + model.SupportVectorIndices[j] = i + 1; + ++j; + } + } + else + { + // classification + int l = prob.Count; + int nr_class; + int[] label; + int[] start; + int[] count; + int[] perm = new int[l]; + + // group training data of the same class + svm_group_classes(prob, out nr_class, out label, out start, out count, perm); + + + if (nr_class == 1) + info("WARNING: training data in only one class. See README for details.\n"); + + Node[][] x = new Node[l][]; + int i; + for (i = 0; i < l; i++) + x[i] = prob.X[perm[i]]; + + // calculate weighted C + + double[] weighted_C = new double[nr_class]; + for (i = 0; i < nr_class; i++) + weighted_C[i] = param.C; + for (i = 0; i < nr_class; i++) + { + if (!param.Weights.ContainsKey(label[i])) + Console.Error.Write("WARNING: class label " + label[i] + " specified in weight is not found\n"); + else + weighted_C[i] *= param.Weights[label[i]]; + } + + // train k*(k-1)/2 models + + bool[] nonzero = new bool[l]; + for (i = 0; i < l; i++) + nonzero[i] = false; + decision_function[] f = new decision_function[nr_class * (nr_class - 1) / 2]; + + double[] probA = null, probB = null; + if (param.Probability) + { + probA = new double[nr_class * (nr_class - 1) / 2]; + probB = new double[nr_class * (nr_class - 1) / 2]; + } + + int p = 0; + for (i = 0; i < nr_class; i++) + for (int j = i + 1; j < nr_class; j++) + { + Problem sub_prob = new Problem(); + int si = start[i], sj = start[j]; + int ci = count[i], cj = count[j]; + sub_prob.Count = ci + cj; + sub_prob.X = new Node[sub_prob.Count][]; + sub_prob.Y = new double[sub_prob.Count]; + int k; + for (k = 0; k < ci; k++) + { + sub_prob.X[k] = x[si + k]; + sub_prob.Y[k] = +1; + } + for (k = 0; k < cj; k++) + { + sub_prob.X[ci + k] = x[sj + k]; + sub_prob.Y[ci + k] = -1; + } + + if (param.Probability) + { + double[] probAB = new double[2]; + svm_binary_svc_probability(sub_prob, param, weighted_C[i], weighted_C[j], probAB); + probA[p] = probAB[0]; + probB[p] = probAB[1]; + } + + f[p] = svm_train_one(sub_prob, param, weighted_C[i], weighted_C[j]); + for (k = 0; k < ci; k++) + if (!nonzero[si + k] && Math.Abs(f[p].alpha[k]) > 0) + nonzero[si + k] = true; + for (k = 0; k < cj; k++) + if (!nonzero[sj + k] && Math.Abs(f[p].alpha[ci + k]) > 0) + nonzero[sj + k] = true; + ++p; + } + + // build output + + model.NumberOfClasses = nr_class; + + model.ClassLabels = new int[nr_class]; + for (i = 0; i < nr_class; i++) + model.ClassLabels[i] = label[i]; + + model.Rho = new double[nr_class * (nr_class - 1) / 2]; + for (i = 0; i < nr_class * (nr_class - 1) / 2; i++) + model.Rho[i] = f[i].rho; + + if (param.Probability) + { + model.PairwiseProbabilityA = new double[nr_class * (nr_class - 1) / 2]; + model.PairwiseProbabilityB = new double[nr_class * (nr_class - 1) / 2]; + for (i = 0; i < nr_class * (nr_class - 1) / 2; i++) + { + model.PairwiseProbabilityA[i] = probA[i]; + model.PairwiseProbabilityB[i] = probB[i]; + } + } + else + { + model.PairwiseProbabilityA = null; + model.PairwiseProbabilityA = null; + } + + int nnz = 0; + int[] nz_count = new int[nr_class]; + model.NumberOfSVPerClass = new int[nr_class]; + for (i = 0; i < nr_class; i++) + { + int nSV = 0; + for (int j = 0; j < count[i]; j++) + if (nonzero[start[i] + j]) + { + ++nSV; + ++nnz; + } + model.NumberOfSVPerClass[i] = nSV; + nz_count[i] = nSV; + } + + info("Total nSV = " + nnz + "\n"); + + model.SupportVectorCount = nnz; + model.SupportVectors = new Node[nnz][]; + model.SupportVectorIndices = new int[nnz]; + p = 0; + for (i = 0; i < l; i++) + if (nonzero[i]) + { + model.SupportVectors[p] = x[i]; + model.SupportVectorIndices[p++] = perm[i] + 1; + } + + int[] nz_start = new int[nr_class]; + nz_start[0] = 0; + for (i = 1; i < nr_class; i++) + nz_start[i] = nz_start[i - 1] + nz_count[i - 1]; + + model.SupportVectorCoefficients = new double[nr_class - 1][]; + for (i = 0; i < nr_class - 1; i++) + model.SupportVectorCoefficients[i] = new double[nnz]; + + p = 0; + for (i = 0; i < nr_class; i++) + for (int j = i + 1; j < nr_class; j++) + { + // classifier (i,j): coefficients with + // i are in sv_coef[j-1][nz_start[i]...], + // j are in sv_coef[i][nz_start[j]...] + + int si = start[i]; + int sj = start[j]; + int ci = count[i]; + int cj = count[j]; + + int q = nz_start[i]; + int k; + for (k = 0; k < ci; k++) + if (nonzero[si + k]) + model.SupportVectorCoefficients[j - 1][q++] = f[p].alpha[k]; + q = nz_start[j]; + for (k = 0; k < cj; k++) + if (nonzero[sj + k]) + model.SupportVectorCoefficients[i][q++] = f[p].alpha[ci + k]; + ++p; + } + } + return model; + } + + // Stratified cross validation + public static void svm_cross_validation(Problem prob, Parameter param, int nr_fold, double[] target) + { + int i; + int[] fold_start = new int[nr_fold + 1]; + int l = prob.Count; + int[] perm = new int[l]; + + // stratified cv may not give leave-one-out rate + // Each class to l folds -> some folds may have zero elements + if ((param.SvmType == SvmType.C_SVC || + param.SvmType == SvmType.NU_SVC) && nr_fold < l) + { + int nr_class; + int[] label; + int[] start; + int[] count; + + svm_group_classes(prob, out nr_class, out label, out start, out count, perm); + + + // random shuffle and then data grouped by fold using the array perm + int[] fold_count = new int[nr_fold]; + int c; + int[] index = new int[l]; + for (i = 0; i < l; i++) + index[i] = perm[i]; + for (c = 0; c < nr_class; c++) + for (i = 0; i < count[c]; i++) + { + int j = i + rand.Next(count[c] - i); + do { int _ = index[start[c] + j]; index[start[c] + j] = index[start[c] + i]; index[start[c] + i] = _; } while (false); + } + for (i = 0; i < nr_fold; i++) + { + fold_count[i] = 0; + for (c = 0; c < nr_class; c++) + fold_count[i] += (i + 1) * count[c] / nr_fold - i * count[c] / nr_fold; + } + fold_start[0] = 0; + for (i = 1; i <= nr_fold; i++) + fold_start[i] = fold_start[i - 1] + fold_count[i - 1]; + for (c = 0; c < nr_class; c++) + for (i = 0; i < nr_fold; i++) + { + int begin = start[c] + i * count[c] / nr_fold; + int end = start[c] + (i + 1) * count[c] / nr_fold; + for (int j = begin; j < end; j++) + { + perm[fold_start[i]] = index[j]; + fold_start[i]++; + } + } + fold_start[0] = 0; + for (i = 1; i <= nr_fold; i++) + fold_start[i] = fold_start[i - 1] + fold_count[i - 1]; + } + else + { + for (i = 0; i < l; i++) perm[i] = i; + for (i = 0; i < l; i++) + { + int j = i + rand.Next(l - i); + do { int _ = perm[i]; perm[i] = perm[j]; perm[j] = _; } while (false); + } + for (i = 0; i <= nr_fold; i++) + fold_start[i] = i * l / nr_fold; + } + + for (i = 0; i < nr_fold; i++) + { + int begin = fold_start[i]; + int end = fold_start[i + 1]; + int j, k; + Problem subprob = new Problem(); + + subprob.Count = l - (end - begin); + subprob.X = new Node[subprob.Count][]; + subprob.Y = new double[subprob.Count]; + + k = 0; + for (j = 0; j < begin; j++) + { + subprob.X[k] = prob.X[perm[j]]; + subprob.Y[k] = prob.Y[perm[j]]; + ++k; + } + for (j = end; j < l; j++) + { + subprob.X[k] = prob.X[perm[j]]; + subprob.Y[k] = prob.Y[perm[j]]; + ++k; + } + Model submodel = svm_train(subprob, param); + if (param.Probability && + (param.SvmType == SvmType.C_SVC || + param.SvmType == SvmType.NU_SVC)) + { + double[] prob_estimates = new double[svm_get_nr_class(submodel)]; + for (j = begin; j < end; j++) + target[perm[j]] = svm_predict_probability(submodel, prob.X[perm[j]], prob_estimates); + } + else + for (j = begin; j < end; j++) + target[perm[j]] = svm_predict(submodel, prob.X[perm[j]]); + } + } + + public static SvmType svm_get_svm_type(Model model) + { + return model.Parameter.SvmType; + } + + public static int svm_get_nr_class(Model model) + { + return model.NumberOfClasses; + } + + public static void svm_get_labels(Model model, int[] label) + { + if (model.ClassLabels != null) + for (int i = 0; i < model.NumberOfClasses; i++) + label[i] = model.ClassLabels[i]; + } + + public static void svm_get_sv_indices(Model model, int[] indices) + { + if (model.SupportVectorIndices != null) + for (int i = 0; i < model.SupportVectorCount; i++) + indices[i] = model.SupportVectorIndices[i]; + } + + public static int svm_get_nr_sv(Model model) + { + return model.SupportVectorCount; + } + + public static double svm_get_svr_probability(Model model) + { + if ((model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) && + model.PairwiseProbabilityA != null) + return model.PairwiseProbabilityA[0]; + else + { + Console.Error.Write("Model doesn't contain information for SVR probability inference\n"); + return 0; + } + } + + public static double svm_predict_values(Model model, Node[] x, double[] dec_values) + { + int i; + if (model.Parameter.SvmType == SvmType.ONE_CLASS || + model.Parameter.SvmType == SvmType.EPSILON_SVR || + model.Parameter.SvmType == SvmType.NU_SVR) + { + double[] sv_coef = model.SupportVectorCoefficients[0]; + double sum = 0; + for (i = 0; i < model.SupportVectorCount; i++) + sum += sv_coef[i] * Kernel.KernelFunction(x, model.SupportVectors[i], model.Parameter); + sum -= model.Rho[0]; + dec_values[0] = sum; + + if (model.Parameter.SvmType == SvmType.ONE_CLASS) + return (sum > 0) ? 1 : -1; + else + return sum; + } + else + { + int nr_class = model.NumberOfClasses; + int l = model.SupportVectorCount; + + double[] kvalue = new double[l]; + for (i = 0; i < l; i++) + kvalue[i] = Kernel.KernelFunction(x, model.SupportVectors[i], model.Parameter); + + int[] start = new int[nr_class]; + start[0] = 0; + for (i = 1; i < nr_class; i++) + start[i] = start[i - 1] + model.NumberOfSVPerClass[i - 1]; + + int[] vote = new int[nr_class]; + for (i = 0; i < nr_class; i++) + vote[i] = 0; + + int p = 0; + for (i = 0; i < nr_class; i++) + for (int j = i + 1; j < nr_class; j++) + { + double sum = 0; + int si = start[i]; + int sj = start[j]; + int ci = model.NumberOfSVPerClass[i]; + int cj = model.NumberOfSVPerClass[j]; + + int k; + double[] coef1 = model.SupportVectorCoefficients[j - 1]; + double[] coef2 = model.SupportVectorCoefficients[i]; + for (k = 0; k < ci; k++) + sum += coef1[si + k] * kvalue[si + k]; + for (k = 0; k < cj; k++) + sum += coef2[sj + k] * kvalue[sj + k]; + sum -= model.Rho[p]; + dec_values[p] = sum; + + if (dec_values[p] > 0) + ++vote[i]; + else + ++vote[j]; + p++; + } + + int vote_max_idx = 0; + for (i = 1; i < nr_class; i++) + if (vote[i] > vote[vote_max_idx]) + vote_max_idx = i; + + return model.ClassLabels[vote_max_idx]; + } + } + + public static double svm_predict(Model model, Node[] x) + { + int nr_class = model.NumberOfClasses; + double[] dec_values; + if (model.Parameter.SvmType == SvmType.ONE_CLASS || + model.Parameter.SvmType == SvmType.EPSILON_SVR || + model.Parameter.SvmType == SvmType.NU_SVR) + dec_values = new double[1]; + else + dec_values = new double[nr_class * (nr_class - 1) / 2]; + double pred_result = svm_predict_values(model, x, dec_values); + return pred_result; + } + + public static double svm_predict_probability(Model model, Node[] x, double[] prob_estimates) + { + if ((model.Parameter.SvmType == SvmType.C_SVC || model.Parameter.SvmType == SvmType.NU_SVC) && + model.PairwiseProbabilityA != null && model.PairwiseProbabilityB != null) + { + int i; + int nr_class = model.NumberOfClasses; + double[] dec_values = new double[nr_class * (nr_class - 1) / 2]; + svm_predict_values(model, x, dec_values); + + double min_prob = 1e-7; + double[,] pairwise_prob = new double[nr_class, nr_class]; + + int k = 0; + for (i = 0; i < nr_class; i++) + for (int j = i + 1; j < nr_class; j++) + { + pairwise_prob[i, j] = Math.Min(Math.Max(sigmoid_predict(dec_values[k], model.PairwiseProbabilityA[k], model.PairwiseProbabilityB[k]), min_prob), 1 - min_prob); + pairwise_prob[j, i] = 1 - pairwise_prob[i, j]; + k++; + } + multiclass_probability(nr_class, pairwise_prob, prob_estimates); + + int prob_max_idx = 0; + for (i = 1; i < nr_class; i++) + if (prob_estimates[i] > prob_estimates[prob_max_idx]) + prob_max_idx = i; + return model.ClassLabels[prob_max_idx]; + } + else + return svm_predict(model, x); + } + + public static String svm_check_parameter(Problem prob, Parameter param) + { + // svm_type + + SvmType svm_type = param.SvmType; + + // kernel_type, degree + + KernelType kernel_type = param.KernelType; + + if (param.Gamma < 0) + return "gamma < 0"; + + if (param.Degree < 0) + return "degree of polynomial kernel < 0"; + + // cache_size,eps,C,nu,p,shrinking + + if (param.CacheSize <= 0) + return "cache_size <= 0"; + + if (param.EPS <= 0) + return "eps <= 0"; + + if (svm_type == SvmType.C_SVC || + svm_type == SvmType.EPSILON_SVR || + svm_type == SvmType.NU_SVR) + if (param.C <= 0) + return "C <= 0"; + + if (svm_type == SvmType.NU_SVC || + svm_type == SvmType.ONE_CLASS || + svm_type == SvmType.NU_SVR) + if (param.Nu <= 0 || param.Nu > 1) + return "nu <= 0 or nu > 1"; + + if (svm_type == SvmType.EPSILON_SVR) + if (param.P < 0) + return "p < 0"; + + if (param.Probability && + svm_type == SvmType.ONE_CLASS) + return "one-class SVM probability output not supported yet"; + + // check whether nu-svc is feasible + + if (svm_type == SvmType.NU_SVC) + { + int l = prob.Count; + int max_nr_class = 16; + int nr_class = 0; + int[] label = new int[max_nr_class]; + int[] count = new int[max_nr_class]; + + int i; + for (i = 0; i < l; i++) + { + int this_label = (int)prob.Y[i]; + int j; + for (j = 0; j < nr_class; j++) + if (this_label == label[j]) + { + ++count[j]; + break; + } + + if (j == nr_class) + { + if (nr_class == max_nr_class) + { + max_nr_class *= 2; + int[] new_data = new int[max_nr_class]; + Array.Copy(label, 0, new_data, 0, label.Length); + label = new_data; + + new_data = new int[max_nr_class]; + Array.Copy(count, 0, new_data, 0, count.Length); + count = new_data; + } + label[nr_class] = this_label; + count[nr_class] = 1; + ++nr_class; + } + } + + for (i = 0; i < nr_class; i++) + { + int n1 = count[i]; + for (int j = i + 1; j < nr_class; j++) + { + int n2 = count[j]; + if (param.Nu * (n1 + n2) / 2 > Math.Min(n1, n2)) + return "specified nu is infeasible"; + } + } + } + + return null; + } + + public static int svm_check_probability_model(Model model) + { + if (((model.Parameter.SvmType == SvmType.C_SVC || model.Parameter.SvmType == SvmType.NU_SVC) && + model.PairwiseProbabilityA != null && model.PairwiseProbabilityB != null) || + ((model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) && + model.PairwiseProbabilityA != null)) + return 1; + else + return 0; + } + } +} diff --git a/BotSharp.MachineLearning/SVM/TemporaryCulture.cs b/BotSharp.MachineLearning/SVM/TemporaryCulture.cs new file mode 100644 index 00000000..0b451e3c --- /dev/null +++ b/BotSharp.MachineLearning/SVM/TemporaryCulture.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Globalization; +using System.Threading; + +namespace SVM.BotSharp.MachineLearning +{ + internal static class TemporaryCulture + { + private static CultureInfo _culture; + + public static void Start() + { + _culture = Thread.CurrentThread.CurrentCulture; + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + } + + public static void Stop() + { + Thread.CurrentThread.CurrentCulture = _culture; + } + } +} diff --git a/BotSharp.MachineLearning/SVM/Training.cs b/BotSharp.MachineLearning/SVM/Training.cs new file mode 100644 index 00000000..87ae2a80 --- /dev/null +++ b/BotSharp.MachineLearning/SVM/Training.cs @@ -0,0 +1,232 @@ +/* + * SVM.NET Library + * Copyright (C) 2008 Matthew Johnson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +using System; +using System.Collections.Generic; + +namespace SVM.BotSharp.MachineLearning +{ + /// + /// Class containing the routines to train SVM models. + /// + public static class Training + { + /// + /// Whether the system will output information to the console during the training process. + /// + public static bool IsVerbose + { + get + { + return Procedures.IsVerbose; + } + set + { + Procedures.IsVerbose = value; + } + } + + private static double doCrossValidation(Problem problem, Parameter parameters, int nr_fold) + { + int i; + double[] target = new double[problem.Count]; + Procedures.svm_cross_validation(problem, parameters, nr_fold, target); + int total_correct = 0; + double total_error = 0; + double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0; + if (parameters.SvmType == SvmType.EPSILON_SVR || parameters.SvmType == SvmType.NU_SVR) + { + for (i = 0; i < problem.Count; i++) + { + double y = problem.Y[i]; + double v = target[i]; + total_error += (v - y) * (v - y); + sumv += v; + sumy += y; + sumvv += v * v; + sumyy += y * y; + sumvy += v * y; + } + return(problem.Count * sumvy - sumv * sumy) / (Math.Sqrt(problem.Count * sumvv - sumv * sumv) * Math.Sqrt(problem.Count * sumyy - sumy * sumy)); + } + else + for (i = 0; i < problem.Count; i++) + if (target[i] == problem.Y[i]) + ++total_correct; + return (double)total_correct / problem.Count; + } + + public static void SetRandomSeed(int seed) + { + Procedures.setRandomSeed(seed); + } + + /// + /// Legacy. Allows use as if this was svm_train. See libsvm documentation for details on which arguments to pass. + /// + /// + [Obsolete("Provided only for legacy compatibility, use the other Train() methods")] + public static void Train(params string[] args) + { + Parameter parameters; + Problem problem; + bool crossValidation; + int nrfold; + string modelFilename; + parseCommandLine(args, out parameters, out problem, out crossValidation, out nrfold, out modelFilename); + if (crossValidation) + PerformCrossValidation(problem, parameters, nrfold); + else Model.Write(modelFilename, Train(problem, parameters)); + } + + /// + /// Performs cross validation. + /// + /// The training data + /// The parameters to test + /// The number of cross validations to use + /// The cross validation score + public static double PerformCrossValidation(Problem problem, Parameter parameters, int nrfold) + { + string error = Procedures.svm_check_parameter(problem, parameters); + if (error == null) + return doCrossValidation(problem, parameters, nrfold); + else throw new Exception(error); + } + + /// + /// Trains a model using the provided training data and parameters. + /// + /// The training data + /// The parameters to use + /// A trained SVM Model + public static Model Train(Problem problem, Parameter parameters) + { + string error = Procedures.svm_check_parameter(problem, parameters); + + if (error == null) + return Procedures.svm_train(problem, parameters); + else throw new Exception(error); + } + + private static void parseCommandLine(string[] args, out Parameter parameters, out Problem problem, out bool crossValidation, out int nrfold, out string modelFilename) + { + int i; + + parameters = new Parameter(); + // default values + + crossValidation = false; + nrfold = 0; + + // parse options + for (i = 0; i < args.Length; i++) + { + if (args[i][0] != '-') + break; + ++i; + switch (args[i - 1][1]) + { + + case 's': + parameters.SvmType = (SvmType)int.Parse(args[i]); + break; + + case 't': + parameters.KernelType = (KernelType)int.Parse(args[i]); + break; + + case 'd': + parameters.Degree = int.Parse(args[i]); + break; + + case 'g': + parameters.Gamma = double.Parse(args[i]); + break; + + case 'r': + parameters.Coefficient0 = double.Parse(args[i]); + break; + + case 'n': + parameters.Nu = double.Parse(args[i]); + break; + + case 'm': + parameters.CacheSize = double.Parse(args[i]); + break; + + case 'c': + parameters.C = double.Parse(args[i]); + break; + + case 'e': + parameters.EPS = double.Parse(args[i]); + break; + + case 'p': + parameters.P = double.Parse(args[i]); + break; + + case 'h': + parameters.Shrinking = int.Parse(args[i]) == 1; + break; + + case 'b': + parameters.Probability = int.Parse(args[i]) == 1; + break; + + case 'v': + crossValidation = true; + nrfold = int.Parse(args[i]); + if (nrfold < 2) + { + throw new ArgumentException("n-fold cross validation: n must >= 2"); + } + break; + + case 'w': + parameters.Weights[int.Parse(args[i - 1].Substring(2))] = double.Parse(args[1]); + break; + + default: + throw new ArgumentException("Unknown Parameter"); + } + } + + // determine filenames + + if (i >= args.Length) + throw new ArgumentException("No input file specified"); + + problem = Problem.Read(args[i]); + + if (parameters.Gamma == 0) + parameters.Gamma = 1.0 / problem.MaxIndex; + + if (i < args.Length - 1) + modelFilename = args[i + 1]; + else + { + int p = args[i].LastIndexOf('/') + 1; + modelFilename = args[i].Substring(p) + ".model"; + } + } + } +} \ No newline at end of file diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj index cc3c0a66..b078963b 100644 --- a/BotSharp.NLP/BotSharp.NLP.csproj +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -18,7 +18,7 @@ - + diff --git a/BotSharp.NLP/Stem/RegexStemmer.cs b/BotSharp.NLP/Stem/RegexStemmer.cs index d33c4fb7..4d98c21e 100644 --- a/BotSharp.NLP/Stem/RegexStemmer.cs +++ b/BotSharp.NLP/Stem/RegexStemmer.cs @@ -1,4 +1,22 @@ -using System; +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/BotSharp.NLP/Tag/NGramTagger.cs b/BotSharp.NLP/Tag/NGramTagger.cs index f9b3e38b..9b73b493 100644 --- a/BotSharp.NLP/Tag/NGramTagger.cs +++ b/BotSharp.NLP/Tag/NGramTagger.cs @@ -1,4 +1,22 @@ -using System; +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/BotSharp.NLP/Tokenize/RegexTokenizer.cs b/BotSharp.NLP/Tokenize/RegexTokenizer.cs index 43cd2a2f..03b04b78 100644 --- a/BotSharp.NLP/Tokenize/RegexTokenizer.cs +++ b/BotSharp.NLP/Tokenize/RegexTokenizer.cs @@ -1,4 +1,22 @@ -using System; +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/README.rst b/README.rst index b9cab1cf..5eb4349b 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,4 @@ -.. image:: https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png - :height: 30px - -The Open Source AI Bot Platform Builder for Enterprise +The Open Source AI Bot Platform Builder for Enterprise ====================================================== *"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our enterprise developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline."* diff --git a/docs/index.rst b/docs/index.rst index cda74fa3..9335bbe4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,8 +3,9 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to BotSharp's documentation! -==================================== +.. image:: https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png + :height: 30px + .. include:: ../README.rst .. toctree::