diff --git a/ChangeLog b/ChangeLog index 8eb7f8e..c29952e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,25 @@ +--- Release 1.0 --- + +2009-09-02 BlackLight + + * all: Most of the code has been rewritten. Now a trained network should + be saved, via save() method, to an XML file, and loaded from that using + the constructor. This way is much cleaner, and easily fixable and + customizable too. The old approach through binary files has been kept yet + through saveToBinary() and loadFromBinary() methods, for + back-compatibility purposes, though it's strongly deprecated. Moreover, + the XML formats (for training the network and for loading a trained + network's information) are fully compatible with the ones used by + NeuralPerl module, mostly a Perl version of this library, so you can use + the same XML files to train and load a neural network using C++ and Perl. + Moreover, a great step forward in the fixing of the dirty old + vulnerability (synaptical random weights overflow) has been made, keeping + the values of the weights very low and ~ 0. I've made tons of tests and + the network never overflowed. Anyway, if you should experience an + overflow (synaptical weights go > 1), just write me. I've also done a + deep re-design of library's classes, using as public methods only the + stuff you *REALLY* need to be public. + --- Release 0.4 --- 2009-08-16 BlackLight diff --git a/VERSION b/VERSION index bd73f47..d3827e7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4 +1.0 diff --git a/examples/Makefile b/examples/Makefile index 59d55e9..45d7a1a 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -2,10 +2,9 @@ all: g++ -Wall -o learnAdd learnAdd.cpp -lneural++ g++ -Wall -o doAdd doAdd.cpp -lneural++ g++ -Wall -o adderFromScratch adderFromScratch.cpp -lneural++ + g++ -Wall -o Add Add.cpp -lneural++ clean: rm learnAdd rm doAdd rm adderFromScratch - rm adder.net - rm adder.xml diff --git a/examples/doAdd.cpp b/examples/doAdd.cpp index f61f2fe..1cba9c1 100644 --- a/examples/doAdd.cpp +++ b/examples/doAdd.cpp @@ -12,7 +12,7 @@ using namespace std; using namespace neuralpp; -#define NETFILE "adder.net" +#define NETFILE "network.xml" int main() { double a,b; diff --git a/examples/learnAdd.cpp b/examples/learnAdd.cpp index 0fd0936..911eb59 100644 --- a/examples/learnAdd.cpp +++ b/examples/learnAdd.cpp @@ -25,10 +25,10 @@ int main() { // => 2 neurons for the input layer // => 2 neurons for the hidden layer // => 1 neuron for the output layer - // => a learning rate == 0.005 (just get it doing some tests until satisfied) + // => a learning rate == 0.002 (just get it doing some tests until satisfied, but remember to keep its value quite low and ~ 0 to keep the network stable) // => 1000 learning steps (i.e. the network will be ready after 1000 training steps to adjust the synaptical weights // => 0.1 as neural threshold (the threshold above which a neuron activates) - NeuralNet net(2, 2, 1, 0.005, 1000, 0.1); + NeuralNet net(2, 2, 1, 0.002, 2000); // Initialize a training XML as a string in 'xml' NeuralNet::initXML(xml); @@ -45,6 +45,10 @@ int main() { xml += NeuralNet::XMLFromSet(id, "-1,-2;-3"); xml += NeuralNet::XMLFromSet(id, "8,9;17"); xml += NeuralNet::XMLFromSet(id, "10,10;20"); + xml += NeuralNet::XMLFromSet(id, "4,1;5"); + xml += NeuralNet::XMLFromSet(id, "2,6;8"); + xml += NeuralNet::XMLFromSet(id, "2,7;9"); + xml += NeuralNet::XMLFromSet(id, "8,9;17"); NeuralNet::closeXML(xml); // Save the XML string just created to a file @@ -61,7 +65,7 @@ int main() { // Save the trained network to a binary file, that can be reloaded from any // application that is going to use that network - net.save("adder.net"); + net.save("network.xml"); cout << "Network trained in " << (t2-t1) << " seconds. You can use adder.net file now to load this network\n"; return 0; } diff --git a/include/neural++.hpp b/include/neural++.hpp index 2db4837..8a05518 100644 --- a/include/neural++.hpp +++ b/include/neural++.hpp @@ -20,11 +20,8 @@ #include "neural++_exception.hpp" -//! Default rand value: |sin(rand)|, always >= 0 and <= 1 -#define RAND (double) ( (rand() / (RAND_MAX/2)) - 1) - -//! Initial value for the inertial momentum of the synapses -#define BETA0 1.0 +#define RAND (double) ( (rand() / 10.0) / ((double) RAND_MAX) ) +#define BETA0 0.8 /** * @namespace neuralpp @@ -74,13 +71,6 @@ namespace neuralpp { */ void updateWeights(); - /** - * @brief It commits the changes made by updateWeights() to the layer l. - * In-class use only - * @param l Layer to commit the changes - */ - void commitChanges (Layer& l); - /** * @brief Get the error made on the expected result as squared deviance * @param ex Expected value @@ -94,6 +84,52 @@ namespace neuralpp { */ double (*actv_f)(double); + /** + * @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you + * build your network by using setExpected. + * @return The expected output value for a certain training phase + */ + double expected() const; + + /** + * @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you + * build your network by using setExpected. + * @return The expected output value for a certain training phase + */ + std::vector getExpected() const; + + /** + * @brief It sets the value you expect from your network (in case the network has an only neuron in its output layer) + * @param ex Expected output value + */ + void setExpected(double ex); + + /** + * @brief Set the values you expect from your network + * @param ex Expected output values + */ + void setExpected(std::vector ex); + + /** + * @brief It updates through back-propagation the weights of the synapsis and + * computes again the output value for epochs times, calling back + * updateWeights and commitChanges functions + */ + void update(); + + /** + * @brief It links the layers of the network (input, hidden, output) + */ + void link(); + + /** + * @brief Splits a string into a vector of doubles, given a delimitator + * @param delim Delimitator + * @param str String to be splitted + * @return Vector of doubles containing splitted values + */ + static std::vector split (char delim, std::string str); + public: Layer* input; Layer* hidden; @@ -139,12 +175,6 @@ namespace neuralpp { */ double getOutput() const; - /** - * @brief Get the threshold of the neurons in the network - * @return The threshold of the neurons - */ - double getThreshold() const; - /** * @brief It gets the output of the network in case the output layer contains more neurons * @return A vector containing the output values of the network @@ -152,37 +182,10 @@ namespace neuralpp { std::vector getOutputs(); /** - * @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you - * build your network by using setExpected. - * @return The expected output value for a certain training phase + * @brief Get the threshold of the neurons in the network + * @return The threshold of the neurons */ - double expected() const; - - /** - * @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you - * build your network by using setExpected. - * @return The expected output value for a certain training phase - */ - std::vector getExpected() const; - - /** - * @brief It sets the value you expect from your network (in case the network has an only neuron in its output layer) - * @param ex Expected output value - */ - void setExpected(double ex); - - /** - * @brief Set the values you expect from your network - * @param ex Expected output values - */ - void setExpected(std::vector ex); - - /** - * @brief It updates through back-propagation the weights of the synapsis and - * computes again the output value for epochs times, calling back - * updateWeights and commitChanges functions - */ - void update(); + double getThreshold() const; /** * @brief It propagates values through the network. Use this when you want to give @@ -196,12 +199,6 @@ namespace neuralpp { */ void setInput (std::vector v); - /** - * @brief It links the layers of the network (input, hidden, output). Don't use unless - * you exactly know what you're doing, it is already called by the constructor - */ - void link(); - /** * @brief Save a trained neural network to a binary file * @param fname Binary file where you're going to save your network @@ -210,6 +207,30 @@ namespace neuralpp { */ void save (const char* fname) throw(NetworkFileWriteException); + /** + * @brief DEPRECATED. Load a trained neural network from a binary file. + * This function is deprecated and kept for back-compatibility. Use + * the XML format instead to load and neural networks and, respectly, + * the NeuralNetwork(const std::string) constructor or the save(const char*) + * methods. + * @param fname Name of the file to be loaded + * @throws NetworkFileNotFoundException When you're trying to load + * an invalid network file + */ + void loadFromBinary (const std::string fname) throw(NetworkFileNotFoundException); + + /** + * @brief DEPRECATED. Save a trained neural network to a binary file. + * This function is deprecated and kept for back-compatibility. Use + * the XML format instead to load and neural networks and, respectly, + * the NeuralNetwork(const std::string) constructor or the save(const char*) + * methods. + * @param fname Name of the file to be saved with the network information + * @throws NetworkFileWriteException When you try to write the network + * information to an invalid file + */ + void saveToBinary (const char* fname) throw(NetworkFileWriteException); + /** * @brief Train a network using a training set loaded from an XML file. A sample XML file * is available in examples/adder.xml @@ -225,14 +246,6 @@ namespace neuralpp { */ static void initXML (std::string& xml); - /** - * @brief Splits a string into a vector of doubles, given a delimitator - * @param delim Delimitator - * @param str String to be splitted - * @return Vector of doubles containing splitted values - */ - static std::vector split (char delim, std::string str); - /** * @brief Get a training set from a string and copies it to an XML * For example, these strings could be training sets for making sums: @@ -271,13 +284,9 @@ namespace neuralpp { public: /** - * @brief Constructor - * @param i Input neuron - * @param o Output neuron - * @param w Weight for the synapsis - * @param d Delta for the synapsis + * @brief Empty constructor (it does nothing) */ - Synapsis(Neuron* i, Neuron* o, double w, double d); + Synapsis() {} /** * @brief Constructor @@ -424,6 +433,9 @@ namespace neuralpp { */ void setProp (double p); + void setSynIn (size_t n); + void setSynOut (size_t n); + /** * @brief Get the activation value of the neuron * @return Activation value for the neuron diff --git a/src/layer.cpp b/src/layer.cpp index 5c75130..7619236 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -19,8 +19,8 @@ using std::vector; namespace neuralpp { Layer::Layer(size_t sz, double (*a) (double), double th) { for (size_t i = 0; i < sz; i++) { - Neuron n(a); - elements.push_back(n); + Neuron n(a,th); + elements.push_back(n); } threshold = th; @@ -46,16 +46,18 @@ namespace neuralpp { void Layer::link(Layer& l) { srand((unsigned) time(NULL)); + + for (size_t i = 0; i < l.size(); i++) + l.elements[i].setSynOut(size()); + + for (size_t i = 0; i < size(); i++) + elements[i].setSynIn(l.size()); for (size_t i = 0; i < l.size(); i++) { - Neuron *n1 = &(l.elements[i]); - for (size_t j = 0; j < size(); j++) { - Neuron *n2 = &(elements[j]); - Synapsis s(n1, n2, RAND, actv_f); - - n1->push_out(s); - n2->push_in(s); + Synapsis *s = new Synapsis( &(l.elements[i]), &(elements[i]), RAND, actv_f ); + l.elements[i].synOut(j) = *s; + elements[j].synIn(i) = *s; } } } diff --git a/src/neuralnet.cpp b/src/neuralnet.cpp index 79cebd0..7a434d8 100644 --- a/src/neuralnet.cpp +++ b/src/neuralnet.cpp @@ -42,27 +42,11 @@ namespace neuralpp { actv_f = a; threshold = th; - input = new Layer(in_size, __actv, th); - hidden = new Layer(hidden_size, __actv, th); - output = new Layer(out_size, __actv, th); - link(); - } - - /*NeuralNet::NeuralNet(size_t in_size, size_t hidden_size, - size_t out_size, double (*a) (double), - double l, int e, double th) { - - epochs = e; - ref_epochs = epochs; - l_rate = l; - actv_f = a; - threshold = th; - input = new Layer(in_size, a, th); hidden = new Layer(hidden_size, a, th); output = new Layer(out_size, a, th); link(); - }*/ + } double NeuralNet::getOutput() const { return (*output)[0].getActv(); @@ -143,7 +127,9 @@ namespace neuralpp { (-l_rate) * (z-d) * f * y; Dk += ( (z-d) * f * s->getWeight() ); + s->setDelta(out_delta); + (*hidden)[j].synOut(i).setDelta(out_delta); } } @@ -166,13 +152,12 @@ namespace neuralpp { (-l_rate) * d * x; s->setDelta(hidden_delta); + (*input)[j].synOut(i).setDelta(hidden_delta); } } - } - void NeuralNet::commitChanges(Layer& l) { - for (size_t i = 0; i < l.size(); i++) { - Neuron *n = &(l[i]); + for (size_t i = 0; i < output->size(); i++) { + Neuron *n = &((*output)[i]); for (size_t j = 0; j < n->nIn(); j++) { Synapsis *s = &(n->synIn(j)); @@ -181,20 +166,204 @@ namespace neuralpp { s->setDelta(0.0); } } + + for (size_t i = 0; i < hidden->size(); i++) { + Neuron *n = &((*hidden)[i]); + + for (size_t j = 0; j < n->nOut(); j++) { + Synapsis *s = &(n->synOut(j)); + s->setWeight(s->getWeight() + + s->getDelta()); + s->setDelta(0.0); + } + } + + for (size_t i = 0; i < hidden->size(); i++) { + Neuron *n = &((*hidden)[i]); + + for (size_t j = 0; j < n->nIn(); j++) { + Synapsis *s = &(n->synIn(j)); + s->setWeight(s->getWeight() + + s->getDelta()); + s->setDelta(0.0); + } + } + + for (size_t i = 0; i < input->size(); i++) { + Neuron *n = &((*input)[i]); + + for (size_t j = 0; j < n->nOut(); j++) { + Synapsis *s = &(n->synOut(j)); + s->setWeight(s->getWeight() + + s->getDelta()); + s->setDelta(0.0); + } + } } void NeuralNet::update() { epochs = ref_epochs; while ((epochs--) > 0) { - updateWeights(); - commitChanges(*output); - commitChanges(*hidden); propagate(); + updateWeights(); } } void NeuralNet::save (const char *fname) throw(NetworkFileWriteException) { + ofstream out(fname); + stringstream xml(stringstream::in | stringstream::out); + + if (!out) + throw NetworkFileWriteException(); + + xml << "\n" + << "\n" + << "\n\n" + << "\n" + << "\tsize() << "\">\n" + << "\tsize() << "\">\n" + << "\tsize() << "\">\n\n"; + + for (unsigned int i = 0; i < hidden->size(); i++) { + int nin = (*hidden)[i].nIn(); + + for (int j = 0; j < nin; j++) + xml << "\t\n"; + } + + for (unsigned int i = 0; i < output->size(); i++) { + int nin = (*output)[i].nIn(); + + for (int j = 0; j < nin; j++) + xml << "\t\n"; + } + + xml << "\n"; + out << xml.str(); + } + + NeuralNet::NeuralNet(const string fname) throw(NetworkFileNotFoundException) { + unsigned int in_size = 0, hid_size = 0, out_size = 0; + vector< vector > in_hid_synapses, hid_out_synapses; + + CMarkup xml; + xml.Load(fname.c_str()); + + if (!xml.IsWellFormed()) { + throw InvalidXMLException(); + return; + } + + if (xml.FindElem("network")) { + if (xml.GetAttrib("epochs").empty()) + throw InvalidXMLException(); + + if (xml.GetAttrib("learning_rate").empty()) + throw InvalidXMLException(); + + epochs = atoi(xml.GetAttrib("epochs").c_str()); + l_rate = atof(xml.GetAttrib("learning_rate").c_str()); + threshold = 0.0; + + if (!xml.GetAttrib("threshold").empty()) + threshold = atof(xml.GetAttrib("threshold").c_str()); + + while (xml.FindChildElem("layer")) { + if (xml.GetChildAttrib("class").empty()) + throw InvalidXMLException(); + + if (xml.GetChildAttrib("size").empty()) + throw InvalidXMLException(); + + if (!xml.GetChildAttrib("class").compare("input")) + in_size = atoi(xml.GetChildAttrib("size").c_str()); + else if (!xml.GetChildAttrib("class").compare("hidden")) + hid_size = atoi(xml.GetChildAttrib("size").c_str()); + else if (!xml.GetChildAttrib("class").compare("output")) + out_size = atoi(xml.GetChildAttrib("size").c_str()); + else + throw InvalidXMLException(); + } + + if (in_size && hid_size && out_size) { + in_hid_synapses = vector< vector >(in_size); + + for (unsigned int i=0; i < in_size; i++) + in_hid_synapses[i] = vector(hid_size); + + hid_out_synapses = vector< vector >(hid_size); + + for (unsigned int i=0; i < hid_size; i++) + hid_out_synapses[i] = vector(out_size); + } + + while (xml.FindChildElem("synapsis")) { + if (!(in_size && hid_size && out_size)) + throw InvalidXMLException(); + + if (xml.GetChildAttrib("class").empty()) + throw InvalidXMLException(); + + if (xml.GetChildAttrib("input").empty()) + throw InvalidXMLException(); + + if (xml.GetChildAttrib("output").empty()) + throw InvalidXMLException(); + + if (xml.GetChildAttrib("weight").empty()) + throw InvalidXMLException(); + + unsigned int in = atoi(xml.GetChildAttrib("input").c_str()); + unsigned int out = atoi(xml.GetChildAttrib("output").c_str()); + + if (xml.GetChildAttrib("class") == "inhid") { + if (in >= in_size || out >= hid_size) + throw InvalidXMLException(); + + in_hid_synapses[in][out] = atof(xml.GetChildAttrib("weight").c_str()); + } + + if (xml.GetChildAttrib("class") == "hidout") { + if (in >= hid_size || out >= out_size) + throw InvalidXMLException(); + + hid_out_synapses[in][out] = atof(xml.GetChildAttrib("weight").c_str()); + } + } + } + + *this = NeuralNet(in_size, hid_size, out_size, l_rate, epochs, threshold); + + hidden->link(*input); + output->link(*hidden); + + // Restore synapses + for (unsigned int i = 0; i < input->size(); i++) { + for (unsigned int j = 0; j < hidden->size(); j++) + (*input)[i].synOut(j).setWeight( in_hid_synapses[i][j] ); + } + + for (unsigned int i = 0; i < output->size(); i++) { + for (unsigned int j = 0; j < hidden->size(); j++) + (*output)[i].synIn(j).setWeight( (hid_out_synapses[j][i]) ); + } + + for (unsigned int i = 0; i < hidden->size(); i++) { + for (unsigned int j = 0; j < input->size(); j++) + (*hidden)[i].synIn(j).setWeight( (in_hid_synapses[j][i]) ); + } + + for (unsigned int i = 0; i < hidden->size(); i++) { + for (unsigned int j = 0; j < output->size(); j++) + (*hidden)[i].synOut(j).setWeight( hid_out_synapses[i][j] ); + } + } + + void NeuralNet::saveToBinary (const char *fname) throw(NetworkFileWriteException) { struct netrecord record; ofstream out(fname); @@ -308,7 +477,7 @@ namespace neuralpp { out.close(); } - NeuralNet::NeuralNet(const string fname) throw(NetworkFileNotFoundException) { + void NeuralNet::loadFromBinary (const string fname) throw(NetworkFileNotFoundException) { struct netrecord record; ifstream in(fname.c_str()); @@ -479,7 +648,6 @@ namespace neuralpp { setInput(input); setExpected(output); - propagate(); update(); } } diff --git a/src/neuron.cpp b/src/neuron.cpp index 2424380..a03ead3 100644 --- a/src/neuron.cpp +++ b/src/neuron.cpp @@ -24,8 +24,8 @@ namespace neuralpp { Neuron::Neuron(vector < Synapsis > i, vector < Synapsis > o, double (*a) (double), double th) { - in = i; - out = o; + in.assign(i.begin(), i.end()); + out.assign(o.begin(), o.end()); actv_f = a; threshold = th; } @@ -38,11 +38,11 @@ namespace neuralpp { return out[i]; } - void Neuron::push_in(Synapsis s) { + void Neuron::push_in(Synapsis s) { in.push_back(s); } - void Neuron::push_out(Synapsis s) { + void Neuron::push_out(Synapsis s) { out.push_back(s); } @@ -51,10 +51,17 @@ namespace neuralpp { } void Neuron::setActv(double val) { - //actv_val = actv_f(val); actv_val = val; } + void Neuron::setSynIn (size_t n) { + in = vector(n); + } + + void Neuron::setSynOut (size_t n) { + out = vector(n); + } + size_t Neuron::nIn() { return in.size(); } diff --git a/src/synapsis.cpp b/src/synapsis.cpp index d77a5f0..ed40025 100644 --- a/src/synapsis.cpp +++ b/src/synapsis.cpp @@ -15,14 +15,6 @@ #include "neural++.hpp" namespace neuralpp { - Synapsis::Synapsis(Neuron * i, Neuron * o, double w, double d) { - in = i; - out = o; - weight = w; - delta = d; - prev_delta = 0; - } - Synapsis::Synapsis(Neuron * i, Neuron * o, double (*a) (double)) { srand((unsigned) time(NULL)); @@ -75,9 +67,6 @@ namespace neuralpp { } void Synapsis::setDelta(double d) throw(InvalidSynapticalWeightException) { - if (d > 1.0) - throw InvalidSynapticalWeightException(); - prev_delta = delta; delta = d; }