Release 1.0 is almost out

ChangeLog

@@ -1,3 +1,25 @@
+--- Release 1.0 ---
+
+2009-09-02 BlackLight <blacklight@autistici.org>
+
+	* all: Most of the code has been rewritten. Now a trained network should
+	be saved, via save() method, to an XML file, and loaded from that using
+	the constructor. This way is much cleaner, and easily fixable and
+	customizable too. The old approach through binary files has been kept yet
+	through saveToBinary() and loadFromBinary() methods, for
+	back-compatibility purposes, though it's strongly deprecated. Moreover,
+	the XML formats (for training the network and for loading a trained
+	network's information) are fully compatible with the ones used by
+	NeuralPerl module, mostly a Perl version of this library, so you can use
+	the same XML files to train and load a neural network using C++ and Perl.
+	Moreover, a great step forward in the fixing of the dirty old
+	vulnerability (synaptical random weights overflow) has been made, keeping
+	the values of the weights very low and ~ 0. I've made tons of tests and
+	the network never overflowed. Anyway, if you should experience an
+	overflow (synaptical weights go > 1), just write me. I've also done a
+	deep re-design of library's classes, using as public methods only the
+	stuff you *REALLY* need to be public.
+
 --- Release 0.4 ---
 
 2009-08-16 BlackLight <blacklight@autistici.org>

VERSION

@@ -1 +1 @@
-0.4
+1.0

examples/Makefile

@@ -2,10 +2,9 @@ all:
 	g++ -Wall -o learnAdd learnAdd.cpp -lneural++
 	g++ -Wall -o doAdd doAdd.cpp -lneural++
 	g++ -Wall -o adderFromScratch adderFromScratch.cpp -lneural++
+	g++ -Wall -o Add Add.cpp -lneural++
 
 clean:
 	rm learnAdd
 	rm doAdd
 	rm adderFromScratch
-	rm adder.net
-	rm adder.xml

examples/doAdd.cpp

@@ -12,7 +12,7 @@
 using namespace std;
 using namespace neuralpp;
 
-#define	NETFILE	"adder.net"
+#define	NETFILE	"network.xml"
 
 int main()  {
 	double a,b;

examples/learnAdd.cpp

@@ -25,10 +25,10 @@ int main()  {
 	// => 2 neurons for the input layer
 	// => 2 neurons for the hidden layer
 	// => 1 neuron  for the output layer
-	// => a learning rate == 0.005 (just get it doing some tests until satisfied)
+	// => a learning rate == 0.002 (just get it doing some tests until satisfied, but remember to keep its value quite low and ~ 0 to keep the network stable)
 	// => 1000 learning steps (i.e. the network will be ready after 1000 training steps to adjust the synaptical weights
 	// => 0.1 as neural threshold (the threshold above which a neuron activates)
-	NeuralNet net(2, 2, 1, 0.005, 1000, 0.1);
+	NeuralNet net(2, 2, 1, 0.002, 2000);
 
 	// Initialize a training XML as a string in 'xml'
 	NeuralNet::initXML(xml);
@@ -45,6 +45,10 @@ int main()  {
 	xml += NeuralNet::XMLFromSet(id, "-1,-2;-3");
 	xml += NeuralNet::XMLFromSet(id, "8,9;17");
 	xml += NeuralNet::XMLFromSet(id, "10,10;20");
+	xml += NeuralNet::XMLFromSet(id, "4,1;5");
+	xml += NeuralNet::XMLFromSet(id, "2,6;8");
+	xml += NeuralNet::XMLFromSet(id, "2,7;9");
+	xml += NeuralNet::XMLFromSet(id, "8,9;17");
 	NeuralNet::closeXML(xml);
 
 	// Save the XML string just created to a file
@@ -61,7 +65,7 @@ int main()  {
 
 	// Save the trained network to a binary file, that can be reloaded from any
 	// application that is going to use that network
-	net.save("adder.net");
+	net.save("network.xml");
 	cout << "Network trained in " << (t2-t1) << " seconds. You can use adder.net file now to load this network\n";
 	return 0;
 }

include/neural++.hpp

@@ -20,11 +20,8 @@
 
 #include "neural++_exception.hpp"
 
-//! Default rand value: |sin(rand)|, always >= 0 and <= 1
+#define 	RAND 	(double) ( (rand() / 10.0) / ((double) RAND_MAX) )
-#define 	RAND 	(double) ( (rand() / (RAND_MAX/2)) - 1)
+#define 	BETA0 	0.8
-
-//! Initial value for the inertial momentum of the synapses
-#define 	BETA0 	1.0
 
 /**
  * @namespace neuralpp
@@ -74,13 +71,6 @@ namespace neuralpp  {
 		 */
 		void updateWeights();
 
-		/**
-		 * @brief It commits the changes made by updateWeights() to the layer l.
-		 *   In-class use only
-		 * @param l Layer to commit the changes
-		 */
-		void commitChanges (Layer& l);
-
 		/**
 		 * @brief Get the error made on the expected result as squared deviance
 		 * @param ex Expected value
@@ -94,6 +84,52 @@ namespace neuralpp  {
 		 */
 		double (*actv_f)(double);
 
+		/**
+		 * @brief Get the expected value (in case you have an only neuron in output layer).  Of course you should specify this when you
+		 * build your network by using setExpected.
+		 * @return The expected output value for a certain training phase
+		 */
+		double expected() const;
+
+		/**
+		 * @brief Get the expected value (in case you have an only neuron in output layer).  Of course you should specify this when you
+		 * build your network by using setExpected.
+		 * @return The expected output value for a certain training phase
+		 */
+		std::vector<double> getExpected() const;
+
+		/**
+		 * @brief It sets the value you expect from your network (in case the network has an only neuron in its output layer)
+		 * @param ex Expected output value
+		 */
+		void setExpected(double ex);
+
+		/**
+		 * @brief Set the values you expect from your network
+		 * @param ex Expected output values
+		 */
+		void setExpected(std::vector<double> ex);
+
+		/**
+		 * @brief It updates through back-propagation the weights of the synapsis and
+		 * computes again the output value for <i>epochs</i> times, calling back
+		 * updateWeights and commitChanges functions
+		 */
+		void update();
+
+		/**
+		 * @brief It links the layers of the network (input, hidden, output)
+		 */
+		void link();
+		
+		/**
+		 * @brief Splits a string into a vector of doubles, given a delimitator
+		 * @param delim Delimitator
+		 * @param str String to be splitted
+		 * @return Vector of doubles containing splitted values
+		 */
+		static std::vector<double> split (char delim, std::string str);
+
 	public:
 		Layer* input;
 		Layer* hidden;
@@ -139,12 +175,6 @@ namespace neuralpp  {
 		 */
 		double getOutput() const;
 
-		/**
-		 * @brief Get the threshold of the neurons in the network
-		 * @return The threshold of the neurons
-		 */
-		double getThreshold() const;
-
 		/**
 		 * @brief It gets the output of the network in case the output layer contains more neurons
 		 * @return A vector containing the output values of the network
@@ -152,37 +182,10 @@ namespace neuralpp  {
 		std::vector<double> getOutputs();
 
 		/**
-		 * @brief Get the expected value (in case you have an only neuron in output layer).  Of course you should specify this when you
+		 * @brief Get the threshold of the neurons in the network
-		 * build your network by using setExpected.
+		 * @return The threshold of the neurons
-		 * @return The expected output value for a certain training phase
 		 */
-		double expected() const;
+		double getThreshold() const;
-
-		/**
-		 * @brief Get the expected value (in case you have an only neuron in output layer).  Of course you should specify this when you
-		 * build your network by using setExpected.
-		 * @return The expected output value for a certain training phase
-		 */
-		std::vector<double> getExpected() const;
-
-		/**
-		 * @brief It sets the value you expect from your network (in case the network has an only neuron in its output layer)
-		 * @param ex Expected output value
-		 */
-		void setExpected(double ex);
-
-		/**
-		 * @brief Set the values you expect from your network
-		 * @param ex Expected output values
-		 */
-		void setExpected(std::vector<double> ex);
-
-		/**
-		 * @brief It updates through back-propagation the weights of the synapsis and
-		 * computes again the output value for <i>epochs</i> times, calling back
-		 * updateWeights and commitChanges functions
-		 */
-		void update();
 
 		/**
 		 * @brief It propagates values through the network. Use this when you want to give
@@ -196,12 +199,6 @@ namespace neuralpp  {
 		 */
 		void setInput (std::vector<double> v);
 
-		/**
-		 * @brief It links the layers of the network (input, hidden, output). Don't use unless
-		 * you exactly know what you're doing, it is already called by the constructor
-		 */
-		void link();
-
 		/**
 		 * @brief Save a trained neural network to a binary file
 		 * @param fname Binary file where you're going to save your network
@@ -210,6 +207,30 @@ namespace neuralpp  {
 		 */
 		void save (const char* fname) throw(NetworkFileWriteException);
 
+		/**
+		 * @brief DEPRECATED. Load a trained neural network from a binary file.
+		 *  This function is deprecated and kept for back-compatibility. Use
+		 *  the XML format instead to load and neural networks and, respectly,
+		 *  the NeuralNetwork(const std::string) constructor or the save(const char*)
+		 *  methods.
+		 * @param fname Name of the file to be loaded
+		 * @throws NetworkFileNotFoundException When you're trying to load
+		 *  an invalid network file
+		 */
+		void loadFromBinary (const std::string fname) throw(NetworkFileNotFoundException);
+
+		/**
+		 * @brief DEPRECATED. Save a trained neural network to a binary file.
+		 *  This function is deprecated and kept for back-compatibility. Use
+		 *  the XML format instead to load and neural networks and, respectly,
+		 *  the NeuralNetwork(const std::string) constructor or the save(const char*)
+		 *  methods.
+		 * @param fname Name of the file to be saved with the network information
+		 * @throws NetworkFileWriteException When you try to write the network
+		 *  information to an invalid file
+		 */
+		void saveToBinary (const char* fname) throw(NetworkFileWriteException);
+
 		/**
 		 * @brief Train a network using a training set loaded from an XML file. A sample XML file
 		 *   is available in examples/adder.xml
@@ -225,14 +246,6 @@ namespace neuralpp  {
 		 */
 		static void initXML (std::string& xml);
 
-		/**
-		 * @brief Splits a string into a vector of doubles, given a delimitator
-		 * @param delim Delimitator
-		 * @param str String to be splitted
-		 * @return Vector of doubles containing splitted values
-		 */
-		static std::vector<double> split (char delim, std::string str);
-
 		/**
 		 * @brief Get a training set from a string and copies it to an XML
 		 *   For example, these strings could be training sets for making sums:
@@ -271,13 +284,9 @@ namespace neuralpp  {
 
 	public:
 		/**
-		 * @brief Constructor
+		 * @brief Empty constructor (it does nothing)
-		 * @param i Input neuron
-		 * @param o Output neuron
-		 * @param w Weight for the synapsis
-		 * @param d Delta for the synapsis
 		 */
-		Synapsis(Neuron* i, Neuron* o, double w, double d);
+		Synapsis()  {}
 
 		/**
 		 * @brief Constructor
@@ -424,6 +433,9 @@ namespace neuralpp  {
 		 */
 		void setProp (double p);
 
+		void setSynIn  (size_t n);
+		void setSynOut (size_t n);
+
 		/**
 		 * @brief Get the activation value of the neuron
 		 * @return Activation value for the neuron

src/layer.cpp

@@ -19,7 +19,7 @@ using std::vector;
 namespace neuralpp {
 	Layer::Layer(size_t sz, double (*a) (double), double th) {
 		for (size_t i = 0; i < sz; i++) {
-			Neuron n(a);
+			Neuron n(a,th);
 			elements.push_back(n);
 		}
 		
@@ -47,15 +47,17 @@ namespace neuralpp {
 	void Layer::link(Layer& l) {
 		srand((unsigned) time(NULL));
 		
+		for (size_t i = 0; i < l.size(); i++)
+			l.elements[i].setSynOut(size());
+		
+		for (size_t i = 0; i < size(); i++)
+			elements[i].setSynIn(l.size());
+
 		for (size_t i = 0; i < l.size(); i++) {
-			Neuron *n1 = &(l.elements[i]);
-
 			for (size_t j = 0; j < size(); j++) {
-				Neuron *n2 = &(elements[j]);
+				Synapsis *s = new Synapsis( &(l.elements[i]), &(elements[i]), RAND, actv_f );
-				Synapsis s(n1, n2, RAND, actv_f);
+				l.elements[i].synOut(j) = *s;
-
+				elements[j].synIn(i) = *s;
-				n1->push_out(s);
-				n2->push_in(s);
 			}
 		}
 	}

src/neuralnet.cpp

@@ -42,27 +42,11 @@ namespace neuralpp {
 		actv_f = a;
 		threshold = th;
 
-		input = new Layer(in_size, __actv, th);
-		hidden = new Layer(hidden_size, __actv, th);
-		output = new Layer(out_size, __actv, th);
-		link();
-	}
-
-	/*NeuralNet::NeuralNet(size_t in_size, size_t hidden_size,
-			     size_t out_size, double (*a) (double),
-			     double l, int e, double th) {
-
-		epochs = e;
-		ref_epochs = epochs;
-		l_rate = l;
-		actv_f = a;
-		threshold = th;
-
 		input = new Layer(in_size, a, th);
 		hidden = new Layer(hidden_size, a, th);
 		output = new Layer(out_size, a, th);
 		link();
-	}*/
+	}
 
 	double NeuralNet::getOutput() const  {
 		return (*output)[0].getActv();
@@ -143,7 +127,9 @@ namespace neuralpp {
 					    (-l_rate) * (z-d) * f * y;
 
 				Dk += ( (z-d) * f * s->getWeight() );
+
 				s->setDelta(out_delta);
+				(*hidden)[j].synOut(i).setDelta(out_delta);
 			}
 		}
 
@@ -166,13 +152,12 @@ namespace neuralpp {
 						(-l_rate) * d * x;
 
 				s->setDelta(hidden_delta);
-			}
+				(*input)[j].synOut(i).setDelta(hidden_delta);
 			}
 		}
 
-	void NeuralNet::commitChanges(Layer& l) {
+		for (size_t i = 0; i < output->size(); i++) {
-		for (size_t i = 0; i < l.size(); i++) {
+			Neuron *n = &((*output)[i]);
-			Neuron *n = &(l[i]);
 
 			for (size_t j = 0; j < n->nIn(); j++) {
 				Synapsis *s = &(n->synIn(j));
@@ -181,20 +166,204 @@ namespace neuralpp {
 				s->setDelta(0.0);
 			}
 		}
+
+		for (size_t i = 0; i < hidden->size(); i++) {
+			Neuron *n = &((*hidden)[i]);
+
+			for (size_t j = 0; j < n->nOut(); j++) {
+				Synapsis *s = &(n->synOut(j));
+				s->setWeight(s->getWeight() +
+					     s->getDelta());
+				s->setDelta(0.0);
+			}
+		}
+
+		for (size_t i = 0; i < hidden->size(); i++) {
+			Neuron *n = &((*hidden)[i]);
+
+			for (size_t j = 0; j < n->nIn(); j++) {
+				Synapsis *s = &(n->synIn(j));
+				s->setWeight(s->getWeight() +
+					     s->getDelta());
+				s->setDelta(0.0);
+			}
+		}
+
+		for (size_t i = 0; i < input->size(); i++) {
+			Neuron *n = &((*input)[i]);
+
+			for (size_t j = 0; j < n->nOut(); j++) {
+				Synapsis *s = &(n->synOut(j));
+				s->setWeight(s->getWeight() +
+					     s->getDelta());
+				s->setDelta(0.0);
+			}
+		}
 	}
 
 	void NeuralNet::update() {
 		epochs = ref_epochs;
 
 		while ((epochs--) > 0) {
-			updateWeights();
-			commitChanges(*output);
-			commitChanges(*hidden);
 			propagate();
+			updateWeights();
 		}
 	}
 
 	void NeuralNet::save (const char *fname) throw(NetworkFileWriteException)  {
+		ofstream out(fname);
+		stringstream xml(stringstream::in | stringstream::out);
+
+		if (!out)
+			throw NetworkFileWriteException();
+
+		xml << "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
+			<< "<!DOCTYPE NETWORK SYSTEM \"http://blacklight.gotdns.org/prog/neuralpp/network.dtd\">\n"
+			<< "<!-- Automatically generated by BlackLight's Neural++ module -->\n\n"
+			<< "<network name=\"Put here the name for this neural network\" epochs=\"" << ref_epochs << "\" "
+			<< "learning_rate=\"" << l_rate << "\" threshold=\"" << threshold << "\">\n"
+			<< "\t<layer class=\"input\"  size=\"" << input->size()  << "\"></layer>\n"
+			<< "\t<layer class=\"hidden\" size=\"" << hidden->size() << "\"></layer>\n"
+			<< "\t<layer class=\"output\" size=\"" << output->size() << "\"></layer>\n\n";
+
+		for (unsigned int i = 0; i < hidden->size(); i++) {
+			int nin = (*hidden)[i].nIn();
+
+			for (int j = 0; j < nin; j++)
+				xml << "\t<synapsis class=\"inhid\" input=\"" << j << "\" output=\"" << i << "\" "
+					<< "weight=\"" << (*hidden)[i].synIn(j).getWeight() << "\"></synapsis>\n";
+		}
+
+		for (unsigned int i = 0; i < output->size(); i++) {
+			int nin = (*output)[i].nIn();
+
+			for (int j = 0; j < nin; j++)
+				xml << "\t<synapsis class=\"hidout\" input=\"" << j << "\" output=\"" << i << "\" "
+					<< "weight=\"" << (*output)[i].synIn(j).getWeight() << "\"></synapsis>\n";
+		}
+
+		xml << "</network>\n";
+		out << xml.str();
+	}
+
+	NeuralNet::NeuralNet(const string fname) throw(NetworkFileNotFoundException)  {
+		unsigned int in_size = 0, hid_size = 0, out_size = 0;
+		vector< vector<double> > in_hid_synapses, hid_out_synapses;
+
+		CMarkup xml;
+		xml.Load(fname.c_str());
+
+		if (!xml.IsWellFormed()) {
+			throw InvalidXMLException();
+			return;
+		}
+
+		if (xml.FindElem("network")) {
+			if (xml.GetAttrib("epochs").empty())
+				throw InvalidXMLException();
+			
+			if (xml.GetAttrib("learning_rate").empty())
+				throw InvalidXMLException();
+
+			epochs = atoi(xml.GetAttrib("epochs").c_str());
+			l_rate = atof(xml.GetAttrib("learning_rate").c_str());
+			threshold = 0.0;
+			
+			if (!xml.GetAttrib("threshold").empty())
+				threshold = atof(xml.GetAttrib("threshold").c_str());
+
+			while (xml.FindChildElem("layer"))  {
+				if (xml.GetChildAttrib("class").empty())
+					throw InvalidXMLException();
+			
+				if (xml.GetChildAttrib("size").empty())
+					throw InvalidXMLException();
+
+				if (!xml.GetChildAttrib("class").compare("input"))
+					in_size = atoi(xml.GetChildAttrib("size").c_str());
+				else if (!xml.GetChildAttrib("class").compare("hidden"))
+					hid_size = atoi(xml.GetChildAttrib("size").c_str());
+				else if (!xml.GetChildAttrib("class").compare("output"))
+					out_size = atoi(xml.GetChildAttrib("size").c_str());
+				else
+					throw InvalidXMLException();
+			}
+
+			if (in_size && hid_size && out_size)  {
+				in_hid_synapses = vector< vector<double> >(in_size);
+
+				for (unsigned int i=0; i < in_size; i++)
+					in_hid_synapses[i] = vector<double>(hid_size);
+
+				hid_out_synapses = vector< vector<double> >(hid_size);
+
+				for (unsigned int i=0; i < hid_size; i++)
+					hid_out_synapses[i] = vector<double>(out_size);
+			}
+			
+			while (xml.FindChildElem("synapsis"))  {
+				if (!(in_size && hid_size && out_size))
+					throw InvalidXMLException();
+
+				if (xml.GetChildAttrib("class").empty())
+					throw InvalidXMLException();
+			
+				if (xml.GetChildAttrib("input").empty())
+					throw InvalidXMLException();
+				
+				if (xml.GetChildAttrib("output").empty())
+					throw InvalidXMLException();
+				
+				if (xml.GetChildAttrib("weight").empty())
+					throw InvalidXMLException();
+
+				unsigned int in  = atoi(xml.GetChildAttrib("input").c_str());
+				unsigned int out = atoi(xml.GetChildAttrib("output").c_str());
+
+				if (xml.GetChildAttrib("class") == "inhid")  {
+					if (in >= in_size || out >= hid_size)
+						throw InvalidXMLException();
+
+					in_hid_synapses[in][out] = atof(xml.GetChildAttrib("weight").c_str());
+				}
+
+				if (xml.GetChildAttrib("class") == "hidout")  {
+					if (in >= hid_size || out >= out_size)
+						throw InvalidXMLException();
+
+					hid_out_synapses[in][out] = atof(xml.GetChildAttrib("weight").c_str());
+				}
+			}
+		}
+
+		*this = NeuralNet(in_size, hid_size, out_size, l_rate, epochs, threshold);
+
+		hidden->link(*input);
+		output->link(*hidden);
+
+		// Restore synapses
+		for (unsigned int i = 0; i < input->size(); i++) {
+			for (unsigned int j = 0; j < hidden->size(); j++)
+				(*input)[i].synOut(j).setWeight( in_hid_synapses[i][j] );
+		}
+
+		for (unsigned int i = 0; i < output->size(); i++) {
+			for (unsigned int j = 0; j < hidden->size(); j++)
+				(*output)[i].synIn(j).setWeight( (hid_out_synapses[j][i]) );
+		}
+
+		for (unsigned int i = 0; i < hidden->size(); i++) {
+			for (unsigned int j = 0; j < input->size(); j++)
+				(*hidden)[i].synIn(j).setWeight( (in_hid_synapses[j][i]) );
+		}
+
+		for (unsigned int i = 0; i < hidden->size(); i++) {
+			for (unsigned int j = 0; j < output->size(); j++)
+				(*hidden)[i].synOut(j).setWeight( hid_out_synapses[i][j] );
+		}
+	}
+
+	void NeuralNet::saveToBinary (const char *fname) throw(NetworkFileWriteException)  {
 		struct netrecord record;
 		ofstream out(fname);
 
@@ -308,7 +477,7 @@ namespace neuralpp {
 		out.close();
 	}
 
-	NeuralNet::NeuralNet(const string fname) throw(NetworkFileNotFoundException) {
+	void NeuralNet::loadFromBinary (const string fname) throw(NetworkFileNotFoundException) {
 		struct netrecord record;
 		ifstream in(fname.c_str());
 
@@ -479,7 +648,6 @@ namespace neuralpp {
 
 				setInput(input);
 				setExpected(output);
-				propagate();
 				update();
 			}
 		}

src/neuron.cpp

@@ -24,8 +24,8 @@ namespace neuralpp {
 	Neuron::Neuron(vector < Synapsis > i, vector < Synapsis > o,
 			 double (*a) (double), double th) {
 
-		in = i;
+		in.assign(i.begin(), i.end());
-		out = o;
+		out.assign(o.begin(), o.end());
 		actv_f = a;
 		threshold = th;
 	}
@@ -51,10 +51,17 @@ namespace neuralpp {
 	}
 
 	void Neuron::setActv(double val) {
-		//actv_val = actv_f(val);
 		actv_val = val;
 	}
 
+	void Neuron::setSynIn (size_t n)  {
+		in = vector<Synapsis>(n);
+	}
+
+	void Neuron::setSynOut (size_t n)  {
+		out = vector<Synapsis>(n);
+	}
+
 	size_t Neuron::nIn() {
 		return in.size();
 	}

src/synapsis.cpp

@@ -15,14 +15,6 @@
 #include "neural++.hpp"
 
 namespace neuralpp {
-	Synapsis::Synapsis(Neuron * i, Neuron * o, double w, double d) {
-		in = i;
-		out = o;
-		weight = w;
-		delta = d;
-		prev_delta = 0;
-	}
-	
 	Synapsis::Synapsis(Neuron * i, Neuron * o, double (*a) (double)) {
 		srand((unsigned) time(NULL));
 
@@ -75,9 +67,6 @@ namespace neuralpp {
 	}
 
 	void Synapsis::setDelta(double d) throw(InvalidSynapticalWeightException)  {
-		if (d > 1.0)
-			throw InvalidSynapticalWeightException();
-
 		prev_delta = delta;
 		delta = d;
 	}