blacklight
/
neuralpp
mirror of https://github.com/BlackLight/neuralpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Release 1.0 is almost out

This commit is contained in:
blacklight 2009-09-02 21:16:47 +02:00
parent def2cc98e0
commit 89b0ad2f8a
10 changed files with 329 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ChangeLog
View File

@ -1,3 +1,25 @@
--- Release 1.0 ---
2009-09-02 BlackLight <blacklight@autistici.org>
* all: Most of the code has been rewritten. Now a trained network should
be saved, via save() method, to an XML file, and loaded from that using
the constructor. This way is much cleaner, and easily fixable and
customizable too. The old approach through binary files has been kept yet
through saveToBinary() and loadFromBinary() methods, for
back-compatibility purposes, though it's strongly deprecated. Moreover,
the XML formats (for training the network and for loading a trained
network's information) are fully compatible with the ones used by
NeuralPerl module, mostly a Perl version of this library, so you can use
the same XML files to train and load a neural network using C++ and Perl.
Moreover, a great step forward in the fixing of the dirty old
vulnerability (synaptical random weights overflow) has been made, keeping
the values of the weights very low and ~ 0. I've made tons of tests and
the network never overflowed. Anyway, if you should experience an
overflow (synaptical weights go > 1), just write me. I've also done a
deep re-design of library's classes, using as public methods only the
stuff you *REALLY* need to be public.
--- Release 0.4 ---
2009-08-16 BlackLight <blacklight@autistici.org>

2
VERSION
View File

@ -1 +1 @@
0.4
1.0

3
examples/Makefile
View File

@ -2,10 +2,9 @@ all:
g++ -Wall -o learnAdd learnAdd.cpp -lneural++
g++ -Wall -o doAdd doAdd.cpp -lneural++
g++ -Wall -o adderFromScratch adderFromScratch.cpp -lneural++
g++ -Wall -o Add Add.cpp -lneural++
clean:
rm learnAdd
rm doAdd
rm adderFromScratch
rm adder.net
rm adder.xml

2
examples/doAdd.cpp
View File

@ -12,7 +12,7 @@
using namespace std;
using namespace neuralpp;
#define NETFILE "adder.net"
#define NETFILE "network.xml"
int main() {
double a,b;

10
examples/learnAdd.cpp
View File

@ -25,10 +25,10 @@ int main() {
// => 2 neurons for the input layer
// => 2 neurons for the hidden layer
// => 1 neuron for the output layer
// => a learning rate == 0.005 (just get it doing some tests until satisfied)
// => a learning rate == 0.002 (just get it doing some tests until satisfied, but remember to keep its value quite low and ~ 0 to keep the network stable)
// => 1000 learning steps (i.e. the network will be ready after 1000 training steps to adjust the synaptical weights
// => 0.1 as neural threshold (the threshold above which a neuron activates)
NeuralNet net(2, 2, 1, 0.005, 1000, 0.1);
NeuralNet net(2, 2, 1, 0.002, 2000);
// Initialize a training XML as a string in 'xml'
NeuralNet::initXML(xml);
@ -45,6 +45,10 @@ int main() {
xml += NeuralNet::XMLFromSet(id, "-1,-2;-3");
xml += NeuralNet::XMLFromSet(id, "8,9;17");
xml += NeuralNet::XMLFromSet(id, "10,10;20");
xml += NeuralNet::XMLFromSet(id, "4,1;5");
xml += NeuralNet::XMLFromSet(id, "2,6;8");
xml += NeuralNet::XMLFromSet(id, "2,7;9");
xml += NeuralNet::XMLFromSet(id, "8,9;17");
NeuralNet::closeXML(xml);
// Save the XML string just created to a file
@ -61,7 +65,7 @@ int main() {
// Save the trained network to a binary file, that can be reloaded from any
// application that is going to use that network
net.save("adder.net");
net.save("network.xml");
cout << "Network trained in " << (t2-t1) << " seconds. You can use adder.net file now to load this network\n";
return 0;
}

148
include/neural++.hpp
View File

@ -20,11 +20,8 @@
#include "neural++_exception.hpp"
//! Default rand value: |sin(rand)|, always >= 0 and <= 1
#define RAND (double) ( (rand() / (RAND_MAX/2)) - 1)
//! Initial value for the inertial momentum of the synapses
#define BETA0 1.0
#define RAND (double) ( (rand() / 10.0) / ((double) RAND_MAX) )
#define BETA0 0.8
/**
* @namespace neuralpp
@ -74,13 +71,6 @@ namespace neuralpp {
*/
void updateWeights();
/**
* @brief It commits the changes made by updateWeights() to the layer l.
* In-class use only
* @param l Layer to commit the changes
*/
void commitChanges (Layer& l);
/**
* @brief Get the error made on the expected result as squared deviance
* @param ex Expected value
@ -94,6 +84,52 @@ namespace neuralpp {
*/
double (*actv_f)(double);
/**
* @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you
* build your network by using setExpected.
* @return The expected output value for a certain training phase
*/
double expected() const;
/**
* @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you
* build your network by using setExpected.
* @return The expected output value for a certain training phase
*/
std::vector<double> getExpected() const;
/**
* @brief It sets the value you expect from your network (in case the network has an only neuron in its output layer)
* @param ex Expected output value
*/
void setExpected(double ex);
/**
* @brief Set the values you expect from your network
* @param ex Expected output values
*/
void setExpected(std::vector<double> ex);
/**
* @brief It updates through back-propagation the weights of the synapsis and
* computes again the output value for <i>epochs</i> times, calling back
* updateWeights and commitChanges functions
*/
void update();
/**
* @brief It links the layers of the network (input, hidden, output)
*/
void link();
/**
* @brief Splits a string into a vector of doubles, given a delimitator
* @param delim Delimitator
* @param str String to be splitted
* @return Vector of doubles containing splitted values
*/
static std::vector<double> split (char delim, std::string str);
public:
Layer* input;
Layer* hidden;
@ -139,12 +175,6 @@ namespace neuralpp {
*/
double getOutput() const;
/**
* @brief Get the threshold of the neurons in the network
* @return The threshold of the neurons
*/
double getThreshold() const;
/**
* @brief It gets the output of the network in case the output layer contains more neurons
* @return A vector containing the output values of the network
@ -152,37 +182,10 @@ namespace neuralpp {
std::vector<double> getOutputs();
/**
* @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you
* build your network by using setExpected.
* @return The expected output value for a certain training phase
* @brief Get the threshold of the neurons in the network
* @return The threshold of the neurons
*/
double expected() const;
/**
* @brief Get the expected value (in case you have an only neuron in output layer). Of course you should specify this when you
* build your network by using setExpected.
* @return The expected output value for a certain training phase
*/
std::vector<double> getExpected() const;
/**
* @brief It sets the value you expect from your network (in case the network has an only neuron in its output layer)
* @param ex Expected output value
*/
void setExpected(double ex);
/**
* @brief Set the values you expect from your network
* @param ex Expected output values
*/
void setExpected(std::vector<double> ex);
/**
* @brief It updates through back-propagation the weights of the synapsis and
* computes again the output value for <i>epochs</i> times, calling back
* updateWeights and commitChanges functions
*/
void update();
double getThreshold() const;
/**
* @brief It propagates values through the network. Use this when you want to give
@ -196,12 +199,6 @@ namespace neuralpp {
*/
void setInput (std::vector<double> v);
/**
* @brief It links the layers of the network (input, hidden, output). Don't use unless
* you exactly know what you're doing, it is already called by the constructor
*/
void link();
/**
* @brief Save a trained neural network to a binary file
* @param fname Binary file where you're going to save your network
@ -210,6 +207,30 @@ namespace neuralpp {
*/
void save (const char* fname) throw(NetworkFileWriteException);
/**
* @brief DEPRECATED. Load a trained neural network from a binary file.
* This function is deprecated and kept for back-compatibility. Use
* the XML format instead to load and neural networks and, respectly,
* the NeuralNetwork(const std::string) constructor or the save(const char*)
* methods.
* @param fname Name of the file to be loaded
* @throws NetworkFileNotFoundException When you're trying to load
* an invalid network file
*/
void loadFromBinary (const std::string fname) throw(NetworkFileNotFoundException);
/**
* @brief DEPRECATED. Save a trained neural network to a binary file.
* This function is deprecated and kept for back-compatibility. Use
* the XML format instead to load and neural networks and, respectly,
* the NeuralNetwork(const std::string) constructor or the save(const char*)
* methods.
* @param fname Name of the file to be saved with the network information
* @throws NetworkFileWriteException When you try to write the network
* information to an invalid file
*/
void saveToBinary (const char* fname) throw(NetworkFileWriteException);
/**
* @brief Train a network using a training set loaded from an XML file. A sample XML file
* is available in examples/adder.xml
@ -225,14 +246,6 @@ namespace neuralpp {
*/
static void initXML (std::string& xml);
/**
* @brief Splits a string into a vector of doubles, given a delimitator
* @param delim Delimitator
* @param str String to be splitted
* @return Vector of doubles containing splitted values
*/
static std::vector<double> split (char delim, std::string str);
/**
* @brief Get a training set from a string and copies it to an XML
* For example, these strings could be training sets for making sums:
@ -271,13 +284,9 @@ namespace neuralpp {
public:
/**
* @brief Constructor
* @param i Input neuron
* @param o Output neuron
* @param w Weight for the synapsis
* @param d Delta for the synapsis
* @brief Empty constructor (it does nothing)
*/
Synapsis(Neuron* i, Neuron* o, double w, double d);
Synapsis() {}
/**
* @brief Constructor
@ -424,6 +433,9 @@ namespace neuralpp {
*/
void setProp (double p);
void setSynIn (size_t n);
void setSynOut (size_t n);
/**
* @brief Get the activation value of the neuron
* @return Activation value for the neuron

20
src/layer.cpp
View File

@ -19,8 +19,8 @@ using std::vector;
namespace neuralpp {
Layer::Layer(size_t sz, double (*a) (double), double th) {
for (size_t i = 0; i < sz; i++) {
Neuron n(a);
elements.push_back(n);
Neuron n(a,th);
elements.push_back(n);
}
threshold = th;
@ -46,16 +46,18 @@ namespace neuralpp {
void Layer::link(Layer& l) {
srand((unsigned) time(NULL));
for (size_t i = 0; i < l.size(); i++)
l.elements[i].setSynOut(size());
for (size_t i = 0; i < size(); i++)
elements[i].setSynIn(l.size());
for (size_t i = 0; i < l.size(); i++) {
Neuron *n1 = &(l.elements[i]);
for (size_t j = 0; j < size(); j++) {
Neuron *n2 = &(elements[j]);
Synapsis s(n1, n2, RAND, actv_f);
n1->push_out(s);
n2->push_in(s);
Synapsis *s = new Synapsis( &(l.elements[i]), &(elements[i]), RAND, actv_f );
l.elements[i].synOut(j) = *s;
elements[j].synIn(i) = *s;
}
}
}

220
src/neuralnet.cpp
View File

@ -42,27 +42,11 @@ namespace neuralpp {
actv_f = a;
threshold = th;
input = new Layer(in_size, __actv, th);
hidden = new Layer(hidden_size, __actv, th);
output = new Layer(out_size, __actv, th);
link();
}
/*NeuralNet::NeuralNet(size_t in_size, size_t hidden_size,
size_t out_size, double (*a) (double),
double l, int e, double th) {
epochs = e;
ref_epochs = epochs;
l_rate = l;
actv_f = a;
threshold = th;
input = new Layer(in_size, a, th);
hidden = new Layer(hidden_size, a, th);
output = new Layer(out_size, a, th);
link();
}*/
}
double NeuralNet::getOutput() const {
return (*output)[0].getActv();
@ -143,7 +127,9 @@ namespace neuralpp {
(-l_rate) * (z-d) * f * y;
Dk += ( (z-d) * f * s->getWeight() );
s->setDelta(out_delta);
(*hidden)[j].synOut(i).setDelta(out_delta);
}
}
@ -166,13 +152,12 @@ namespace neuralpp {
(-l_rate) * d * x;
s->setDelta(hidden_delta);
(*input)[j].synOut(i).setDelta(hidden_delta);
}
}
}
void NeuralNet::commitChanges(Layer& l) {
for (size_t i = 0; i < l.size(); i++) {
Neuron *n = &(l[i]);
for (size_t i = 0; i < output->size(); i++) {
Neuron *n = &((*output)[i]);
for (size_t j = 0; j < n->nIn(); j++) {
Synapsis *s = &(n->synIn(j));
@ -181,20 +166,204 @@ namespace neuralpp {
s->setDelta(0.0);
}
}
for (size_t i = 0; i < hidden->size(); i++) {
Neuron *n = &((*hidden)[i]);
for (size_t j = 0; j < n->nOut(); j++) {
Synapsis *s = &(n->synOut(j));
s->setWeight(s->getWeight() +
s->getDelta());
s->setDelta(0.0);
}
}
for (size_t i = 0; i < hidden->size(); i++) {
Neuron *n = &((*hidden)[i]);
for (size_t j = 0; j < n->nIn(); j++) {
Synapsis *s = &(n->synIn(j));
s->setWeight(s->getWeight() +
s->getDelta());
s->setDelta(0.0);
}
}
for (size_t i = 0; i < input->size(); i++) {
Neuron *n = &((*input)[i]);
for (size_t j = 0; j < n->nOut(); j++) {
Synapsis *s = &(n->synOut(j));
s->setWeight(s->getWeight() +
s->getDelta());
s->setDelta(0.0);
}
}
}
void NeuralNet::update() {
epochs = ref_epochs;
while ((epochs--) > 0) {
updateWeights();
commitChanges(*output);
commitChanges(*hidden);
propagate();
updateWeights();
}
}
void NeuralNet::save (const char *fname) throw(NetworkFileWriteException) {
ofstream out(fname);
stringstream xml(stringstream::in | stringstream::out);
if (!out)
throw NetworkFileWriteException();
xml << "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
<< "<!DOCTYPE NETWORK SYSTEM \"http://blacklight.gotdns.org/prog/neuralpp/network.dtd\">\n"
<< "<!-- Automatically generated by BlackLight's Neural++ module -->\n\n"
<< "<network name=\"Put here the name for this neural network\" epochs=\"" << ref_epochs << "\" "
<< "learning_rate=\"" << l_rate << "\" threshold=\"" << threshold << "\">\n"
<< "\t<layer class=\"input\" size=\"" << input->size() << "\"></layer>\n"
<< "\t<layer class=\"hidden\" size=\"" << hidden->size() << "\"></layer>\n"
<< "\t<layer class=\"output\" size=\"" << output->size() << "\"></layer>\n\n";
for (unsigned int i = 0; i < hidden->size(); i++) {
int nin = (*hidden)[i].nIn();
for (int j = 0; j < nin; j++)
xml << "\t<synapsis class=\"inhid\" input=\"" << j << "\" output=\"" << i << "\" "
<< "weight=\"" << (*hidden)[i].synIn(j).getWeight() << "\"></synapsis>\n";
}
for (unsigned int i = 0; i < output->size(); i++) {
int nin = (*output)[i].nIn();
for (int j = 0; j < nin; j++)
xml << "\t<synapsis class=\"hidout\" input=\"" << j << "\" output=\"" << i << "\" "
<< "weight=\"" << (*output)[i].synIn(j).getWeight() << "\"></synapsis>\n";
}
xml << "</network>\n";
out << xml.str();
}
NeuralNet::NeuralNet(const string fname) throw(NetworkFileNotFoundException) {
unsigned int in_size = 0, hid_size = 0, out_size = 0;
vector< vector<double> > in_hid_synapses, hid_out_synapses;
CMarkup xml;
xml.Load(fname.c_str());
if (!xml.IsWellFormed()) {
throw InvalidXMLException();
return;
}
if (xml.FindElem("network")) {
if (xml.GetAttrib("epochs").empty())
throw InvalidXMLException();
if (xml.GetAttrib("learning_rate").empty())
throw InvalidXMLException();
epochs = atoi(xml.GetAttrib("epochs").c_str());
l_rate = atof(xml.GetAttrib("learning_rate").c_str());
threshold = 0.0;
if (!xml.GetAttrib("threshold").empty())
threshold = atof(xml.GetAttrib("threshold").c_str());
while (xml.FindChildElem("layer")) {
if (xml.GetChildAttrib("class").empty())
throw InvalidXMLException();
if (xml.GetChildAttrib("size").empty())
throw InvalidXMLException();
if (!xml.GetChildAttrib("class").compare("input"))
in_size = atoi(xml.GetChildAttrib("size").c_str());
else if (!xml.GetChildAttrib("class").compare("hidden"))
hid_size = atoi(xml.GetChildAttrib("size").c_str());
else if (!xml.GetChildAttrib("class").compare("output"))
out_size = atoi(xml.GetChildAttrib("size").c_str());
else
throw InvalidXMLException();
}
if (in_size && hid_size && out_size) {
in_hid_synapses = vector< vector<double> >(in_size);
for (unsigned int i=0; i < in_size; i++)
in_hid_synapses[i] = vector<double>(hid_size);
hid_out_synapses = vector< vector<double> >(hid_size);
for (unsigned int i=0; i < hid_size; i++)
hid_out_synapses[i] = vector<double>(out_size);
}
while (xml.FindChildElem("synapsis")) {
if (!(in_size && hid_size && out_size))
throw InvalidXMLException();
if (xml.GetChildAttrib("class").empty())
throw InvalidXMLException();
if (xml.GetChildAttrib("input").empty())
throw InvalidXMLException();
if (xml.GetChildAttrib("output").empty())
throw InvalidXMLException();
if (xml.GetChildAttrib("weight").empty())
throw InvalidXMLException();
unsigned int in = atoi(xml.GetChildAttrib("input").c_str());
unsigned int out = atoi(xml.GetChildAttrib("output").c_str());
if (xml.GetChildAttrib("class") == "inhid") {
if (in >= in_size || out >= hid_size)
throw InvalidXMLException();
in_hid_synapses[in][out] = atof(xml.GetChildAttrib("weight").c_str());
}
if (xml.GetChildAttrib("class") == "hidout") {
if (in >= hid_size || out >= out_size)
throw InvalidXMLException();
hid_out_synapses[in][out] = atof(xml.GetChildAttrib("weight").c_str());
}
}
}
*this = NeuralNet(in_size, hid_size, out_size, l_rate, epochs, threshold);
hidden->link(*input);
output->link(*hidden);
// Restore synapses
for (unsigned int i = 0; i < input->size(); i++) {
for (unsigned int j = 0; j < hidden->size(); j++)
(*input)[i].synOut(j).setWeight( in_hid_synapses[i][j] );
}
for (unsigned int i = 0; i < output->size(); i++) {
for (unsigned int j = 0; j < hidden->size(); j++)
(*output)[i].synIn(j).setWeight( (hid_out_synapses[j][i]) );
}
for (unsigned int i = 0; i < hidden->size(); i++) {
for (unsigned int j = 0; j < input->size(); j++)
(*hidden)[i].synIn(j).setWeight( (in_hid_synapses[j][i]) );
}
for (unsigned int i = 0; i < hidden->size(); i++) {
for (unsigned int j = 0; j < output->size(); j++)
(*hidden)[i].synOut(j).setWeight( hid_out_synapses[i][j] );
}
}
void NeuralNet::saveToBinary (const char *fname) throw(NetworkFileWriteException) {
struct netrecord record;
ofstream out(fname);
@ -308,7 +477,7 @@ namespace neuralpp {
out.close();
}
NeuralNet::NeuralNet(const string fname) throw(NetworkFileNotFoundException) {
void NeuralNet::loadFromBinary (const string fname) throw(NetworkFileNotFoundException) {
struct netrecord record;
ifstream in(fname.c_str());
@ -479,7 +648,6 @@ namespace neuralpp {
setInput(input);
setExpected(output);
propagate();
update();
}
}

17
src/neuron.cpp
View File

@ -24,8 +24,8 @@ namespace neuralpp {
Neuron::Neuron(vector < Synapsis > i, vector < Synapsis > o,
double (*a) (double), double th) {
in = i;
out = o;
in.assign(i.begin(), i.end());
out.assign(o.begin(), o.end());
actv_f = a;
threshold = th;
}
@ -38,11 +38,11 @@ namespace neuralpp {
return out[i];
}
void Neuron::push_in(Synapsis s) {
void Neuron::push_in(Synapsis s) {
in.push_back(s);
}
void Neuron::push_out(Synapsis s) {
void Neuron::push_out(Synapsis s) {
out.push_back(s);
}
@ -51,10 +51,17 @@ namespace neuralpp {
}
void Neuron::setActv(double val) {
//actv_val = actv_f(val);
actv_val = val;
}
void Neuron::setSynIn (size_t n) {
in = vector<Synapsis>(n);
}
void Neuron::setSynOut (size_t n) {
out = vector<Synapsis>(n);
}
size_t Neuron::nIn() {
return in.size();
}

11
src/synapsis.cpp
View File

@ -15,14 +15,6 @@
#include "neural++.hpp"
namespace neuralpp {
Synapsis::Synapsis(Neuron * i, Neuron * o, double w, double d) {
in = i;
out = o;
weight = w;
delta = d;
prev_delta = 0;
}
Synapsis::Synapsis(Neuron * i, Neuron * o, double (*a) (double)) {
srand((unsigned) time(NULL));
@ -75,9 +67,6 @@ namespace neuralpp {
}
void Synapsis::setDelta(double d) throw(InvalidSynapticalWeightException) {
if (d > 1.0)
throw InvalidSynapticalWeightException();
prev_delta = delta;
delta = d;
}