From 0da7cec739ca3cf0cf4dd8fa32b8ca8fe7717f76 Mon Sep 17 00:00:00 2001
From: Franck Dary <franck.dary@etu.univ-amu.fr>
Date: Fri, 11 Jan 2019 16:44:47 +0100
Subject: [PATCH] Added a skeleton for a GeneticAlgorithm class
---
neural_network/include/GeneticAlgorithm.hpp | 89 ++++++++++++++
neural_network/include/MLP.hpp | 73 +-----------
neural_network/include/NeuralNetwork.hpp | 71 ++++++++++++
neural_network/src/GeneticAlgorithm.cpp | 96 +++++++++++++++
neural_network/src/MLP.cpp | 122 --------------------
neural_network/src/NeuralNetwork.cpp | 122 ++++++++++++++++++++
6 files changed, 379 insertions(+), 194 deletions(-)
create mode 100644 neural_network/include/GeneticAlgorithm.hpp
create mode 100644 neural_network/src/GeneticAlgorithm.cpp
diff --git a/neural_network/include/GeneticAlgorithm.hpp b/neural_network/include/GeneticAlgorithm.hpp
new file mode 100644
index 0000000..caf3e6e
--- /dev/null
+++ b/neural_network/include/GeneticAlgorithm.hpp
@@ -0,0 +1,89 @@
+#ifndef GENETICALGORITHM__H
+#define GENETICALGORITHM__H
+
+#include <dynet/nodes.h>
+#include <dynet/dynet.h>
+#include <dynet/training.h>
+#include <dynet/timing.h>
+#include <dynet/expr.h>
+#include <dynet/io.h>
+#include <string>
+#include "NeuralNetwork.hpp"
+#include "FeatureModel.hpp"
+
+class GeneticAlgorithm : public NeuralNetwork
+{
+ private :
+
+ /// @brief An individual is a MLP
+ class Individual
+ {
+ private :
+
+ /// @brief The Layers of the MLP.
+ std::vector<Layer> layers;
+ /// @brief The parameters corresponding to the layers of the MLP.
+ std::vector< std::vector<dynet::Parameter> > parameters;
+
+ public :
+
+ /// @brief Create a new individual for the population.
+ ///
+ /// @param topology The topology the underlying MLP will take.
+ /// @param model The Collection of parameters of the GeneticAlgorithm.
+ /// @param nbInputs The size of the input layer of the MLP.
+ /// @param nbOutputs The size of the output layer of the MLP.
+ Individual(const std::string & topology, dynet::ParameterCollection & model, int nbInputs, int nbOutputs);
+ };
+
+ private :
+
+ /// @brief Load this GeneticAlgorithm from a file.
+ ///
+ /// @param filename The name of the file where the GeneticAlgorithm is stored.
+ void load(const std::string & filename);
+
+ public :
+
+ /// @brief Create a new untrained GeneticAlgorithm from scratch.
+ GeneticAlgorithm();
+
+ /// @brief Create and load an already trained GeneticAlgorithm from a file.
+ ///
+ /// @param filename The file where the GeneticAlgorithm is stored.
+ GeneticAlgorithm(const std::string & filename);
+
+ /// @brief initialize a new untrained GeneticAlgorithm from a desired topology.
+ ///
+ /// @param nbInputs The size of the input.
+ /// @param topology Description of the GeneticAlgorithm.
+ /// @param nbOutputs The size of the output.
+ void init(int nbInputs, const std::string & topology, int nbOutputs) override;
+
+ /// @brief Give a score to each possible class, given an input.
+ ///
+ /// @param fd The input to use.
+ ///
+ /// @return A vector containing one score per possible class.
+ std::vector<float> predict(FeatureModel::FeatureDescription & fd) override;
+
+ /// @brief Update the parameters according to the given gold class.
+ ///
+ /// @param fd The input to use.
+ /// @param gold The gold class of this input.
+ ///
+ /// @return The loss.
+ float update(FeatureModel::FeatureDescription & fd, int gold) override;
+
+ /// @brief Save the GeneticAlgorithm to a file.
+ ///
+ /// @param filename The file to write the GeneticAlgorithm to.
+ void save(const std::string & filename) override;
+
+ /// @brief Print the topology of the GeneticAlgorithm.
+ ///
+ /// @param output Where the topology will be printed.
+ void printTopology(FILE * output) override;
+};
+
+#endif
diff --git a/neural_network/include/MLP.hpp b/neural_network/include/MLP.hpp
index fab20cf..5c73975 100644
--- a/neural_network/include/MLP.hpp
+++ b/neural_network/include/MLP.hpp
@@ -15,62 +15,6 @@
/// Once trained, it can also be used to predict the class of a certain input.
class MLP : public NeuralNetwork
{
- public :
-
- /// @brief Activation function for a MLP Layer.
- enum Activation
- {
- SIGMOID,
- TANH,
- RELU,
- ELU,
- LINEAR,
- SPARSEMAX,
- CUBE,
- SOFTMAX
- };
-
- /// @brief Get the string corresponding to an Activation.
- ///
- /// @param a The activation.
- ///
- /// @return The string corresponding to a.
- static std::string activation2str(Activation a);
-
- /// @brief Get the Activation corresponding to a string.
- ///
- /// @param s The string.
- ///
- /// @return The Activation corresponding to s. If s is unknown, the program abort.
- static Activation str2activation(std::string s);
-
- /// @brief A simple struct that represents a MLP Layer.
- struct Layer
- {
- /// @brief Number of input neurons of this Layer.
- int input_dim;
- /// @brief Number of output neurons of this Layer.
- int output_dim;
-
- /// @brief The dropout rate to apply to this Layer when training.
- float dropout_rate;
- /// @brief The activation function for this Layer.
- Activation activation;
-
- /// @brief Construct a new Layer
- ///
- /// @param input_dim
- /// @param output_dim
- /// @param dropout_rate
- /// @param activation
- Layer(int input_dim, int output_dim,
- float dropout_rate, Activation activation);
- /// @brief Print a description of this Layer.
- ///
- /// @param file Where to print the output.
- void print(FILE * file);
- };
-
private :
/// @brief The Layers of the MLP.
@@ -96,10 +40,6 @@ class MLP : public NeuralNetwork
void addLayerToModel(Layer & layer);
/// @brief Abort the program if the layers are not compatible.
void checkLayersCompatibility();
- /// @brief Set dynet and srand() seeds.
- ///
- /// @return The DynetParams containing the set seed.
- dynet::DynetParams & getDefaultParams();
/// @brief Compute the image of input x by the Multi Layer Perceptron.
///
/// @param cg The current computation graph.
@@ -107,13 +47,6 @@ class MLP : public NeuralNetwork
///
/// @return The result (values of the output Layer) of the computation of x by the Multi Layer Perceptron.
dynet::Expression run(dynet::ComputationGraph & cg, dynet::Expression x);
- /// @brief Compute the image of an expression by an activation function.
- ///
- /// @param h The expression we want the image of.
- /// @param f The activation function.
- ///
- /// @return f(h)
- inline dynet::Expression activate(dynet::Expression h, Activation f);
/// @brief Print the parameters.
///
/// @param output Where the parameters will be printed to.
@@ -146,10 +79,6 @@ class MLP : public NeuralNetwork
/// This function will use loadStruct and loadParameters.
/// @param filename The file from which the MLP will be loaded.
void load(const std::string & filename);
- /// @brief Initialize the dynet library.
- ///
- /// Must be called only once, and before any call to dynet functions.
- void initDynet();
/// @brief Get the loss expression
///
/// @param output Output from the neural network
@@ -170,7 +99,7 @@ class MLP : public NeuralNetwork
/// @param nbInputs The size of the input layer of the MLP.
/// @param topology Description of each hidden Layer of the MLP.
/// @param nbOutputs The size of the output layer of the MLP.
- void init(int nbInputs, const std::string & topology, int nbOutputs);
+ void init(int nbInputs, const std::string & topology, int nbOutputs) override;
/// @brief Construct a new MLP for training.
MLP();
/// @brief Read and construct a trained MLP from a file.
diff --git a/neural_network/include/NeuralNetwork.hpp b/neural_network/include/NeuralNetwork.hpp
index 00e2fd1..9f6a6df 100644
--- a/neural_network/include/NeuralNetwork.hpp
+++ b/neural_network/include/NeuralNetwork.hpp
@@ -14,6 +14,62 @@ class NeuralNetwork
{
protected :
+ /// @brief Activation function for a Layer.
+ enum Activation
+ {
+ SIGMOID,
+ TANH,
+ RELU,
+ ELU,
+ LINEAR,
+ SPARSEMAX,
+ CUBE,
+ SOFTMAX
+ };
+
+ /// @brief Get the string corresponding to an Activation.
+ ///
+ /// @param a The activation.
+ ///
+ /// @return The string corresponding to a.
+ static std::string activation2str(Activation a);
+
+ /// @brief Get the Activation corresponding to a string.
+ ///
+ /// @param s The string.
+ ///
+ /// @return The Activation corresponding to s. If s is unknown, the program abort.
+ static Activation str2activation(std::string s);
+
+ /// @brief A simple struct that represents a Layer.
+ struct Layer
+ {
+ /// @brief Number of input neurons of this Layer.
+ int input_dim;
+ /// @brief Number of output neurons of this Layer.
+ int output_dim;
+
+ /// @brief The dropout rate to apply to this Layer when training.
+ float dropout_rate;
+ /// @brief The activation function for this Layer.
+ Activation activation;
+
+ /// @brief Construct a new Layer
+ ///
+ /// @param input_dim
+ /// @param output_dim
+ /// @param dropout_rate
+ /// @param activation
+ Layer(int input_dim, int output_dim,
+ float dropout_rate, Activation activation);
+ /// @brief Print a description of this Layer.
+ ///
+ /// @param file Where to print the output.
+ void print(FILE * file);
+ };
+
+ protected :
+
/// @brief The seed that will be used by RNG (srand and dynet)
static int randomSeed;
@@ -32,6 +88,21 @@ class NeuralNetwork
///
/// @return A dynet Expression of value fv that can be used as an input in the NeuralNetwork
dynet::Expression featValue2Expression(dynet::ComputationGraph & cg, const FeatureModel::FeatureValue & fv);
+ /// @brief Set dynet and srand() seeds.
+ ///
+ /// @return The DynetParams containing the set seed.
+ dynet::DynetParams & getDefaultParams();
+ /// @brief Initialize the dynet library.
+ ///
+ /// Must be called only once, and before any call to dynet functions.
+ void initDynet();
+ /// @brief Compute the image of an expression by an activation function.
+ ///
+ /// @param h The expression we want the image of.
+ /// @param f The activation function.
+ ///
+ /// @return f(h)
+ dynet::Expression activate(dynet::Expression h, Activation f);
public :
diff --git a/neural_network/src/GeneticAlgorithm.cpp b/neural_network/src/GeneticAlgorithm.cpp
new file mode 100644
index 0000000..6194b70
--- /dev/null
+++ b/neural_network/src/GeneticAlgorithm.cpp
@@ -0,0 +1,96 @@
+#include "GeneticAlgorithm.hpp"
+#include "ProgramParameters.hpp"
+
+GeneticAlgorithm::GeneticAlgorithm()
+{
+ randomSeed = ProgramParameters::seed;
+ initDynet();
+}
+
+GeneticAlgorithm::GeneticAlgorithm(const std::string & filename)
+{
+ randomSeed = ProgramParameters::seed;
+ initDynet();
+
+ load(filename);
+}
+
+void GeneticAlgorithm::init(int nbInputs, const std::string & topology, int nbOutputs)
+{
+ fprintf(stderr, "init of genetic\n");
+}
+
+std::vector<float> GeneticAlgorithm::predict(FeatureModel::FeatureDescription & fd)
+{
+
+}
+
+float GeneticAlgorithm::update(FeatureModel::FeatureDescription & fd, int gold)
+{
+
+}
+
+void GeneticAlgorithm::save(const std::string & filename)
+{
+
+}
+
+void GeneticAlgorithm::printTopology(FILE * output)
+{
+
+}
+
+void GeneticAlgorithm::load(const std::string & filename)
+{
+
+}
+
+GeneticAlgorithm::Individual::Individual(const std::string & topology, dynet::ParameterCollection & model, int nbInputs, int nbOutputs)
+{
+ std::string topo = topology;
+ std::replace(topo.begin(), topo.end(), '(', ' ');
+ std::replace(topo.begin(), topo.end(), ')', ' ');
+
+ auto groups = split(topo);
+ for (auto group : groups)
+ {
+ if(group.empty())
+ continue;
+
+ std::replace(group.begin(), group.end(), ',', ' ');
+ auto layer = split(group);
+
+ if (layer.size() != 2)
+ {
+ fprintf(stderr, "ERROR (%s) : invalid topology \'%s\'. Aborting.\n", ERRINFO, topology.c_str());
+ exit(1);
+ }
+
+ int input = layers.empty() ? nbInputs : layers.back().output_dim;
+ int output = std::stoi(layer[0]);
+ layers.emplace_back(input, output, 0, str2activation(layer[1]));
+ }
+
+ layers.emplace_back(layers.back().output_dim, nbOutputs, 0.0, Activation::LINEAR);
+
+ if(layers.empty())
+ {
+ fprintf(stderr, "ERROR (%s) : constructed mlp with 0 layers. Aborting.\n", ERRINFO);
+ exit(1);
+ }
+
+ for(unsigned int i = 0; i < layers.size()-1; i++)
+ if(layers[i].output_dim != layers[i+1].input_dim)
+ {
+ fprintf(stderr, "ERROR (%s) : constructed mlp with incompatible layers. Aborting.\n", ERRINFO);
+ exit(1);
+ }
+
+ for (auto & layer : layers)
+ {
+ dynet::Parameter W = model.add_parameters({(unsigned)layer.output_dim, (unsigned)layer.input_dim});
+ dynet::Parameter b = model.add_parameters({(unsigned)layer.output_dim});
+ parameters.push_back({W,b});
+ }
+}
+
diff --git a/neural_network/src/MLP.cpp b/neural_network/src/MLP.cpp
index add22c6..39f6ecb 100644
--- a/neural_network/src/MLP.cpp
+++ b/neural_network/src/MLP.cpp
@@ -1,76 +1,5 @@
#include "MLP.hpp"
-std::string MLP::activation2str(Activation a)
-{
- switch(a)
- {
- case LINEAR :
- return "LINEAR";
- break;
- case RELU :
- return "RELU";
- break;
- case ELU :
- return "ELU";
- break;
- case CUBE :
- return "CUBE";
- break;
- case SIGMOID :
- return "SIGMOID";
- break;
- case TANH :
- return "TANH";
- break;
- case SOFTMAX :
- return "SOFTMAX";
- break;
- case SPARSEMAX :
- return "SPARSEMAX";
- break;
- default :
- break;
- }
-
- return "UNKNOWN";
-}
-
-MLP::Activation MLP::str2activation(std::string s)
-{
- if(s == "LINEAR")
- return LINEAR;
- else if(s == "RELU")
- return RELU;
- else if(s == "ELU")
- return ELU;
- else if(s == "CUBE")
- return CUBE;
- else if(s == "SIGMOID")
- return SIGMOID;
- else if(s == "TANH")
- return TANH;
- else if(s == "SOFTMAX")
- return SOFTMAX;
- else if(s == "SPARSEMAX")
- return SPARSEMAX;
- else
- {
- fprintf(stderr, "ERROR (%s) : invalid activation \'%s\'. Aborting\n",ERRINFO, s.c_str());
- exit(1);
- }
-
- return LINEAR;
-}
-
-void MLP::initDynet()
-{
- if(dynetIsInit)
- return;
-
- dynetIsInit = true;
- dynet::initialize(getDefaultParams());
-}
-
MLP::MLP()
{
randomSeed = ProgramParameters::seed;
@@ -162,15 +91,6 @@ void MLP::checkLayersCompatibility()
}
}
-MLP::Layer::Layer(int input_dim, int output_dim,
- float dropout_rate, Activation activation)
-{
- this->input_dim = input_dim;
- this->output_dim = output_dim;
- this->dropout_rate = dropout_rate;
- this->activation = activation;
-}
-
std::vector<float> MLP::predict(FeatureModel::FeatureDescription & fd)
{
bool currentDropoutActive = dropoutActive;
@@ -284,16 +204,6 @@ dynet::Expression MLP::errorCorrectionLoss(dynet::ComputationGraph & cg, dynet::
return dynet::sum(lossExpr);
}
-dynet::DynetParams & MLP::getDefaultParams()
-{
- static dynet::DynetParams params;
- params.random_seed = randomSeed;
-
- std::srand(params.random_seed);
-
- return params;
-}
-
dynet::Expression MLP::run(dynet::ComputationGraph & cg, dynet::Expression x)
{
static std::vector< std::pair<std::string,dynet::Expression> > exprForDebug;
@@ -378,38 +288,6 @@ dynet::Expression MLP::run(dynet::ComputationGraph & cg, dynet::Expression x)
return h_cur;
}
-inline dynet::Expression MLP::activate(dynet::Expression h, Activation f)
-{
- switch(f)
- {
- case LINEAR :
- return h;
- break;
- case RELU :
- return rectify(h);
- break;
- case ELU :
- return elu(h);
- break;
- case SIGMOID :
- return logistic(h);
- break;
- case TANH :
- return tanh(h);
- break;
- case SOFTMAX :
- return softmax(h);
- break;
- default :
- break;
- }
-
- fprintf(stderr, "ERROR (%s) : Activation not implemented \'%s\'. Aborting.\n", ERRINFO, activation2str(f).c_str());
- exit(1);
-
- return h;
-}
-
void MLP::printParameters(FILE * output)
{
fprintf(output, "Parameters : NOT IMPLEMENTED\n");
diff --git a/neural_network/src/NeuralNetwork.cpp b/neural_network/src/NeuralNetwork.cpp
index d12adfc..75a8ccb 100644
--- a/neural_network/src/NeuralNetwork.cpp
+++ b/neural_network/src/NeuralNetwork.cpp
@@ -44,3 +44,125 @@ dynet::Expression NeuralNetwork::featValue2Expression(dynet::ComputationGraph &
return dynet::concatenate(expressions);
}
+dynet::DynetParams & NeuralNetwork::getDefaultParams()
+{
+ static dynet::DynetParams params;
+ params.random_seed = randomSeed;
+
+ std::srand(params.random_seed);
+
+ return params;
+}
+
+void NeuralNetwork::initDynet()
+{
+ if(dynetIsInit)
+ return;
+
+ dynetIsInit = true;
+ dynet::initialize(getDefaultParams());
+}
+
+std::string NeuralNetwork::activation2str(Activation a)
+{
+ switch(a)
+ {
+ case LINEAR :
+ return "LINEAR";
+ break;
+ case RELU :
+ return "RELU";
+ break;
+ case ELU :
+ return "ELU";
+ break;
+ case CUBE :
+ return "CUBE";
+ break;
+ case SIGMOID :
+ return "SIGMOID";
+ break;
+ case TANH :
+ return "TANH";
+ break;
+ case SOFTMAX :
+ return "SOFTMAX";
+ break;
+ case SPARSEMAX :
+ return "SPARSEMAX";
+ break;
+ default :
+ break;
+ }
+
+ return "UNKNOWN";
+}
+
+NeuralNetwork::Activation NeuralNetwork::str2activation(std::string s)
+{
+ if(s == "LINEAR")
+ return LINEAR;
+ else if(s == "RELU")
+ return RELU;
+ else if(s == "ELU")
+ return ELU;
+ else if(s == "CUBE")
+ return CUBE;
+ else if(s == "SIGMOID")
+ return SIGMOID;
+ else if(s == "TANH")
+ return TANH;
+ else if(s == "SOFTMAX")
+ return SOFTMAX;
+ else if(s == "SPARSEMAX")
+ return SPARSEMAX;
+ else
+ {
+ fprintf(stderr, "ERROR (%s) : invalid activation \'%s\'. Aborting\n",ERRINFO, s.c_str());
+ exit(1);
+ }
+
+ return LINEAR;
+}
+
+NeuralNetwork::Layer::Layer(int input_dim, int output_dim,
+ float dropout_rate, Activation activation)
+{
+ this->input_dim = input_dim;
+ this->output_dim = output_dim;
+ this->dropout_rate = dropout_rate;
+ this->activation = activation;
+}
+
+dynet::Expression NeuralNetwork::activate(dynet::Expression h, Activation f)
+{
+ switch(f)
+ {
+ case LINEAR :
+ return h;
+ break;
+ case RELU :
+ return rectify(h);
+ break;
+ case ELU :
+ return elu(h);
+ break;
+ case SIGMOID :
+ return logistic(h);
+ break;
+ case TANH :
+ return tanh(h);
+ break;
+ case SOFTMAX :
+ return softmax(h);
+ break;
+ default :
+ break;
+ }
+
+ fprintf(stderr, "ERROR (%s) : Activation not implemented \'%s\'. Aborting.\n", ERRINFO, activation2str(f).c_str());
+ exit(1);
+
+ return h;
+}
+
--
GitLab