From 6ba54eae83fc3ab22784204569e9205cb3a83440 Mon Sep 17 00:00:00 2001
From: hartbook <franck.dary@etu.univ-amu.fr>
Date: Mon, 9 Jul 2018 10:11:50 +0200
Subject: [PATCH] Added dropout support
---
MLP/include/MLP.hpp | 1 +
MLP/src/MLP.cpp | 18 +++++++++++++-----
tape_machine/src/Classifier.cpp | 2 +-
3 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/MLP/include/MLP.hpp b/MLP/include/MLP.hpp
index 28c7878..91bfea8 100644
--- a/MLP/include/MLP.hpp
+++ b/MLP/include/MLP.hpp
@@ -52,6 +52,7 @@ class MLP
dynet::ParameterCollection model;
dynet::AmsgradTrainer trainer;
bool trainMode;
+ bool dropoutActive;
private :
diff --git a/MLP/src/MLP.cpp b/MLP/src/MLP.cpp
index e2789b7..283cef2 100644
--- a/MLP/src/MLP.cpp
+++ b/MLP/src/MLP.cpp
@@ -68,6 +68,7 @@ MLP::MLP(std::vector<Layer> layers)
dynet::initialize(getDefaultParams());
trainMode = true;
+ dropoutActive = true;
checkLayersCompatibility();
@@ -109,6 +110,8 @@ MLP::Layer::Layer(int input_dim, int output_dim,
std::vector<float> MLP::predict(FeatureModel::FeatureDescription & fd)
{
+ bool currentDropoutActive = dropoutActive;
+ dropoutActive = false;
dynet::ComputationGraph cg;
std::vector<dynet::Expression> expressions;
@@ -120,6 +123,8 @@ std::vector<float> MLP::predict(FeatureModel::FeatureDescription & fd)
dynet::Expression output = run(cg, input);
+ dropoutActive = currentDropoutActive;
+
return as_vector(cg.forward(output));
}
@@ -193,14 +198,12 @@ dynet::Expression MLP::run(dynet::ComputationGraph & cg, dynet::Expression x)
dynet::Expression a = dynet::affine_transform({b, W, h_cur});
// Apply activation function
dynet::Expression h = activate(a, layers[l].activation);
- h_cur = h;
// Take care of dropout
- /*
dynet::Expression h_dropped;
if(layers[l].dropout_rate > 0){
- if(dropout_active){
+ if(dropoutActive){
dynet::Expression mask = random_bernoulli(cg,
- {layers[l].output_dim}, 1 - layers[l].dropout_rate);
+ {(unsigned int)layers[l].output_dim}, 1 - layers[l].dropout_rate);
h_dropped = cmult(h, mask);
}
else{
@@ -212,7 +215,6 @@ dynet::Expression MLP::run(dynet::ComputationGraph & cg, dynet::Expression x)
}
h_cur = h_dropped;
- */
}
return h_cur;
@@ -309,6 +311,9 @@ int MLP::trainOnBatch(Examples & examples, int start, int end)
int MLP::getScoreOnBatch(Examples & examples, int start, int end)
{
+ bool currentDropoutActive = dropoutActive;
+ dropoutActive = false;
+
dynet::ComputationGraph cg;
std::vector<dynet::Expression> inputs;
std::vector<unsigned int> goldClasses;
@@ -354,6 +359,8 @@ int MLP::getScoreOnBatch(Examples & examples, int start, int end)
nbCorrect++;
}
+ dropoutActive = currentDropoutActive;
+
return nbCorrect;
}
@@ -429,6 +436,7 @@ MLP::MLP(const std::string & filename)
dynet::initialize(getDefaultParams());
trainMode = false;
+ dropoutActive = false;
load(filename);
}
diff --git a/tape_machine/src/Classifier.cpp b/tape_machine/src/Classifier.cpp
index 4da0fcd..55dc6cd 100644
--- a/tape_machine/src/Classifier.cpp
+++ b/tape_machine/src/Classifier.cpp
@@ -133,7 +133,7 @@ void Classifier::initClassifier(Config & config)
for (auto feat : fd.values)
nbInputs += feat.vec->size();
- mlp.reset(new MLP({{nbInputs, nbHidden, 0.0, MLP::Activation::RELU},
+ mlp.reset(new MLP({{nbInputs, nbHidden, 0.3, MLP::Activation::RELU},
{nbHidden, nbOutputs, 0.0, MLP::Activation::LINEAR}}));
}
--
GitLab