Added a way to get sequences mean entropy and standard deviation during training

decoder/src/Decoder.cpp

@@ -72,11 +72,7 @@ void Decoder::decode()
     {
       nbActionsInSequence++;
 
-      for (unsigned int i = 0; i < 2 && i < weightedActions.size(); i++)
-      {
-        auto it = weightedActions.begin() + i;
-        entropyAccumulator -= it->second.first - (it->second.first - weightedActions[0].second.first);
-      }
+      entropyAccumulator += Classifier::computeEntropy(weightedActions);
 
       if (config.head >= 1 && config.getTape(ProgramParameters::sequenceDelimiterTape)[config.head-1] != ProgramParameters::sequenceDelimiter)
         justFlipped = false;

trainer/include/Trainer.hpp

@@ -59,7 +59,7 @@ class Trainer
   /// @brief Get the scores of the classifiers on the dev dataset.
   ///
   /// @return Map from each Classifier name to their score.
-  std::map<std::string, float> getScoreOnDev();
+  std::map<std::string, std::pair<float, std::pair<float, float> > > getScoreOnDev();
 
   public :
 

trainer/src/Trainer.cpp

@@ -12,7 +12,7 @@ Trainer::Trainer(TransitionMachine & tm, BD & bd, Config & config, BD * devBD, C
 {
 }
 
-std::map<std::string, float> Trainer::getScoreOnDev()
+std::map<std::string, std::pair<float, std::pair<float, float> > > Trainer::getScoreOnDev()
 {
   if (!devConfig)
     return {};
@@ -25,6 +25,11 @@ std::map<std::string, float> Trainer::getScoreOnDev()
   if (ProgramParameters::debug)
     fprintf(stderr, "Computing score on dev set\n");
 
+  int nbActionsInSequence = 0;
+  float entropyAccumulator = 0.0;
+  bool justFlipped = false;
+  std::vector<float> entropies;
+
   while (!devConfig->isFinal())
   {
     TransitionMachine::State * currentState = tm.getCurrentState();
@@ -82,12 +87,46 @@ std::map<std::string, float> Trainer::getScoreOnDev()
       TransitionMachine::Transition * transition = tm.getTransition(actionName);
       tm.takeTransition(transition);
       devConfig->moveHead(transition->headMvt);
+
+      if (ProgramParameters::printEntropy)
+      {
+        nbActionsInSequence++;
+
+        entropyAccumulator += Classifier::computeEntropy(weightedActions);
+
+        if (devConfig->head >= 1 && devConfig->getTape(ProgramParameters::sequenceDelimiterTape)[devConfig->head-1] != ProgramParameters::sequenceDelimiter)
+        justFlipped = false;
+
+        if ((devConfig->head >= 1 && devConfig->getTape(ProgramParameters::sequenceDelimiterTape)[devConfig->head-1] == ProgramParameters::sequenceDelimiter && !justFlipped))
+        {
+          justFlipped = true;
+          entropyAccumulator /= nbActionsInSequence;
+          nbActionsInSequence = 0;
+          entropies.emplace_back(entropyAccumulator);
+          entropyAccumulator = 0.0;
+        }
+      }
+
     }
   }
 
-  std::map<std::string, float> scores;
+  std::map<std::string, std::pair<float,std::pair<float,float> > > scores;
   for (auto & it : counts)
-    scores[it.first] = 100.0 * it.second.second / it.second.first;
+  {
+    scores[it.first].first = 100.0 * it.second.second / it.second.first;
+    if (ProgramParameters::printEntropy)
+    {
+      for (float f : entropies)
+        scores[it.first].second.first += f;
+
+      scores[it.first].second.first /= entropies.size();
+
+      for (float f : entropies)
+        scores[it.first].second.second += (f-scores[it.first].second.first)*(f-scores[it.first].second.first);
+
+      scores[it.first].second.second /= entropies.size();
+    }
+  }
 
   return scores;
 }
@@ -273,9 +312,9 @@ void Trainer::printScoresAndSave(FILE * output)
   {
     for (auto & it : devScores)
     {
-      if (bestScores.count(it.first) == 0 || bestScores[it.first] < it.second)
+      if (bestScores.count(it.first) == 0 || bestScores[it.first] < it.second.first)
       {
-        bestScores[it.first] = it.second;
+        bestScores[it.first] = it.second.first;
         saved[it.first] = true;
       }
       else
@@ -314,8 +353,10 @@ void Trainer::printScoresAndSave(FILE * output)
     names.emplace_back(it.first);
     acc.emplace_back("accuracy");
     train.emplace_back(": train(" + float2str(scores[it.first], "%.2f") + "%)");
-    dev.emplace_back(devConfig ? "dev(" +float2str(devScores[it.first], "%.2f") + "%)" : "");
+    dev.emplace_back(devConfig ? "dev(" +float2str(devScores[it.first].first, "%.2f") + "%)" : "");
     savedStr.emplace_back(saved[it.first] ? "SAVED" : "");
+    if (ProgramParameters::printEntropy)
+      savedStr.back() += " Entropy[" + float2str(devScores[it.first].second.first, "%.2f") + "\u00B1" + float2str(devScores[it.first].second.second, "%.2f") + "]";
   }
 
   if (ProgramParameters::interactive)

trainer/src/macaon_train.cpp

@@ -40,6 +40,7 @@ po::options_description getOptionsDescription()
   opt.add_options()
     ("help,h", "Produce this help message")
     ("debug,d", "Print infos on stderr")
+    ("printEntropy", "Print mean entropy and standard deviation accross sequences")
     ("optimizer", po::value<std::string>()->default_value("amsgrad"),
       "The learning algorithm to use : amsgrad | adam | sgd")
     ("dev", po::value<std::string>()->default_value(""),
@@ -237,6 +238,7 @@ int main(int argc, char * argv[])
   ProgramParameters::bdName = vm["bd"].as<std::string>();
   ProgramParameters::mcdName = vm["mcd"].as<std::string>();
   ProgramParameters::debug = vm.count("debug") == 0 ? false : true;
+  ProgramParameters::printEntropy = vm.count("printEntropy") == 0 ? false : true;
   ProgramParameters::printTime = vm.count("printTime") == 0 ? false : true;
   ProgramParameters::trainName = vm["train"].as<std::string>();
   ProgramParameters::devName = vm["dev"].as<std::string>();

transition_machine/include/Classifier.hpp

@@ -70,6 +70,13 @@ class Classifier
   /// @param threshold Print only the n most weighted Action.
   static void printWeightedActions(FILE * output, WeightedActions & wa, int threshold = 5);
 
+  /// @brief Get a measure of confidence.
+  ///
+  /// @param wa The actions with a score associated for each one.
+  ///
+  /// @return A measure of confidence.
+  static float computeEntropy(WeightedActions & wa);
+
   /// @brief Convert a string to its corresponding Type.
   ///
   /// If s is unknown, the program aborts.

transition_machine/src/Classifier.cpp

@@ -266,3 +266,16 @@ void Classifier::explainCostOfActions(FILE * output, Config & config)
   }
 }
 
+float Classifier::computeEntropy(WeightedActions & wa)
+{
+  float entropy = 0.0;
+ 
+  for (unsigned int i = 0; i < 2 && i < wa.size(); i++)
+  {
+    auto it = wa.begin() + i;
+    entropy -= it->second.first - (it->second.first - wa[0].second.first);
+  }
+
+  return entropy;
+}
+