We now save only best epochs in training

trainer/include/Trainer.hpp

@@ -35,6 +35,9 @@ void processAllExamples(
   void printIterationScores(FILE * output,
     std::map< std::string, std::pair<int, int> > & nbExamplesTrain,
     std::map< std::string, std::pair<int, int> > & nbExamplesDev,
+    std::map< std::string, std::vector<float> > & trainScores,
+    std::map< std::string, std::vector<float> > & devScores,
+    std::map<std::string, int> & bestIter,
     int nbIter, int curIter);
 
   void shuffleAllExamples(std::map<Classifier*, MLP::Examples > &);

trainer/src/Trainer.cpp

@@ -68,6 +68,9 @@ void Trainer::processAllExamples(
 void Trainer::printIterationScores(FILE * output,
   std::map< std::string, std::pair<int, int> > & nbExamplesTrain,
   std::map< std::string, std::pair<int, int> > & nbExamplesDev,
+  std::map< std::string, std::vector<float> > & trainScores,
+  std::map< std::string, std::vector<float> > & devScores,
+  std::map<std::string, int> & bestIter,
   int nbIter, int curIter)
 {
   fprintf(output, "Iteration %d/%d :\n", curIter+1, nbIter);
@@ -75,10 +78,34 @@ void Trainer::printIterationScores(FILE * output,
   {
     float scoreTrain = 100.0*it.second.second / it.second.first;
     float scoreDev = devConfig ? 100.0*nbExamplesDev[it.first].second / nbExamplesDev[it.first].first : -1.0;
+
+    trainScores[it.first].emplace_back(scoreTrain);
+    devScores[it.first].emplace_back(scoreDev);
+
+    bool isBest = curIter ? false : true;
     if (devConfig)
-      fprintf(output, "\t%s accuracy : train(%.2f%%) dev(%.2f%%)\n", it.first.c_str(), scoreTrain, scoreDev);
+    {
+      if (scoreDev > devScores[it.first][bestIter[it.first]])
+      {
+        isBest = true;
+        bestIter[it.first] = devScores[it.first].size()-1;
+      }
+    }
     else
-      fprintf(output, "\t%s accuracy : train(%.2f%%)\n", it.first.c_str(), scoreTrain);
+    {
+      if (scoreTrain > trainScores[it.first][bestIter[it.first]])
+      {
+        isBest = true;
+        bestIter[it.first] = trainScores[it.first].size()-1;
+      }
+    }
+
+    if (devConfig)
+      fprintf(output, "\t%s accuracy : train(%.2f%%) dev(%.2f%%)", it.first.c_str(), scoreTrain, scoreDev);
+    else
+      fprintf(output, "\t%s accuracy : train(%.2f%%)", it.first.c_str(), scoreTrain);
+
+    fprintf(output, "%s", isBest ? " SAVED\n" : "\n");
   }
 }
 
@@ -100,6 +127,10 @@ void Trainer::trainBatched(int nbIter, int batchSize, bool mustShuffle)
   if(devMcd && devConfig)
     getExamplesByClassifier(devExamples, *devConfig);
 
+  std::map< std::string, std::vector<float> > trainScores;
+  std::map< std::string, std::vector<float> > devScores;
+  std::map<std::string, int> bestIter;
+
   for (int i = 0; i < nbIter; i++)
   {
     std::map< std::string, std::pair<int, int> > nbExamplesTrain;
@@ -120,14 +151,16 @@ void Trainer::trainBatched(int nbIter, int batchSize, bool mustShuffle)
         return c->getScoreOnBatch(ex, s, e); 
       });
 
-    printIterationScores(stderr, nbExamplesTrain, nbExamplesDev, nbIter, i);
-  }
+    printIterationScores(stderr, nbExamplesTrain, nbExamplesDev,
+                         trainScores, devScores, bestIter, nbIter, i);
 
     auto & classifiers = tm.getClassifiers();
     for(Classifier * cla : classifiers)
       if(cla->needsTrain())
+        if(bestIter[cla->name] == i)
           cla->save();
   }
+}
 
 void Trainer::train(int nbIter, int batchSize, bool mustShuffle)
 {