Added a way to delete duplicates training examples

maca_common/include/Fann_file.h

@@ -52,6 +52,7 @@ class Fann_file
   unsigned int get_nb_examples();
   unsigned int get_nb_classes();
   void over_sample(float ratio);
+  void delete_duplicates();
 };
 
 #endif

maca_common/src/Fann_file.c

@@ -3,6 +3,7 @@
 #include "string.h"
 #include <algorithm>
 #include <map>
+#include <set>
 
 Fann_file::Fann_file(const char * filename, unsigned int classifier_id)
 {
@@ -162,3 +163,25 @@ void Fann_file::over_sample(float ratio)
   }
 }
 
+void Fann_file::delete_duplicates()
+{
+  std::vector<unsigned int> new_order;
+  std::set<std::string> examples;
+
+  for(int index : order)
+  {
+    std::string example = std::to_string(classes[index]);
+
+    for(int feat : features[index])
+      example += "," + std::to_string(feat);
+
+    if(examples.count(example))
+      continue;
+
+    examples.insert(example);
+    new_order.push_back(index);
+  }
+
+  order = new_order;
+}
+

maca_trans_parser/src/maca_tm_train.c

@@ -158,6 +158,7 @@ void train_nn(context * ctx)
     Fann_file fann_train(ctx->fann_filename, classif_nb);
     Fann_file fann_dev(ctx->fann_dev_filename, classif_nb);
 
+    fann_train.delete_duplicates();
     fann_train.over_sample(0.0); //The ratio between the nb of examples of the majority class and of the minority class will not be inferior to this value
 
     auto first_example_batch = fann_train.get_batch(1);