diff --git a/decoder/src/Decoder.cpp b/decoder/src/Decoder.cpp index 55920a8f825ac301cb301e2423c4920e40f5aaae..1a70e995b6b7f15b9be78b4c2f4fc4df58b064fd 100644 --- a/decoder/src/Decoder.cpp +++ b/decoder/src/Decoder.cpp @@ -72,11 +72,7 @@ void Decoder::decode() { nbActionsInSequence++; - for (unsigned int i = 0; i < 2 && i < weightedActions.size(); i++) - { - auto it = weightedActions.begin() + i; - entropyAccumulator -= it->second.first - (it->second.first - weightedActions[0].second.first); - } + entropyAccumulator += Classifier::computeEntropy(weightedActions); if (config.head >= 1 && config.getTape(ProgramParameters::sequenceDelimiterTape)[config.head-1] != ProgramParameters::sequenceDelimiter) justFlipped = false; diff --git a/trainer/include/Trainer.hpp b/trainer/include/Trainer.hpp index 1ca9132747529d7715bbdcdfa243736ba737f715..045c8003615580be80cff9dbd58f9a88e8129837 100644 --- a/trainer/include/Trainer.hpp +++ b/trainer/include/Trainer.hpp @@ -59,7 +59,7 @@ class Trainer /// @brief Get the scores of the classifiers on the dev dataset. /// /// @return Map from each Classifier name to their score. - std::map<std::string, float> getScoreOnDev(); + std::map<std::string, std::pair<float, std::pair<float, float> > > getScoreOnDev(); public : diff --git a/trainer/src/Trainer.cpp b/trainer/src/Trainer.cpp index 3f43c060a84b6664f541365e4b6b58546783ec5d..b3b55b9702aebdd31e4f67b71e92b4729956a7d3 100644 --- a/trainer/src/Trainer.cpp +++ b/trainer/src/Trainer.cpp @@ -12,7 +12,7 @@ Trainer::Trainer(TransitionMachine & tm, BD & bd, Config & config, BD * devBD, C { } -std::map<std::string, float> Trainer::getScoreOnDev() +std::map<std::string, std::pair<float, std::pair<float, float> > > Trainer::getScoreOnDev() { if (!devConfig) return {}; @@ -25,6 +25,11 @@ std::map<std::string, float> Trainer::getScoreOnDev() if (ProgramParameters::debug) fprintf(stderr, "Computing score on dev set\n"); + int nbActionsInSequence = 0; + float entropyAccumulator = 0.0; + bool justFlipped = false; + std::vector<float> entropies; + while (!devConfig->isFinal()) { TransitionMachine::State * currentState = tm.getCurrentState(); @@ -82,12 +87,46 @@ std::map<std::string, float> Trainer::getScoreOnDev() TransitionMachine::Transition * transition = tm.getTransition(actionName); tm.takeTransition(transition); devConfig->moveHead(transition->headMvt); + + if (ProgramParameters::printEntropy) + { + nbActionsInSequence++; + + entropyAccumulator += Classifier::computeEntropy(weightedActions); + + if (devConfig->head >= 1 && devConfig->getTape(ProgramParameters::sequenceDelimiterTape)[devConfig->head-1] != ProgramParameters::sequenceDelimiter) + justFlipped = false; + + if ((devConfig->head >= 1 && devConfig->getTape(ProgramParameters::sequenceDelimiterTape)[devConfig->head-1] == ProgramParameters::sequenceDelimiter && !justFlipped)) + { + justFlipped = true; + entropyAccumulator /= nbActionsInSequence; + nbActionsInSequence = 0; + entropies.emplace_back(entropyAccumulator); + entropyAccumulator = 0.0; + } + } + } } - std::map<std::string, float> scores; + std::map<std::string, std::pair<float,std::pair<float,float> > > scores; for (auto & it : counts) - scores[it.first] = 100.0 * it.second.second / it.second.first; + { + scores[it.first].first = 100.0 * it.second.second / it.second.first; + if (ProgramParameters::printEntropy) + { + for (float f : entropies) + scores[it.first].second.first += f; + + scores[it.first].second.first /= entropies.size(); + + for (float f : entropies) + scores[it.first].second.second += (f-scores[it.first].second.first)*(f-scores[it.first].second.first); + + scores[it.first].second.second /= entropies.size(); + } + } return scores; } @@ -273,9 +312,9 @@ void Trainer::printScoresAndSave(FILE * output) { for (auto & it : devScores) { - if (bestScores.count(it.first) == 0 || bestScores[it.first] < it.second) + if (bestScores.count(it.first) == 0 || bestScores[it.first] < it.second.first) { - bestScores[it.first] = it.second; + bestScores[it.first] = it.second.first; saved[it.first] = true; } else @@ -314,8 +353,10 @@ void Trainer::printScoresAndSave(FILE * output) names.emplace_back(it.first); acc.emplace_back("accuracy"); train.emplace_back(": train(" + float2str(scores[it.first], "%.2f") + "%)"); - dev.emplace_back(devConfig ? "dev(" +float2str(devScores[it.first], "%.2f") + "%)" : ""); + dev.emplace_back(devConfig ? "dev(" +float2str(devScores[it.first].first, "%.2f") + "%)" : ""); savedStr.emplace_back(saved[it.first] ? "SAVED" : ""); + if (ProgramParameters::printEntropy) + savedStr.back() += " Entropy[" + float2str(devScores[it.first].second.first, "%.2f") + "\u00B1" + float2str(devScores[it.first].second.second, "%.2f") + "]"; } if (ProgramParameters::interactive) diff --git a/trainer/src/macaon_train.cpp b/trainer/src/macaon_train.cpp index 980b2d08b5f9105c46a228b1269ac5a124b6d95b..4122411df4e55d1c8b33e613386221c2be77a47d 100644 --- a/trainer/src/macaon_train.cpp +++ b/trainer/src/macaon_train.cpp @@ -40,6 +40,7 @@ po::options_description getOptionsDescription() opt.add_options() ("help,h", "Produce this help message") ("debug,d", "Print infos on stderr") + ("printEntropy", "Print mean entropy and standard deviation accross sequences") ("optimizer", po::value<std::string>()->default_value("amsgrad"), "The learning algorithm to use : amsgrad | adam | sgd") ("dev", po::value<std::string>()->default_value(""), @@ -237,6 +238,7 @@ int main(int argc, char * argv[]) ProgramParameters::bdName = vm["bd"].as<std::string>(); ProgramParameters::mcdName = vm["mcd"].as<std::string>(); ProgramParameters::debug = vm.count("debug") == 0 ? false : true; + ProgramParameters::printEntropy = vm.count("printEntropy") == 0 ? false : true; ProgramParameters::printTime = vm.count("printTime") == 0 ? false : true; ProgramParameters::trainName = vm["train"].as<std::string>(); ProgramParameters::devName = vm["dev"].as<std::string>(); diff --git a/transition_machine/include/Classifier.hpp b/transition_machine/include/Classifier.hpp index 1a3bbf6150a933954d8e122d2b31adc0f9d81c6c..667f6a1770e767630c90560ebf622aa68e0fe444 100644 --- a/transition_machine/include/Classifier.hpp +++ b/transition_machine/include/Classifier.hpp @@ -70,6 +70,13 @@ class Classifier /// @param threshold Print only the n most weighted Action. static void printWeightedActions(FILE * output, WeightedActions & wa, int threshold = 5); + /// @brief Get a measure of confidence. + /// + /// @param wa The actions with a score associated for each one. + /// + /// @return A measure of confidence. + static float computeEntropy(WeightedActions & wa); + /// @brief Convert a string to its corresponding Type. /// /// If s is unknown, the program aborts. diff --git a/transition_machine/src/Classifier.cpp b/transition_machine/src/Classifier.cpp index 5ff853b6bb0b69126727c5f7a56488dc61f2176c..106b189c20ab00942394d0d982f2ca50ac10c006 100644 --- a/transition_machine/src/Classifier.cpp +++ b/transition_machine/src/Classifier.cpp @@ -266,3 +266,16 @@ void Classifier::explainCostOfActions(FILE * output, Config & config) } } +float Classifier::computeEntropy(WeightedActions & wa) +{ + float entropy = 0.0; + + for (unsigned int i = 0; i < 2 && i < wa.size(); i++) + { + auto it = wa.begin() + i; + entropy -= it->second.first - (it->second.first - wa[0].second.first); + } + + return entropy; +} +