macaon_decode.cpp

/*Copyright (c) 2019 Alexis Nasr && Franck Dary

 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:i

 The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.*/
/// @file macaon_decode.cpp
/// @author Franck Dary
/// @version 1.0
/// @date 2018-08-07

#include <cstdio>
#include <cstdlib>
#include <boost/program_options.hpp>
#include "BD.hpp"
#include "Config.hpp"
#include "TransitionMachine.hpp"
#include "Decoder.hpp"

namespace po = boost::program_options;

/// @brief Get the list of mandatory and optional program arguments.
///
/// @return The lists.
po::options_description getOptionsDescription()
{
  po::options_description desc("Command-Line Arguments ");

  po::options_description req("Required");
  req.add_options()
    ("expName", po::value<std::string>()->required(),
      "Name of this experiment")
    ("tm", po::value<std::string>()->required(),
      "File describing the Tape Machine to use")
    ("bd", po::value<std::string>()->required(),
      "BD file that describes the multi-tapes buffer")
    ("mcd", po::value<std::string>()->required(),
      "MCD file that describes the input")
    ("input,I", po::value<std::string>()->required(),
      "Input file formated according to the mcd, or rawInput");

  po::options_description opt("Optional");
  opt.add_options()
    ("help,h", "Produce this help message")
    ("debug,d", "Print infos on stderr")
    ("delayedOutput", "Print the output only at the end")
    ("showActions", "Print actions predicted by each classifier")
    ("noNeuralNetwork", "Don't use any neural network, useful to speed up debug")
    ("dicts", po::value<std::string>()->default_value(""),
      "The .dict file describing all the dictionaries to be used in the experiement. By default the filename specified in the .tm file will be used")
    ("featureModels", po::value<std::string>()->default_value(""),
      "For each classifier, specify what .fm (feature model) file to use. By default the filename specified in the .cla file will be used. Example : --featureModel Parser=parser.fm,Tagger=tagger.fm")

    ("printEntropy", "Print entropy for each sequence")
    ("sequenceDelimiterTape", po::value<std::string>()->default_value("EOS"),
      "The name of the buffer's tape that contains the delimiter token for a sequence")
    ("sequenceDelimiter", po::value<std::string>()->default_value("1"),
      "The value of the token that act as a delimiter for sequences")
    ("showFeatureRepresentation", po::value<int>()->default_value(0),
      "For each state of the Config, show its feature representation")
    ("readSize", po::value<int>()->default_value(0),
      "The number of lines of input that will be read and stored in memory at once.")
    ("dictCapacity", po::value<int>()->default_value(50000),
      "The maximal size of each Dict (number of differents embeddings).")
    ("maxStackSize", po::value<int>()->default_value(200),
      "The maximal size of the stack (dependency parsing).")
    ("interactive", po::value<bool>()->default_value(true),
      "Is the shell interactive ? Display advancement informations")
    ("rawInput", "Is the input file raw text ?")
    ("tapeToMask", po::value<std::string>()->default_value("FORM"),
      "The name of the Tape for which some of the elements will be masked.")
    ("maskRate", po::value<float>()->default_value(0.0),
      "The rate of elements of the Tape that will be masked.")
    ("lang", po::value<std::string>()->default_value("fr"),
      "Language you are working with");

  po::options_description analysis("Error analysis related options");
  analysis.add_options()
    ("errorAnalysis", "Print an analysis of errors")
    ("meanEntropy", "Print the mean entropy for error types")
    ("onlyPrefixes", "Only uses the prefixes of error categories")
    ("printOutputEntropy", "For each cell of the output, print the associated entropy")
    ("nbErrorsToShow", po::value<int>()->default_value(10),
      "Display only the X most common errors")
    ("classifier", po::value<std::string>()->default_value(""),
      "Name of the monitored classifier, if not specified monitor everyone");

  po::options_description beam("Beam search related options");
  beam.add_options()
    ("beamSize", po::value<int>()->default_value(1),
      "Number of nodes to explore for each depth of the tree of all the possible configurations")
    ("nbChilds", po::value<int>()->default_value(3),
      "Number of childs to consider for each explored node");

  desc.add(req).add(opt).add(analysis).add(beam);

  return desc;
}

/// @brief Store the program arguments inside a variables_map
///
/// @param od The description of all the possible options.
/// @param argc The number of arguments given to this program.
/// @param argv The values of arguments given to this program.
///
/// @return The variables map
po::variables_map checkOptions(po::options_description & od, int argc, char ** argv)
{
  po::variables_map vm;

  try {po::store(po::parse_command_line(argc, argv, od), vm);}
  catch(std::exception& e)
  {
    std::cerr << "Error: " << e.what() << "\n";
    od.print(std::cerr);
    exit(1);
  }

  if (vm.count("help"))
  {
    std::cout << od << "\n";
    exit(0);
  }

  try {po::notify(vm);}
  catch(std::exception& e)
  {
    std::cerr << "Error: " << e.what() << "\n";
    od.print(std::cerr);
    exit(1);
  }

  return vm;
}

/// @brief Uses a pre-trained TransitionMachine to predict and add information to a structured input file.
///
/// @param argc The number of arguments given to this program.
/// @param argv[] Array of arguments given to this program.
///
/// @return 0 if there was no crash.
int main(int argc, char * argv[])
{
  auto od = getOptionsDescription();

  po::variables_map vm = checkOptions(od, argc, argv);

  ProgramParameters::expName = vm["expName"].as<std::string>();
  ProgramParameters::tmName = vm["tm"].as<std::string>();
  ProgramParameters::bdName = vm["bd"].as<std::string>();
  ProgramParameters::input = vm["input"].as<std::string>();
  ProgramParameters::mcdName = vm["mcd"].as<std::string>();
  ProgramParameters::debug = vm.count("debug") == 0 ? false : true;
  ProgramParameters::delayedOutput = vm.count("delayedOutput") == 0 ? false : true;
  ProgramParameters::showActions = vm.count("showActions") == 0 ? false : true;
  ProgramParameters::noNeuralNetwork = vm.count("noNeuralNetwork") == 0 ? false : true;
  ProgramParameters::interactive = vm["interactive"].as<bool>();
  ProgramParameters::rawInput = vm.count("rawInput") == 0 ? false : true;
  ProgramParameters::errorAnalysis = vm.count("errorAnalysis") == 0 ? false : true;
  ProgramParameters::nbErrorsToShow = vm["nbErrorsToShow"].as<int>();
  ProgramParameters::meanEntropy = vm.count("meanEntropy") == 0 ? false : true;
  ProgramParameters::onlyPrefixes = vm.count("onlyPrefixes") == 0 ? false : true;
  ProgramParameters::printOutputEntropy = vm.count("printOutputEntropy") == 0 ? false : true;
  ProgramParameters::dicts = vm["dicts"].as<std::string>();
  ProgramParameters::printEntropy = vm.count("printEntropy") == 0 ? false : true;
  ProgramParameters::lang = vm["lang"].as<std::string>();
  ProgramParameters::sequenceDelimiterTape = vm["sequenceDelimiterTape"].as<std::string>();
  ProgramParameters::sequenceDelimiter = vm["sequenceDelimiter"].as<std::string>();
  ProgramParameters::showFeatureRepresentation = vm["showFeatureRepresentation"].as<int>();
  ProgramParameters::tapeSize = ProgramParameters::rawInput ? 200000 : util::getNbLines(ProgramParameters::input);
  ProgramParameters::readSize = vm["readSize"].as<int>();
  if (ProgramParameters::readSize == 0)
    ProgramParameters::readSize = ProgramParameters::tapeSize;
  ProgramParameters::dictCapacity = vm["dictCapacity"].as<int>();
  ProgramParameters::maxStackSize = vm["maxStackSize"].as<int>();
  ProgramParameters::beamSize = vm["beamSize"].as<int>();
  ProgramParameters::nbChilds = vm["nbChilds"].as<int>();
  ProgramParameters::tapeToMask = vm["tapeToMask"].as<std::string>();
  ProgramParameters::maskRate = vm["maskRate"].as<float>();
  ProgramParameters::optimizer = "none";
  std::string featureModels = vm["featureModels"].as<std::string>();
  if (!featureModels.empty())
  {
    auto byClassifiers = util::split(featureModels, ',');
    for (auto & classifier : byClassifiers)
    {
      auto parts = util::split(classifier, '=');
      if (parts.size() != 2)
      {
        fprintf(stderr, "ERROR (%s) : wrong format for argument of option featureModels. Aborting.\n", ERRINFO);
        exit(1);
      }
      ProgramParameters::featureModelByClassifier[parts[0]] = parts[1];
    }
  }

  const char * MACAON_DIR = std::getenv("MACAON_DIR");
  std::string slash = "/";
  ProgramParameters::expPath = MACAON_DIR + slash + ProgramParameters::lang + slash + "bin/" + ProgramParameters::expName + slash;

  ProgramParameters::tmFilename = ProgramParameters::expPath + ProgramParameters::tmName;
  ProgramParameters::bdFilename = ProgramParameters::expPath + ProgramParameters::bdName;
  ProgramParameters::mcdFilename = ProgramParameters::mcdName;

  TransitionMachine tapeMachine(false);

  BD bd(ProgramParameters::bdFilename, ProgramParameters::mcdFilename);
  Config config(bd, ProgramParameters::input);

  Decoder decoder(tapeMachine, config);

  decoder.decode();

  return 0;
}