Added strategy for tokenizer

transition_machine/src/Oracle.cpp

@@ -379,6 +379,45 @@ void Oracle::createDatabase()
     return 0;
   })));
 
+  str2oracle.emplace("strategy_tokenizer", std::unique_ptr<Oracle>(new Oracle(
+  [](Oracle *)
+  {
+  },
+  [](Config & c, Oracle *)
+  {
+    if (c.pastActions.size() == 0)
+      return std::string("MOVE tokenizer 0");
+
+    std::string previousState = util::noAccentLower(c.pastActions.getElem(0).first);
+    std::string previousAction = util::noAccentLower(c.pastActions.getElem(0).second.name);
+    std::string newState;
+    int movement = 0;
+
+    if (previousState == "signature")
+    {
+      newState = "tokenizer";
+      movement = 1;
+    }
+    else if (previousState == "tokenizer")
+    {
+      if (util::split(previousAction, ' ')[0] == "splitword" || util::split(previousAction, ' ')[0] == "endword")
+        newState = "signature";
+      else
+        newState = "tokenizer";
+
+      if (util::split(previousAction, ' ')[0] == "splitword")
+      {
+        int nbSplit = util::split(util::split(previousAction, ' ')[1], '@').size();
+        movement = nbSplit-1;
+      }
+    }
+    return "MOVE " + newState + " " + std::to_string(movement);
+  },
+  [](Config &, Oracle *, const std::string &)
+  {
+    return 0;
+  })));
+
   str2oracle.emplace("strategy_tokenizer,tagger", std::unique_ptr<Oracle>(new Oracle(
   [](Oracle *)
   {