Fixed dynamic oracle for tokenization transitions

reading_machine/src/Transition.cpp

@@ -197,9 +197,15 @@ void Transition::initIgnoreChar()
 {
   sequence.emplace_back(Action::ignoreCurrentCharacter());
 
-  costDynamic = [](const Config &)
+  costDynamic = [](const Config & config)
   {
-    return 0;
+    auto letter = fmt::format("{}", config.getLetter(config.getCharacterIndex()));
+    auto goldWord = util::splitAsUtf8(config.getConst("FORM", config.getWordIndex(), 0).get());
+    auto curWord = util::splitAsUtf8(config.getAsFeature("FORM", config.getWordIndex()).get());
+    if (curWord.size() >= goldWord.size())
+      return 0;
+
+    return goldWord[curWord.size()] == letter ? 1 : 0;
   };
 
   costStatic = costDynamic;
@@ -231,6 +237,9 @@ void Transition::initAddCharToWord()
     if (!config.hasCharacter(config.getCharacterIndex()))
       return std::numeric_limits<int>::max();
 
+    if (!config.isToken(config.getWordIndex()))
+      return std::numeric_limits<int>::max();
+
     auto letter = fmt::format("{}", config.getLetter(config.getCharacterIndex()));
     auto & goldWord = config.getConst("FORM", config.getWordIndex(), 0).get();
     auto & curWord = config.getAsFeature("FORM", config.getWordIndex()).get();