diff --git a/trainer/src/TrainInfos.cpp b/trainer/src/TrainInfos.cpp
index ab7e10618ff568c28431a75f056224bed7f6af6b..ed52f748b4a432981261dd3dab05d3c9f36059df 100644
--- a/trainer/src/TrainInfos.cpp
+++ b/trainer/src/TrainInfos.cpp
@@ -212,6 +212,8 @@ void TrainInfos::computeTrainScores(Config & c)
{
if (it.first == "Parser")
addTrainScore(it.first, (scoresFloat["UAS"] + scoresFloat["LAS"] + scoresFloat["Sentences"]) / 3);
+ else if (it.first == "Segmenter")
+ addTrainScore(it.first, scoresFloat["Sentences"]);
else if (it.first == "Tokenizer")
addTrainScore(it.first, scoresFloat["Tokens"]);
else if (it.first == "Tagger")
@@ -268,6 +270,8 @@ void TrainInfos::computeDevScores(Config & c)
{
if (it.first == "Parser")
addDevScore(it.first, (scoresFloat["UAS"] + scoresFloat["LAS"] + scoresFloat["Sentences"]) / 3);
+ else if (it.first == "Segmenter")
+ addDevScore(it.first, scoresFloat["Sentences"]);
else if (it.first == "Tokenizer")
addDevScore(it.first, scoresFloat["Tokens"]);
else if (it.first == "Tagger")
diff --git a/transition_machine/include/ActionBank.hpp b/transition_machine/include/ActionBank.hpp
index 695068cb19811094c0b42260cdc0c964f69a7aba..a42c8ad2103c4dd0f11f04e4aba0cfe5559741f7 100644
--- a/transition_machine/include/ActionBank.hpp
+++ b/transition_machine/include/ActionBank.hpp
@@ -60,10 +60,11 @@ class ActionBank
/// @param config The Config simpleBufferWrite would be called with.
/// @param tapeName The name of the tape simpleBufferWrite would be called with.
/// @param relativeIndex The relative index simpleBufferWrite would be called with.
+ /// @param checkEmptyness If true, check if cell that will be written to is empty.
///
/// @return Whether or not a call to simpleBufferWrite is possible.
static bool simpleBufferWriteAppliable(Config & config,
- const std::string & tapeName, int relativeIndex);
+ const std::string & tapeName, int relativeIndex, bool checkEmptyness);
/// @brief Whether or not the transform rule is appliable.
///
@@ -97,9 +98,10 @@ class ActionBank
/// \param tapeName The tape we will write to
/// \param value The value we will write
/// \param relativeIndex The write index relative to the buffer's head
+ /// \param checkEmptyness If true, check if the cell that will be written to is empty
///
/// \return A BasicAction doing all of that
- static Action::BasicAction bufferWrite(std::string tapeName, std::string value, int relativeIndex);
+ static Action::BasicAction bufferWrite(std::string tapeName, std::string value, int relativeIndex, bool checkEmptyness);
/// \brief Append a string to a cell in the buffer
///
@@ -176,9 +178,10 @@ class ActionBank
/// \param tapeName The tape we will write to
/// \param value The value we will write
/// \param stackIndex The stack index of the buffer index
+ /// \param checkEmptyness If true, check if the cell it will write to is empty
///
/// \return A BasicAction doing all of that
- static Action::BasicAction stackWrite(std::string tapeName, std::string value, int stackIndex);
+ static Action::BasicAction stackWrite(std::string tapeName, std::string value, int stackIndex, bool checkEmptyness);
/// \brief Push the head of the buffer into the stack
///
diff --git a/transition_machine/src/ActionBank.cpp b/transition_machine/src/ActionBank.cpp
index 7dedf72aa5c146cb5333b8c1852d2cdcf8b2a511..f4a9e42d02a3129651735dab2b5aa1a3641e8d22 100644
--- a/transition_machine/src/ActionBank.cpp
+++ b/transition_machine/src/ActionBank.cpp
@@ -130,7 +130,7 @@ Action::BasicAction ActionBank::rawInputBeginsWith(std::string word)
return basicAction;
}
-Action::BasicAction ActionBank::bufferWrite(std::string tapeName, std::string value, int relativeIndex)
+Action::BasicAction ActionBank::bufferWrite(std::string tapeName, std::string value, int relativeIndex, bool checkEmptyness)
{
auto apply = [tapeName, value, relativeIndex](Config & c, Action::BasicAction &)
{
@@ -140,9 +140,9 @@ Action::BasicAction ActionBank::bufferWrite(std::string tapeName, std::string va
{
simpleBufferWrite(c, tapeName, "", relativeIndex);
};
- auto appliable = [tapeName, relativeIndex](Config & c, Action::BasicAction &)
+ auto appliable = [tapeName, relativeIndex, checkEmptyness](Config & c, Action::BasicAction &)
{
- return simpleBufferWriteAppliable(c, tapeName, relativeIndex);
+ return simpleBufferWriteAppliable(c, tapeName, relativeIndex, checkEmptyness);
};
Action::BasicAction basicAction =
{Action::BasicAction::Type::Write, value, apply, undo, appliable};
@@ -150,7 +150,7 @@ Action::BasicAction ActionBank::bufferWrite(std::string tapeName, std::string va
return basicAction;
}
-Action::BasicAction ActionBank::stackWrite(std::string tapeName, std::string value, int stackIndex)
+Action::BasicAction ActionBank::stackWrite(std::string tapeName, std::string value, int stackIndex, bool checkEmptyness)
{
auto apply = [tapeName, value, stackIndex](Config & c, Action::BasicAction &)
{
@@ -164,13 +164,13 @@ Action::BasicAction ActionBank::stackWrite(std::string tapeName, std::string val
int relativeIndex = bufferIndex - c.getHead();
simpleBufferWrite(c, tapeName, "", relativeIndex);
};
- auto appliable = [tapeName, stackIndex](Config & c, Action::BasicAction &)
+ auto appliable = [tapeName, stackIndex, checkEmptyness](Config & c, Action::BasicAction &)
{
if (!c.stackHasIndex(stackIndex))
return false;
int bufferIndex = c.stackGetElem(stackIndex);
int relativeIndex = bufferIndex - c.getHead();
- return simpleBufferWriteAppliable(c, tapeName, relativeIndex);
+ return simpleBufferWriteAppliable(c, tapeName, relativeIndex, checkEmptyness);
};
Action::BasicAction basicAction =
@@ -441,9 +441,51 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
int relativeIndex = std::stoi(object[1]);
if (object[0] == "b")
- sequence.emplace_back(bufferWrite(tapeName, value, relativeIndex));
+ sequence.emplace_back(bufferWrite(tapeName, value, relativeIndex, true));
else if (object[0] == "s")
- sequence.emplace_back(stackWrite(tapeName, value, relativeIndex));
+ sequence.emplace_back(stackWrite(tapeName, value, relativeIndex, true));
+ }
+ else if(std::string(b1) == "REWRITE")
+ {
+ if (sscanf(name.c_str(), "%s %s %s %s", b1, b4, b2, b3) != 4)
+ invalidNameAndAbort(ERRINFO);
+
+ std::string tapeName(b2);
+ std::string value(b3);
+ auto object = util::split(b4, '.');
+
+ if (object.size() != 2)
+ invalidNameAndAbort(ERRINFO);
+
+ int relativeIndex = std::stoi(object[1]);
+
+ if (object[0] == "b")
+ sequence.emplace_back(bufferWrite(tapeName, value, relativeIndex, false));
+ else if (object[0] == "s")
+ sequence.emplace_back(stackWrite(tapeName, value, relativeIndex, false));
+ }
+ else if (std::string(b1) == "SEGMENT")
+ {
+ sequence.emplace_back(bufferWrite(ProgramParameters::sequenceDelimiterTape, ProgramParameters::sequenceDelimiter, 0, false));
+ // Update the IDs of the words in the new sentence
+ {
+ auto apply = [](Config & c, Action::BasicAction &)
+ {
+ c.setEosTouched();
+ c.updateIdsInSequence();
+ };
+ auto undo = [](Config &, Action::BasicAction &)
+ {
+ };
+ auto appliable = [](Config &, Action::BasicAction &)
+ {
+ return true;
+ };
+ Action::BasicAction basicAction =
+ {Action::BasicAction::Type::Write, "", apply, undo, appliable};
+
+ sequence.emplace_back(basicAction);
+ }
}
else if(std::string(b1) == "TOLOWER" || std::string(b1) == "TOUPPER")
{
@@ -508,7 +550,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
auto splits = util::split(name);
for(int i = startRelIndex; i <= endRelIndex; i++)
- sequence.emplace_back(bufferWrite(tapeName, splits[4+i-startRelIndex], i));
+ sequence.emplace_back(bufferWrite(tapeName, splits[4+i-startRelIndex], i, true));
}
else if(std::string(b1) == "RULE")
{
@@ -563,7 +605,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
auto undo = [](Config & c, Action::BasicAction &)
{simpleBufferWrite(c, "ID", std::string(""), 0);};
auto appliable = [](Config & c, Action::BasicAction &)
- {return simpleBufferWriteAppliable(c, "ID", 0);};
+ {return simpleBufferWriteAppliable(c, "ID", 0, true);};
Action::BasicAction basicAction =
{Action::BasicAction::Type::Write, "", apply, undo, appliable};
@@ -601,7 +643,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
for (unsigned int i = 0; i < splited.size(); i++)
{
- sequence.emplace_back(bufferWrite("FORM", splited[i], i));
+ sequence.emplace_back(bufferWrite("FORM", splited[i], i, true));
int splitedSize = (int)splited.size();
auto apply = [i, splitedSize](Config & c, Action::BasicAction &)
@@ -609,7 +651,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
auto undo = [i](Config & c, Action::BasicAction &)
{simpleBufferWrite(c, "ID", std::string(""), i);};
auto appliable = [i](Config & c, Action::BasicAction &)
- {return simpleBufferWriteAppliable(c, "ID", i);};
+ {return simpleBufferWriteAppliable(c, "ID", i, true);};
Action::BasicAction basicAction =
{Action::BasicAction::Type::Write, "", apply, undo, appliable};
@@ -704,7 +746,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
return false;
}
- return simpleBufferWriteAppliable(c, "GOV", s0-b0);
+ return simpleBufferWriteAppliable(c, "GOV", s0-b0, true);
};
Action::BasicAction basicAction =
{Action::BasicAction::Type::Write, "", apply, undo, appliable};
@@ -731,7 +773,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
return false;
int b0 = c.getHead();
int s0 = c.stackTop();
- return simpleBufferWriteAppliable(c, "LABEL", s0-b0);
+ return simpleBufferWriteAppliable(c, "LABEL", s0-b0, true);
};
Action::BasicAction basicAction2 =
{Action::BasicAction::Type::Write, b2, apply2, undo2, appliable2};
@@ -786,7 +828,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
return false;
}
- return simpleBufferWriteAppliable(c, "GOV", 0);
+ return simpleBufferWriteAppliable(c, "GOV", 0, true);
};
Action::BasicAction basicAction =
{Action::BasicAction::Type::Write, "", apply, undo, appliable};
@@ -805,7 +847,7 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
};
auto appliable2 = [](Config & c, Action::BasicAction &)
{
- return simpleBufferWriteAppliable(c, "LABEL", 0);
+ return simpleBufferWriteAppliable(c, "LABEL", 0, true);
};
Action::BasicAction basicAction2 =
{Action::BasicAction::Type::Write, b2, apply2, undo2, appliable2};
@@ -817,35 +859,50 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
}
else if(std::string(b1) == "EOS")
{
+ if (sscanf(name.c_str(), "%s %s", b1, b4) != 2)
+ invalidNameAndAbort(ERRINFO);
+
+ auto object = util::split(b4, '.');
+
+ if (object.size() != 2)
+ invalidNameAndAbort(ERRINFO);
+
+ int relativeIndex = std::stoi(object[1]);
+
// Puting the EOS tag on the last element of the sentence.
+ if (object[0] == "s")
{
- auto apply = [](Config & c, Action::BasicAction &)
+ auto apply = [relativeIndex](Config & c, Action::BasicAction &)
{
int b0 = c.getHead();
- int s0 = c.stackTop();
- simpleBufferWrite(c, ProgramParameters::sequenceDelimiterTape, ProgramParameters::sequenceDelimiter, s0-b0);
- c.setEosTouched();
+ int s = c.stackGetElem(relativeIndex);
+ simpleBufferWrite(c, ProgramParameters::sequenceDelimiterTape, ProgramParameters::sequenceDelimiter, s-b0);
};
- auto undo = [](Config & c, Action::BasicAction)
+ auto undo = [relativeIndex](Config & c, Action::BasicAction)
{
int b0 = c.getHead();
- int s0 = c.stackTop();
- simpleBufferWrite(c, ProgramParameters::sequenceDelimiterTape, "", s0-b0);
+ int s = c.stackGetElem(relativeIndex);
+ simpleBufferWrite(c, ProgramParameters::sequenceDelimiterTape, "", s-b0);
};
- auto appliable = [](Config & c, Action::BasicAction &)
+ auto appliable = [relativeIndex](Config & c, Action::BasicAction &)
{
- return !c.isFinal() && !c.stackEmpty() && c.getTape(ProgramParameters::sequenceDelimiterTape).getHyp(c.stackTop()-c.getHead()) != ProgramParameters::sequenceDelimiter;
+ return !c.isFinal() && !c.stackEmpty() && c.getTape(ProgramParameters::sequenceDelimiterTape).getHyp(c.stackGetElem(relativeIndex)-c.getHead()) != ProgramParameters::sequenceDelimiter;
};
Action::BasicAction basicAction =
{Action::BasicAction::Type::Write, "", apply, undo, appliable};
sequence.emplace_back(basicAction);
}
+ else if (object[0] == "b")
+ {
+ sequence.emplace_back(bufferWrite(ProgramParameters::sequenceDelimiterTape, ProgramParameters::sequenceDelimiter, relativeIndex, false));
+ }
// Update the IDs of the words in the new sentence
{
auto apply = [](Config & c, Action::BasicAction &)
{
+ c.setEosTouched();
c.updateIdsInSequence();
};
auto undo = [](Config &, Action::BasicAction &)
@@ -864,115 +921,121 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
// Chosing root of the sentence and attaching floating words to it.
{
auto apply = [](Config & c, Action::BasicAction & ba)
+ {
+ if (!c.hasTape("GOV"))
+ return;
+
+ ba.data = "";
+ auto & govs = c.getTape("GOV");
+ auto & ids = c.getTape("ID");
+ int b0 = c.getHead();
+ int rootIndex = -1;
+ for (int i = c.stackSize()-1; i >= 0; i--)
{
- ba.data = "";
- auto & govs = c.getTape("GOV");
- auto & ids = c.getTape("ID");
- int b0 = c.getHead();
- int rootIndex = -1;
- for (int i = c.stackSize()-1; i >= 0; i--)
+ auto s = c.stackGetElem(i);
+ if (c.rawInputHeadIndex > 0)
{
- auto s = c.stackGetElem(i);
- if (c.rawInputHeadIndex > 0)
- {
- if (util::split(ids.getHyp(s-b0), '-').size() > 1)
- continue;
- if (util::split(ids.getHyp(s-b0), '.').size() > 1)
- continue;
- }
- else
- {
- if (util::split(ids.getRef(s-b0), '-').size() > 1)
- continue;
- if (util::split(ids.getRef(s-b0), '.').size() > 1)
- continue;
- }
-
- if (govs.getHyp(s-b0).empty() || govs.getHyp(s-b0) == "0")
- {
- if (rootIndex == -1)
- rootIndex = s;
- else
- {
- simpleBufferWrite(c, "GOV", std::to_string(rootIndex-s), s-b0);
- simpleBufferWrite(c, "LABEL", "_", s-b0);
- ba.data += "+"+std::to_string(s-b0);
- }
- }
+ if (util::split(ids.getHyp(s-b0), '-').size() > 1)
+ continue;
+ if (util::split(ids.getHyp(s-b0), '.').size() > 1)
+ continue;
+ }
+ else
+ {
+ if (util::split(ids.getRef(s-b0), '-').size() > 1)
+ continue;
+ if (util::split(ids.getRef(s-b0), '.').size() > 1)
+ continue;
}
- if (rootIndex == -1)
+ if (govs.getHyp(s-b0).empty() || govs.getHyp(s-b0) == "0")
{
- if (c.stackEmpty())
+ if (rootIndex == -1)
+ rootIndex = s;
+ else
{
- c.printForDebug(stderr);
- fprintf(stderr, "ERROR (%s) : no suitable candidate for root. Aborting.\n", ERRINFO);
- exit(1);
+ simpleBufferWrite(c, "GOV", std::to_string(rootIndex-s), s-b0);
+ simpleBufferWrite(c, "LABEL", "_", s-b0);
+ ba.data += "+"+std::to_string(s-b0);
}
+ }
+ }
- rootIndex = c.stackGetElem(c.stackSize()-1);
+ if (rootIndex == -1)
+ {
+ if (c.stackEmpty())
+ {
+ c.printForDebug(stderr);
+ fprintf(stderr, "ERROR (%s) : no suitable candidate for root. Aborting.\n", ERRINFO);
+ exit(1);
}
- simpleBufferWrite(c, "GOV", "0", rootIndex-b0);
- simpleBufferWrite(c, "LABEL", "root", rootIndex-b0);
+ rootIndex = c.stackGetElem(c.stackSize()-1);
+ }
+
+ simpleBufferWrite(c, "GOV", "0", rootIndex-b0);
+ simpleBufferWrite(c, "LABEL", "root", rootIndex-b0);
+
+ // Attaching floating words to new root
+ int sentenceEnd = b0;
+ auto & eos = c.getTape(ProgramParameters::sequenceDelimiterTape);
+ while (sentenceEnd >= 0 && eos.getHyp(sentenceEnd-b0) != ProgramParameters::sequenceDelimiter)
+ sentenceEnd--;
+ int sentenceStart = std::max(0,sentenceEnd-1);
+ while (sentenceStart >= 0 && eos.getHyp(sentenceStart-b0) != ProgramParameters::sequenceDelimiter)
+ sentenceStart--;
+ sentenceStart++;
- // Attaching floating words to new root
- int sentenceEnd = b0;
- auto & eos = c.getTape(ProgramParameters::sequenceDelimiterTape);
- while (sentenceEnd >= 0 && eos.getHyp(sentenceEnd-b0) != ProgramParameters::sequenceDelimiter)
- sentenceEnd--;
- int sentenceStart = std::max(0,sentenceEnd-1);
- while (sentenceStart >= 0 && eos.getHyp(sentenceStart-b0) != ProgramParameters::sequenceDelimiter)
- sentenceStart--;
- sentenceStart++;
+ if (sentenceEnd < 0)
+ {
+ sentenceStart = 0;
+ sentenceEnd = eos.hypSize()-1;
+ }
- if (sentenceEnd < 0)
+ for (int i = sentenceStart; i <= sentenceEnd; i++)
+ {
+ if (c.rawInputHeadIndex > 0)
{
- sentenceStart = 0;
- sentenceEnd = eos.hypSize()-1;
+ if (util::split(ids.getHyp(i-b0), '-').size() > 1)
+ continue;
+ if (util::split(ids.getHyp(i-b0), '.').size() > 1)
+ continue;
}
-
- for (int i = sentenceStart; i <= sentenceEnd; i++)
+ else
{
- if (c.rawInputHeadIndex > 0)
- {
- if (util::split(ids.getHyp(i-b0), '-').size() > 1)
- continue;
- if (util::split(ids.getHyp(i-b0), '.').size() > 1)
- continue;
- }
- else
- {
- if (util::split(ids.getRef(i-b0), '-').size() > 1)
- continue;
- if (util::split(ids.getRef(i-b0), '.').size() > 1)
- continue;
- }
+ if (util::split(ids.getRef(i-b0), '-').size() > 1)
+ continue;
+ if (util::split(ids.getRef(i-b0), '.').size() > 1)
+ continue;
+ }
- if (govs.getHyp(i-b0).empty())
- {
- simpleBufferWrite(c, "GOV", std::to_string(rootIndex-i), i-b0);
- simpleBufferWrite(c, "LABEL", "_", i-b0);
- ba.data += "+"+std::to_string(i-b0);
- }
+ if (govs.getHyp(i-b0).empty())
+ {
+ simpleBufferWrite(c, "GOV", std::to_string(rootIndex-i), i-b0);
+ simpleBufferWrite(c, "LABEL", "_", i-b0);
+ ba.data += "+"+std::to_string(i-b0);
}
+ }
- // Delete the arcs from the previous sentence to the new sentence
- for (int i = b0; i > c.stackTop(); i--)
+ // Delete the arcs from the previous sentence to the new sentence
+ for (int i = b0; i > c.stackTop(); i--)
+ {
+ try
{
- try
+ if (std::stoi(govs[i-b0])+i <= c.stackTop())
{
- if (std::stoi(govs[i-b0])+i <= c.stackTop())
- {
- simpleBufferWrite(c, "GOV", "", i-b0);
- simpleBufferWrite(c, "LABEL", "", i-b0);
- }
+ simpleBufferWrite(c, "GOV", "", i-b0);
+ simpleBufferWrite(c, "LABEL", "", i-b0);
}
- catch (std::exception &) {continue;}
}
- };
+ catch (std::exception &) {continue;}
+ }
+ };
auto undo = [](Config & c, Action::BasicAction & ba)
{
+ if (!c.hasTape("GOV"))
+ return;
+
auto & govs = c.getTape("GOV");
int b0 = c.getHead();
for (int i = c.stackSize()-1; i >= 0; i--)
@@ -996,6 +1059,9 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
};
auto appliable = [](Config & c, Action::BasicAction &)
{
+ if (!c.hasTape("GOV"))
+ return true;
+
return !c.stackEmpty();
};
Action::BasicAction basicAction =
@@ -1020,6 +1086,9 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
};
auto undo = [](Config & c, Action::BasicAction & ba)
{
+ if (ba.data.empty())
+ return;
+
auto elems = util::split(ba.data);
for (auto elem : elems)
if (!elem.empty())
@@ -1028,6 +1097,9 @@ std::vector<Action::BasicAction> ActionBank::str2sequence(const std::string & na
};
auto appliable = [](Config & c, Action::BasicAction &)
{
+ if (!c.hasTape("GOV"))
+ return true;
+
return !c.isFinal() && !c.stackEmpty();
};
Action::BasicAction basicAction =
@@ -1166,19 +1238,22 @@ void ActionBank::simpleBufferWrite(Config & config, const std::string & tapeName
}
bool ActionBank::simpleBufferWriteAppliable(Config & config,
- const std::string & tapeName, int relativeIndex)
+ const std::string & tapeName, int relativeIndex, bool checkEmptyness)
{
auto & tape = config.getTape(tapeName);
int index = config.getHead() + relativeIndex;
+ if (!checkEmptyness)
+ return !(index < 0) && index < tape.size();
+
return !(index < 0) && index < tape.size() && tape.getHyp(relativeIndex).empty();
}
bool ActionBank::isRuleAppliable(Config & config,
const std::string & fromTapeName, const std::string & targetTapeName, int relativeIndex, const std::string & rule)
{
- if (!simpleBufferWriteAppliable(config, targetTapeName, relativeIndex))
+ if (!simpleBufferWriteAppliable(config, targetTapeName, relativeIndex, true))
return false;
return util::ruleIsAppliable(config.getTape(fromTapeName)[relativeIndex], rule);
diff --git a/transition_machine/src/Config.cpp b/transition_machine/src/Config.cpp
index 7b5a79145af8e0ef84a27c0520f4719a656af5ac..38492f13d5882e884bf8b3c9f64693778f88a05c 100644
--- a/transition_machine/src/Config.cpp
+++ b/transition_machine/src/Config.cpp
@@ -944,6 +944,9 @@ void Config::updateIdsInSequence()
while (sentenceEnd >= 0 && eos.getHyp(sentenceEnd-getHead()) != ProgramParameters::sequenceDelimiter)
sentenceEnd--;
+ if (endOfTapes())
+ sentenceEnd = getHead();
+
while (sentenceEnd >= ids.size())
sentenceEnd--;
diff --git a/transition_machine/src/Oracle.cpp b/transition_machine/src/Oracle.cpp
index 7d1b27f060321d174e7b29dc393fca7ec11a7fc3..288faa0975ba1e31a084c1670d57c4413391cb05 100644
--- a/transition_machine/src/Oracle.cpp
+++ b/transition_machine/src/Oracle.cpp
@@ -189,6 +189,25 @@ void Oracle::createDatabase()
return 0;
})));
+ str2oracle.emplace("segmenter", std::unique_ptr<Oracle>(new Oracle(
+ [](Oracle *)
+ {
+ },
+ [](Config &, Oracle *)
+ {
+ fprintf(stderr, "ERROR (%s) : getAction called on Oracle of trainable Classifier. Aborting.\n", ERRINFO);
+ exit(1);
+
+ return std::string("");
+ },
+ [](Config & c, Oracle *, const std::string & action)
+ {
+ if (c.getTape(ProgramParameters::sequenceDelimiterTape).getRef(0) == ProgramParameters::sequenceDelimiter)
+ return action == "EOS b.0" ? 0 : 1;
+
+ return action != "EOS b.0" ? 0 : 1;
+ })));
+
str2oracle.emplace("tagger", std::unique_ptr<Oracle>(new Oracle(
[](Oracle *)
{
@@ -359,6 +378,22 @@ void Oracle::createDatabase()
return 0;
})));
+ str2oracle.emplace("strategy_segmenter", std::unique_ptr<Oracle>(new Oracle(
+ [](Oracle *)
+ {
+ },
+ [](Config & c, Oracle *)
+ {
+ if (c.pastActions.size() == 0)
+ return std::string("MOVE segmenter 0");
+
+ return std::string("MOVE segmenter 1");
+ },
+ [](Config &, Oracle *, const std::string &)
+ {
+ return 0;
+ })));
+
str2oracle.emplace("strategy_tagger", std::unique_ptr<Oracle>(new Oracle(
[](Oracle *)
{
@@ -977,7 +1012,7 @@ void Oracle::createDatabase()
int stackHead = c.stackEmpty() ? 0 : c.stackTop();
if (head >= eos.size())
- return action == "EOS" ? 0 : 1;
+ return action == "EOS s.0" ? 0 : 1;
if (ids.getRef(0).empty())
{