Added tokeparser tool

UD_any/tokeparser/lemmatizer_case.cla

+Name : Lemmatizer_Case
+Type : Prediction
+Oracle : lemma_case
+Feature Model : lemmatizer_case.fm
+Action Set : data/lemmatizer_case.as
+Topology : (100,RELU,0.1)
+Dynamic : yes

UD_any/tokeparser/lemmatizer_case.fm

+# Features classiques
+b.0#POS
+b.0#MORPHO
+b.0#FORM
+# Suffixes 
+b.0#FORM.PART.-4.-4
+b.0#FORM.PART.-3.-3
+b.0#FORM.PART.-2.-2
+b.0#FORM.PART.-1.-1
+b.0#FORM.PART.0.0
+b.0#FORM.PART.1.1
+b.0#FORM.PART.2.2
+b.0#FORM.PART.3.3

UD_any/tokeparser/lemmatizer_lookup.cla

+Name : Lemmatizer_Lookup
+Type : Information
+Oracle : lemma_lookup
+Oracle Filename : data/maca_trans_lemmatizer_exceptions.fplm

UD_any/tokeparser/lemmatizer_rules.cla

+Name : Lemmatizer_Rules
+Type : Prediction
+Oracle : lemma_rules
+Feature Model : lemmatizer_rules.fm
+Action Set : data/lemmatizer_rules.as
+Topology : (500,RELU,0.3)
+Dynamic : yes

UD_any/tokeparser/lemmatizer_rules.fm

+# Features classiques
+b.0#POS
+b.0#XPOS
+b.0#MORPHO
+b.0#FORM
+b.-1#POS
+b.-1#XPOS
+b.-1#MORPHO
+b.-1#FORM
+# Suffixes 
+b.0#FORM.PART.-4.-4
+b.0#FORM.PART.-3.-3
+b.0#FORM.PART.-2.-2
+b.0#FORM.PART.-1.-1
+b.0#FORM.PART.0.0
+b.0#FORM.PART.1.1
+b.0#FORM.PART.2.2
+b.0#FORM.PART.3.3

UD_any/tokeparser/machine.tm

+Name : Tokenizer, Tagger, Morpho, Lemmatizer and Parser Machine
+Dicts : tokeparser.dicts
+%CLASSIFIERS
+strategy strategy.cla
+tokenizer tokenizer.cla
+tagger tagger.cla
+morpho morpho.cla
+lemmatizer_lookup lemmatizer_lookup.cla
+lemmatizer_rules lemmatizer_rules.cla
+lemmatizer_case lemmatizer_case.cla
+parser parser.cla
+%STATES
+strategy strategy
+tokenizer tokenizer
+tagger tagger
+morpho morpho
+lemmatizer_lookup lemmatizer_lookup
+lemmatizer_rules lemmatizer_rules
+lemmatizer_case lemmatizer_case
+parser parser
+%TRANSITIONS
+strategy tokenizer MOVE tokenizer
+strategy tagger MOVE tagger
+strategy morpho MOVE morpho
+strategy lemmatizer_lookup MOVE lemmatizer_lookup
+strategy lemmatizer_rules MOVE lemmatizer_rules
+strategy lemmatizer_case MOVE lemmatizer_case
+strategy parser MOVE parser
+tokenizer strategy *
+tagger strategy *
+morpho strategy *
+lemmatizer_lookup strategy *
+lemmatizer_case strategy *
+lemmatizer_rules strategy *
+parser strategy *

UD_any/tokeparser/morpho.cla

+Name : Morpho
+Type : Prediction
+Oracle : morpho
+Feature Model : morpho.fm
+Action Set : data/morpho_parts.as
+Topology : (500,RELU,0.3)
+Dynamic : yes

UD_any/tokeparser/morpho.fm

+# Features classiques
+# FORM
+b.0#FORM.fasttext
+b.1#FORM.fasttext
+b.2#FORM.fasttext
+b.-1#FORM.fasttext
+b.-2#FORM.fasttext
+# POS
+b.0#POS
+b.-1#POS
+b.-2#POS
+b.-3#POS
+# XPOS
+b.0#XPOS
+b.-1#XPOS
+b.-2#XPOS
+b.-3#XPOS
+# MORPHO
+b.0#MORPHO
+b.-1#MORPHO
+b.-2#MORPHO
+# UPPERCASE
+b.0#FORM.U
+b.1#FORM.U
+# LENGTH
+b.0#FORM.LEN
+# SUFFIXES
+b.0#FORM.PART.-4.-4
+b.0#FORM.PART.-3.-3
+b.0#FORM.PART.-2.-2
+b.0#FORM.PART.-1.-1
+b.0#FORM.PART.0.0
+b.0#FORM.PART.1.1
+b.0#FORM.PART.2.2
+b.0#FORM.PART.3.3
+# ERROR CORRECTION
+#b.1#POS
+#b.1#MORPHO
+#b.2#POS
+#b.2#MORPHO
+#b.3#POS
+#b.3#MORPHO
+#b.4#POS
+#b.4#MORPHO

UD_any/tokeparser/normal.tm

+Name : Tagger, Morpho, Lemmatizer and Parser Machine
+Dicts : tagparser.dicts
+%CLASSIFIERS
+strategy strategy.cla
+tagger tagger.cla
+morpho morpho.cla
+lemmatizer_lookup lemmatizer_lookup.cla
+lemmatizer_rules lemmatizer_rules.cla
+lemmatizer_case lemmatizer_case.cla
+parser parser.cla
+%STATES
+strategy strategy
+tagger tagger
+morpho morpho
+lemmatizer_lookup lemmatizer_lookup
+lemmatizer_rules lemmatizer_rules
+lemmatizer_case lemmatizer_case
+parser parser
+%TRANSITIONS
+strategy tagger MOVE tagger
+strategy morpho MOVE morpho
+strategy lemmatizer_lookup MOVE lemmatizer_lookup
+strategy lemmatizer_rules MOVE lemmatizer_rules
+strategy lemmatizer_case MOVE lemmatizer_case
+strategy parser MOVE parser
+tagger strategy *
+morpho strategy *
+lemmatizer_lookup strategy *
+lemmatizer_case strategy *
+lemmatizer_rules strategy *
+parser strategy *

UD_any/tokeparser/parser.cla

+Name : Parser
+Type : Prediction
+Oracle : parser
+Feature Model : parser.fm
+Action Set : data/parser.as
+Topology : (500,RELU,0.3)
+Dynamic : yes

UD_any/tokeparser/parser.fm

+# Features classiques
+# FORM
+s.0#LEMMA.fasttext
+s.1#LEMMA.fasttext
+s.2#LEMMA.fasttext
+s.0.ldep#LEMMA.fasttext
+s.1.ldep#LEMMA.fasttext
+s.0.rdep#LEMMA.fasttext
+s.1.rdep#LEMMA.fasttext
+s.0.ldep.ldep#LEMMA.fasttext
+s.1.ldep.ldep#LEMMA.fasttext
+s.0.rdep.rdep#LEMMA.fasttext
+s.1.rdep.rdep#LEMMA.fasttext
+s.0.l2dep#LEMMA.fasttext
+s.1.l2dep#LEMMA.fasttext
+s.0.r2dep#LEMMA.fasttext
+s.1.r2dep#LEMMA.fasttext
+b.0#LEMMA.fasttext
+b.-1#LEMMA.fasttext
+b.-2#LEMMA.fasttext
+b.0.ldep#LEMMA.fasttext
+b.1#FORM.fasttext
+b.2#FORM.fasttext
+# POS
+s.0#POS
+s.1#POS
+s.2#POS
+s.0.ldep#POS
+s.1.ldep#POS
+s.0.rdep#POS
+s.1.rdep#POS
+s.0.ldep.ldep#POS
+s.1.ldep.ldep#POS
+s.0.rdep.rdep#POS
+s.1.rdep.rdep#POS
+s.0.l2dep#POS
+s.1.l2dep#POS
+s.0.r2dep#POS
+s.1.r2dep#POS
+b.0#POS
+b.-1#POS
+b.-2#POS
+b.0.ldep#POS
+b.-1.gov#POS
+s.0.gov#POS
+s.1.gov#POS
+# XPOS
+s.0#XPOS
+s.1#XPOS
+s.2#XPOS
+s.0.ldep#XPOS
+s.1.ldep#XPOS
+s.0.rdep#XPOS
+s.1.rdep#XPOS
+s.0.ldep.ldep#XPOS
+s.1.ldep.ldep#XPOS
+s.0.rdep.rdep#XPOS
+s.1.rdep.rdep#XPOS
+s.0.l2dep#XPOS
+s.1.l2dep#XPOS
+s.0.r2dep#XPOS
+s.1.r2dep#XPOS
+b.0#XPOS
+b.-1#XPOS
+b.-2#XPOS
+b.0.ldep#XPOS
+b.-1.gov#XPOS
+s.0.gov#XPOS
+s.1.gov#XPOS
+
+# MORPHO
+s.0#MORPHO
+s.1#MORPHO
+s.2#MORPHO
+s.0.ldep#MORPHO
+s.1.ldep#MORPHO
+s.0.rdep#MORPHO
+s.1.rdep#MORPHO
+s.0.ldep.ldep#MORPHO
+s.1.ldep.ldep#MORPHO
+s.0.rdep.rdep#MORPHO
+s.1.rdep.rdep#MORPHO
+s.0.l2dep#MORPHO
+s.1.l2dep#MORPHO
+s.0.r2dep#MORPHO
+s.1.r2dep#MORPHO
+b.0#MORPHO
+b.-1#MORPHO
+b.-2#MORPHO
+b.0.ldep#MORPHO
+s.1.gov#MORPHO
+b.-1.gov#MORPHO
+s.0.gov#MORPHO
+# LABELS
+s.0.ldep#LABEL
+s.1.ldep#LABEL
+s.0.rdep#LABEL
+s.1.rdep#LABEL
+s.0.ldep.ldep#LABEL
+s.1.ldep.ldep#LABEL
+s.0.rdep.rdep#LABEL
+s.1.rdep.rdep#LABEL
+s.0.l2dep#LABEL
+s.1.l2dep#LABEL
+s.0.r2dep#LABEL
+s.1.r2dep#LABEL
+b.0.ldep#LABEL
+b.-2#LABEL
+b.-1#LABEL
+b.0#LABEL
+s.0#LABEL
+s.1#LABEL
+# DISTANCE
+s.0#DIST.s.1
+b.0#DIST.s.0
+# VALENCY
+s.0#nbr
+s.1#nbr
+s.0#nbl
+s.1#nbl
+# UPPERCASE
+b.0#FORM.U
+b.1#FORM.U
+# EOS
+b.-1#EOS
+# HISTORY
+tc.0
+tc.1
+tc.2
+tc.3
+tc.4
+# ERROR CORRECTION
+#b.1.ldep#LEMMA.fasttext
+#b.1.ldep#POS
+#b.1.ldep#MORPHO
+#b.1.ldep#LABEL
+#b.1#LABEL
+#b.1.gov#POS
+#b.1.gov#MORPHO
+#b.2.ldep#LEMMA.fasttext
+#b.2.ldep#POS
+#b.2.ldep#MORPHO
+#b.2.ldep#LABEL
+#b.2#LABEL
+#b.2.gov#POS
+#b.2.gov#MORPHO

UD_any/tokeparser/parser_gold.cla

+Name : Parser_Gold
+Type : Information
+Oracle : parser_gold
+Oracle Filename : parser.as

UD_any/tokeparser/strategy.cla

+Name : Strategy
+Type : Information
+Oracle : strategy_tokenizer,tagger,morpho,lemmatizer,parser
+Oracle Filename : none

UD_any/tokeparser/tagger.cla

+Name : Tagger
+Type : Prediction
+Oracle : tagger
+Feature Model : tagger.fm
+Action Set : data/tagger.as
+Topology : (500,RELU,0.3)
+Dynamic : yes

UD_any/tokeparser/tagger.fm

+# Features classiques
+# FORM
+b.0#FORM.fasttext
+b.1#FORM.fasttext
+b.2#FORM.fasttext
+b.-1#FORM.fasttext
+b.-2#FORM.fasttext
+# POS
+b.-1#POS
+b.-2#POS
+b.-3#POS
+# UPPERCASE
+b.0#FORM.U
+b.1#FORM.U
+# UPPERCASE
+b.0#FORM.LEN
+# SUFFIXES
+b.0#FORM.PART.-4.-4
+b.0#FORM.PART.-3.-3
+b.0#FORM.PART.-2.-2
+b.0#FORM.PART.-1.-1
+b.0#FORM.PART.0.0
+b.0#FORM.PART.1.1
+b.0#FORM.PART.2.2
+b.0#FORM.PART.3.3
+# ERROR CORRECTION
+#b.1#POS
+#b.2#POS
+#b.3#POS
+#b.4#POS

UD_any/tokeparser/test.bd

+#Index Name   ref/hyp dict    Policy   Must print?#
+###################################################
+0      ID     hyp     none    Final    1
+1      FORM   hyp     form    Final    1
+3      POS    hyp     pos     Final    1
+4      XPOS   hyp     pos     Final    1
+5      MORPHO hyp     morpho  Final    1
+2      LEMMA  hyp     form    Final    1
+6      GOV    hyp     int     Final    1
+7      LABEL  hyp     labels  Final    1
+0      EOS    hyp     eos     Final    0

UD_any/tokeparser/tokenizer.cla

+Name : Tokenizer
+Type : Prediction
+Oracle : tokenizer
+Feature Model : tokenizer.fm
+Action Set : data/tokenizer.as
+Topology : (500,RELU,0.3)
+Dynamic : no

UD_any/tokeparser/tokenizer.fm

+# Features classiques
+# FORM
+b.0#FORM.fasttext
+b.-1#FORM.fasttext
+b.-2#FORM.fasttext
+# UPPERCASE
+b.0#FORM.U
+# UPPERCASE
+b.0#FORM.LEN
+# SUFFIXES
+b.0#FORM.PART.-4.-4
+b.0#FORM.PART.-3.-3
+b.0#FORM.PART.-2.-2
+b.0#FORM.PART.-1.-1
+b.0#FORM.PART.0.0
+b.0#FORM.PART.1.1
+b.0#FORM.PART.2.2
+b.0#FORM.PART.3.3
+# RAW INPUT
+raw.-5
+raw.-4
+raw.-3
+raw.-2
+raw.-1
+raw.0
+raw.2
+raw.3
+raw.4
+raw.5
+raw.6

UD_any/tokeparser/tokeparser.dicts

+#Name   Dimension Mode     #
+############################
+# TOKENIZER
+Tokenizer_bool    02        Embeddings
+Tokenizer_int     05        Embeddings
+Tokenizer_letters 30        Embeddings
+Tokenizer_form    30        Embeddings
+Tokenizer_actions 05        Embeddings
+Tokenizer_entropy 05        Embeddings
+# TAGGER
+Tagger_actions 18        Embeddings _
+Tagger_bool    16        Embeddings _
+Tagger_int     16        Embeddings _
+Tagger_eos     16        Embeddings _
+Tagger_gov     16        Embeddings _
+Tagger_pos     18        Embeddings _
+Tagger_form    30        Embeddings _
+Tagger_lemma   30        Embeddings _
+Tagger_letters 30        Embeddings _
+Tagger_labels  18        Embeddings _
+Tagger_morpho  22        Embeddings _
+# MORPHO
+Morpho_actions 18        Embeddings _
+Morpho_bool    16        Embeddings _
+Morpho_int     16        Embeddings _
+Morpho_eos     16        Embeddings _
+Morpho_gov     16        Embeddings _
+Morpho_pos     18        Embeddings _
+Morpho_form    30        Embeddings _
+Morpho_lemma   30        Embeddings _
+Morpho_letters 30        Embeddings _
+Morpho_labels  18        Embeddings _
+Morpho_morpho  22        Embeddings _
+# LEMMATIZER
+Lemmatizer_Rules_form    30        Embeddings
+Lemmatizer_Rules_letters 10        Embeddings
+Lemmatizer_Rules_pos     30        Embeddings
+Lemmatizer_Rules_morpho  30        Embeddings
+# LEMMATIZER
+Lemmatizer_Case_form    30        Embeddings
+Lemmatizer_Case_letters 10        Embeddings
+Lemmatizer_Case_pos     30        Embeddings
+Lemmatizer_Case_morpho  30        Embeddings
+# PARSER
+Parser_actions 18        Embeddings _
+Parser_bool    16        Embeddings _
+Parser_int     16        Embeddings _
+Parser_eos     16        Embeddings _
+Parser_gov     16        Embeddings _
+Parser_pos     18        Embeddings _
+Parser_form    30        Embeddings _
+Parser_lemma   30        Embeddings _
+Parser_letters 30        Embeddings _
+Parser_labels  18        Embeddings _
+Parser_morpho  22        Embeddings _

UD_any/tokeparser/train.bd

+#Index Name   ref/hyp dict    Policy   Must print?#
+###################################################
+0      ID     hyp     none    FromZero 1
+1      FORM   hyp     form    FromZero 1
+3      POS    hyp     pos     FromZero 1
+4      XPOS   hyp     pos     FromZero 1
+5      MORPHO hyp     morpho  FromZero 1
+2      LEMMA  hyp     form    FromZero 1
+6      GOV    hyp     int     FromZero 1
+7      LABEL  hyp     labels  FromZero 1
+0      EOS    hyp     eos     FromZero 0