Updated tools features and hyperparameters

UD_any/data/feature_models/morpho_incremental.fm

@@ -25,4 +25,10 @@ b.0#FORM.PART.0.0
 b.0#FORM.PART.1.1
 b.0#FORM.PART.2.2
 b.0#FORM.PART.3.3
-
+# RAW INPUT
+raw.0
+raw.2
+raw.3
+raw.4
+raw.5
+raw.6

UD_any/data/feature_models/morpho_nofuture.fm

@@ -25,4 +25,10 @@ b.0#FORM.PART.0.0
 b.0#FORM.PART.1.1
 b.0#FORM.PART.2.2
 b.0#FORM.PART.3.3
-
+# RAW INPUT
+raw.0
+raw.2
+raw.3
+raw.4
+raw.5
+raw.6

UD_any/data/feature_models/parser_nofuture.fm

@@ -101,4 +101,10 @@ tc.1
 tc.2
 tc.3
 tc.4
-
+# RAW INPUT
+raw.0
+raw.2
+raw.3
+raw.4
+raw.5
+raw.6

UD_any/data/feature_models/tagger_incremental.fm

@@ -23,3 +23,10 @@ b.0#FORM.PART.3.3
 # MORPHO
 b.-2#MORPHO
 b.-1#MORPHO
+# RAW INPUT
+raw.0
+raw.2
+raw.3
+raw.4
+raw.5
+raw.6

UD_any/data/feature_models/tagger_nofuture.fm

@@ -20,4 +20,10 @@ b.0#FORM.PART.0.0
 b.0#FORM.PART.1.1
 b.0#FORM.PART.2.2
 b.0#FORM.PART.3.3
-
+# RAW INPUT
+raw.0
+raw.2
+raw.3
+raw.4
+raw.5
+raw.6

UD_any/lemmatizer/lemmatizer.dicts

@@ -6,6 +6,6 @@ Lemmatizer_Rules_pos     30        Embeddings _ 25
 Lemmatizer_Rules_morpho  30        Embeddings _ 50000
 #############################################
 Lemmatizer_Case_form     30        Embeddings _ 50000
-Lemmatizer_Case_letters  10        Embeddings _ 20000
+Lemmatizer_Case_letters  10        Embeddings _ 200000
 Lemmatizer_Case_pos      30        Embeddings _ 25
 Lemmatizer_Case_morpho   30        Embeddings _ 50000

UD_any/tagparser/tagparser.dicts

@@ -21,9 +21,9 @@ Morpho_eos     16        Embeddings _ 5
 Morpho_gov     16        Embeddings _ 100
 Morpho_pos     18        Embeddings _ 21
 Morpho_form    30        Embeddings _ 50000
-Morpho_form.f  30        Embeddings _ 20000
+Morpho_form.f  30        Embeddings _ 200000
 Morpho_lemma   30        Embeddings _ 50000
-Morpho_letters 30        Embeddings _ 20000
+Morpho_letters 30        Embeddings _ 200000
 Morpho_labels  18        Embeddings _ 50
 Morpho_morpho  22        Embeddings _ 50000
 # LEMMATIZER
@@ -44,9 +44,9 @@ Parser_eos     16        Embeddings _ 5
 Parser_gov     16        Embeddings _ 100
 Parser_pos     18        Embeddings _ 21
 Parser_form    30        Embeddings _ 50000
-Parser_form.f  30        Embeddings _ 20000
+Parser_form.f  30        Embeddings _ 200000
 Parser_lemma   30        Embeddings _ 50000
-Parser_letters 30        Embeddings _ 20000
+Parser_letters 30        Embeddings _ 200000
 Parser_labels  18        Embeddings _ 50
 Parser_morpho  22        Embeddings _ 50000
 # SEGMENTER

UD_any/tagparser_sequential/tagparser.dicts

@@ -21,9 +21,9 @@ Morpho_eos     16        Embeddings _ 5
 Morpho_gov     16        Embeddings _ 100
 Morpho_pos     18        Embeddings _ 21
 Morpho_form    30        Embeddings _ 50000
-Morpho_form.f  30        Embeddings _ 20000
+Morpho_form.f  30        Embeddings _ 200000
 Morpho_lemma   30        Embeddings _ 50000
-Morpho_letters 30        Embeddings _ 20000
+Morpho_letters 30        Embeddings _ 200000
 Morpho_labels  18        Embeddings _ 50
 Morpho_morpho  22        Embeddings _ 50000
 # LEMMATIZER
@@ -44,9 +44,9 @@ Parser_eos     16        Embeddings _ 5
 Parser_gov     16        Embeddings _ 100
 Parser_pos     18        Embeddings _ 21
 Parser_form    30        Embeddings _ 50000
-Parser_form.f  30        Embeddings _ 20000
+Parser_form.f  30        Embeddings _ 200000
 Parser_lemma   30        Embeddings _ 50000
-Parser_letters 30        Embeddings _ 20000
+Parser_letters 30        Embeddings _ 200000
 Parser_labels  18        Embeddings _ 50
 Parser_morpho  22        Embeddings _ 50000
 # SEGMENTER

UD_any/tagparser_sequential_strong/tagparser.dicts

@@ -21,9 +21,9 @@ Morpho_eos     16        Embeddings _ 5
 Morpho_gov     16        Embeddings _ 100
 Morpho_pos     18        Embeddings _ 21
 Morpho_form    30        Embeddings _ 50000
-Morpho_form.f  30        Embeddings _ 20000
+Morpho_form.f  30        Embeddings _ 200000
 Morpho_lemma   30        Embeddings _ 50000
-Morpho_letters 30        Embeddings _ 20000
+Morpho_letters 30        Embeddings _ 200000
 Morpho_labels  18        Embeddings _ 50
 Morpho_morpho  22        Embeddings _ 50000
 # LEMMATIZER
@@ -44,9 +44,9 @@ Parser_eos     16        Embeddings _ 5
 Parser_gov     16        Embeddings _ 100
 Parser_pos     18        Embeddings _ 21
 Parser_form    30        Embeddings _ 50000
-Parser_form.f  30        Embeddings _ 20000
+Parser_form.f  30        Embeddings _ 200000
 Parser_lemma   30        Embeddings _ 50000
-Parser_letters 30        Embeddings _ 20000
+Parser_letters 30        Embeddings _ 200000
 Parser_labels  18        Embeddings _ 50
 Parser_morpho  22        Embeddings _ 50000
 # SEGMENTER

UD_any/tokeparser/lemmatizer_case.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : lemma_case
 Feature Model : data/feature_models/lemmatizer_rules.fm
 Action Set : data/lemmatizer_case.as
-Topology : (100,RELU,0.1)
+Topology : (500,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser/lemmatizer_rules.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : lemma_rules
 Feature Model : data/feature_models/lemmatizer_rules.fm
 Action Set : data/lemmatizer_rules.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser/morpho.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : morpho
 Feature Model : data/feature_models/morpho_nofuture.fm
 Action Set : data/morpho_parts.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser/normal.tm

-Name : Tagger, Morpho, Lemmatizer and Parser Machine
-Dicts : tagparser.dicts
+Name : Tokenizer, Tagger, Morpho, Lemmatizer and Parser Machine
+Dicts : tokeparser.dicts
 %CLASSIFIERS
 strategy strategy.cla
+tokenizer tokenizer.cla
 tagger tagger.cla
 morpho morpho.cla
 lemmatizer_lookup lemmatizer_lookup.cla
 lemmatizer_rules lemmatizer_rules.cla
 lemmatizer_case lemmatizer_case.cla
 parser parser.cla
+segmenter segmenter.cla
 %STATES
 strategy strategy
+tokenizer tokenizer
 tagger tagger
 morpho morpho
 lemmatizer_lookup lemmatizer_lookup
 lemmatizer_rules lemmatizer_rules
 lemmatizer_case lemmatizer_case
 parser parser
+segmenter segmenter
 %TRANSITIONS
+strategy tokenizer MOVE tokenizer
 strategy tagger MOVE tagger
 strategy morpho MOVE morpho
 strategy lemmatizer_lookup MOVE lemmatizer_lookup
 strategy lemmatizer_rules MOVE lemmatizer_rules
 strategy lemmatizer_case MOVE lemmatizer_case
 strategy parser MOVE parser
+strategy segmenter MOVE segmenter
+tokenizer strategy *
 tagger strategy *
 morpho strategy *
 lemmatizer_lookup strategy *
 lemmatizer_case strategy *
 lemmatizer_rules strategy *
 parser strategy *
+segmenter strategy *

UD_any/tokeparser/parser.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : parser
 Feature Model : data/feature_models/parser_nofuture.fm
 Action Set : data/parser.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser/segmenter.cla

 Name : Segmenter
 Type : Prediction
 Oracle : segmenter
-Feature Model : data/feature_models/tagger.fm
+Feature Model : data/feature_models/parser_nofuture.fm
 Action Set : data/segmenter.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser/tagger.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : tagger
 Feature Model : data/feature_models/tagger_nofuture.fm
 Action Set : data/tagger.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser/tokenizer.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : tokenizer
 Feature Model : data/feature_models/tokenizer.fm
 Action Set : data/tokenizer.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : no

UD_any/tokeparser/tokeparser.dicts

 #Name   Dimension Mode     #
 ############################
 # TOKENIZER
-Tokenizer_bool    02        Embeddings _ 5
-Tokenizer_int     05        Embeddings _ 200
-Tokenizer_letters 30        Embeddings _ 200000
-Tokenizer_form    30        Embeddings _ 50000
-Tokenizer_form.f  30        Embeddings _ 200000
-Tokenizer_actions 05        Embeddings _ 200
+Tokenizer_actions 18        Embeddings _ 200
+Tokenizer_bool    16        Embeddings _ 20
+Tokenizer_int     16        Embeddings _ 200
+Tokenizer_eos     16        Embeddings _ 20
+Tokenizer_gov     16        Embeddings _ 100
+Tokenizer_pos     18        Embeddings _ 30
+Tokenizer_form    80        Embeddings _ 20
+Tokenizer_form.f  80        Embeddings _ 200000
+Tokenizer_lemma   80        Embeddings _ 20
+Tokenizer_letters 80        Embeddings _ 5000
+Tokenizer_labels  18        Embeddings _ 200
+Tokenizer_morpho  22        Embeddings _ 2000
 # TAGGER
-Tagger_actions 18        Embeddings _ 20
-Tagger_bool    16        Embeddings _ 5
+Tagger_actions 18        Embeddings _ 200
+Tagger_bool    16        Embeddings _ 20
 Tagger_int     16        Embeddings _ 200
-Tagger_eos     16        Embeddings _ 5
+Tagger_eos     16        Embeddings _ 20
 Tagger_gov     16        Embeddings _ 100
-Tagger_pos     18        Embeddings _ 21
-Tagger_form    30        Embeddings _ 50000
-Tagger_form.f  30        Embeddings _ 200000
-Tagger_lemma   30        Embeddings _ 50000
-Tagger_letters 30        Embeddings _ 200000
-Tagger_labels  18        Embeddings _ 50
-Tagger_morpho  22        Embeddings _ 50000
+Tagger_pos     18        Embeddings _ 30
+Tagger_form    80        Embeddings _ 20
+Tagger_form.f  80        Embeddings _ 200000
+Tagger_lemma   80        Embeddings _ 20
+Tagger_letters 80        Embeddings _ 5000
+Tagger_labels  18        Embeddings _ 200
+Tagger_morpho  22        Embeddings _ 2000
 # MORPHO
-Morpho_actions 18        Embeddings _ 1000
-Morpho_bool    16        Embeddings _ 5
+Morpho_actions 18        Embeddings _ 200
+Morpho_bool    16        Embeddings _ 20
 Morpho_int     16        Embeddings _ 200
-Morpho_eos     16        Embeddings _ 5
+Morpho_eos     16        Embeddings _ 20
 Morpho_gov     16        Embeddings _ 100
-Morpho_pos     18        Embeddings _ 21
-Morpho_form    30        Embeddings _ 50000
-Morpho_form.f  30        Embeddings _ 20000
-Morpho_lemma   30        Embeddings _ 50000
-Morpho_letters 30        Embeddings _ 20000
-Morpho_labels  18        Embeddings _ 50
-Morpho_morpho  22        Embeddings _ 50000
-# LEMMATIZER
-Lemmatizer_Rules_form    30        Embeddings _ 50000
-Lemmatizer_Rules_letters 10        Embeddings _ 200000
-Lemmatizer_Rules_pos     30        Embeddings _ 21
-Lemmatizer_Rules_morpho  30        Embeddings _ 50000
-# LEMMATIZER
-Lemmatizer_Case_form    30        Embeddings _ 50000
-Lemmatizer_Case_letters 10        Embeddings _ 200000
-Lemmatizer_Case_pos     30        Embeddings _ 21
-Lemmatizer_Case_morpho  30        Embeddings _ 50000
+Morpho_pos     18        Embeddings _ 30
+Morpho_form    80        Embeddings _ 20
+Morpho_form.f  80        Embeddings _ 200000
+Morpho_lemma   80        Embeddings _ 20
+Morpho_letters 80        Embeddings _ 5000
+Morpho_labels  18        Embeddings _ 200
+Morpho_morpho  22        Embeddings _ 2000
+# LEMMATIZER_RULE
+Lemmatizer_Rules_actions 18        Embeddings _ 2000
+Lemmatizer_Rules_bool    16        Embeddings _ 20
+Lemmatizer_Rules_int     16        Embeddings _ 200
+Lemmatizer_Rules_eos     16        Embeddings _ 20
+Lemmatizer_Rules_gov     16        Embeddings _ 20
+Lemmatizer_Rules_pos     18        Embeddings _ 30
+Lemmatizer_Rules_form    80        Embeddings _ 100000
+Lemmatizer_Rules_form.f  80        Embeddings _ 20
+Lemmatizer_Rules_lemma   80        Embeddings _ 20
+Lemmatizer_Rules_letters 80        Embeddings _ 5000
+Lemmatizer_Rules_labels  18        Embeddings _ 20
+Lemmatizer_Rules_morpho  22        Embeddings _ 2000
+# LEMMATIZER_CASE
+Lemmatizer_Case_actions 18        Embeddings _ 2000
+Lemmatizer_Case_bool    16        Embeddings _ 20
+Lemmatizer_Case_int     16        Embeddings _ 200
+Lemmatizer_Case_eos     16        Embeddings _ 20
+Lemmatizer_Case_gov     16        Embeddings _ 20
+Lemmatizer_Case_pos     18        Embeddings _ 30
+Lemmatizer_Case_form    80        Embeddings _ 100000
+Lemmatizer_Case_form.f  80        Embeddings _ 20
+Lemmatizer_Case_lemma   80        Embeddings _ 20
+Lemmatizer_Case_letters 80        Embeddings _ 5000
+Lemmatizer_Case_labels  18        Embeddings _ 20
+Lemmatizer_Case_morpho  22        Embeddings _ 2000
 # PARSER
 Parser_actions 18        Embeddings _ 200
-Parser_bool    16        Embeddings _ 5
+Parser_bool    16        Embeddings _ 20
 Parser_int     16        Embeddings _ 200
-Parser_eos     16        Embeddings _ 5
+Parser_eos     16        Embeddings _ 20
 Parser_gov     16        Embeddings _ 100
-Parser_pos     18        Embeddings _ 21
-Parser_form    30        Embeddings _ 50000
-Parser_form.f  30        Embeddings _ 20000
-Parser_lemma   30        Embeddings _ 50000
-Parser_letters 30        Embeddings _ 20000
-Parser_labels  18        Embeddings _ 50
-Parser_morpho  22        Embeddings _ 50000
+Parser_pos     18        Embeddings _ 30
+Parser_form    80        Embeddings _ 20
+Parser_form.f  80        Embeddings _ 200000
+Parser_lemma   80        Embeddings _ 20
+Parser_letters 80        Embeddings _ 5000
+Parser_labels  18        Embeddings _ 200
+Parser_morpho  22        Embeddings _ 2000
 # SEGMENTER
-Segmenter_bool    02        Embeddings _ 5
-Segmenter_int     05        Embeddings _ 200
-Segmenter_letters 30        Embeddings _ 200000
-Segmenter_pos     15        Embeddings _ 21
-Segmenter_form    30        Embeddings _ 50000
-Segmenter_form.f  30        Embeddings _ 200000
-Segmenter_actions 05        Embeddings _ 21
-# ERROR_TAGGER
-Error_Tagger_actions 18        Embeddings _
-Error_Tagger_bool    16        Embeddings _
-Error_Tagger_int     16        Embeddings _
-Error_Tagger_eos     16        Embeddings _
-Error_Tagger_gov     16        Embeddings _
-Error_Tagger_pos     18        Embeddings _
-Error_Tagger_form    30        Embeddings _
-Error_Tagger_lemma   30        Embeddings _
-Error_Tagger_letters 30        Embeddings _
-Error_Tagger_labels  18        Embeddings _
-Error_Tagger_morpho  22        Embeddings _
-# ERROR_MORPHO
-Error_Morpho_actions 18        Embeddings _
-Error_Morpho_bool    16        Embeddings _
-Error_Morpho_int     16        Embeddings _
-Error_Morpho_eos     16        Embeddings _
-Error_Morpho_gov     16        Embeddings _
-Error_Morpho_pos     18        Embeddings _
-Error_Morpho_form    30        Embeddings _
-Error_Morpho_lemma   30        Embeddings _
-Error_Morpho_letters 30        Embeddings _
-Error_Morpho_labels  18        Embeddings _
-Error_Morpho_morpho  22        Embeddings _
-# ERROR_PARSER
-Error_Parser_actions 18        Embeddings _
-Error_Parser_bool    16        Embeddings _
-Error_Parser_int     16        Embeddings _
-Error_Parser_eos     16        Embeddings _
-Error_Parser_gov     16        Embeddings _
-Error_Parser_pos     18        Embeddings _
-Error_Parser_form    30        Embeddings _
-Error_Parser_lemma   30        Embeddings _
-Error_Parser_letters 30        Embeddings _
-Error_Parser_labels  18        Embeddings _
-Error_Parser_morpho  22        Embeddings _
+Segmenter_actions 18        Embeddings _ 200
+Segmenter_bool    16        Embeddings _ 20
+Segmenter_int     16        Embeddings _ 200
+Segmenter_eos     16        Embeddings _ 20
+Segmenter_gov     16        Embeddings _ 100
+Segmenter_pos     18        Embeddings _ 30
+Segmenter_form    80        Embeddings _ 20
+Segmenter_form.f  80        Embeddings _ 200000
+Segmenter_lemma   80        Embeddings _ 20
+Segmenter_letters 80        Embeddings _ 5000
+Segmenter_labels  18        Embeddings _ 200
+Segmenter_morpho  22        Embeddings _ 2000

UD_any/tokeparser_incremental/lemmatizer_case.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : lemma_case
 Feature Model : data/feature_models/lemmatizer_rules.fm
 Action Set : data/lemmatizer_case.as
-Topology : (100,RELU,0.1)
+Topology : (800,RELU,0.3)
 Dynamic : yes

UD_any/tokeparser_incremental/lemmatizer_rules.cla

@@ -3,5 +3,5 @@ Type : Prediction
 Oracle : lemma_rules
 Feature Model : data/feature_models/lemmatizer_rules.fm
 Action Set : data/lemmatizer_rules.as
-Topology : (500,RELU,0.3)
+Topology : (800,RELU,0.3)
 Dynamic : yes