Skip to content
Snippets Groups Projects
Commit 66b28aac authored by Franck Dary's avatar Franck Dary
Browse files

Lookup classifier almost working

parent 3296bc2f
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 161 additions and 234 deletions
maca_trans_parser/CMakeLists.txt
+ 1
− 0
View file @ 66b28aac
......@@ -10,6 +10,7 @@ set(SOURCES src/context.c
src/oracle_tagparser_arc_eager.c
src/oracle_tagger.c
src/oracle_lemmatizer_rules.c
src/oracle_lemmatizer_lookup.c
src/simple_decoder_parser_arc_eager.c
# src/oracle_chunker.c
src/simple_decoder_tagparser_arc_eager.c
......
maca_trans_parser/src/classifier.c
+ 59
− 219
View file @ 66b28aac
......@@ -20,6 +20,10 @@ classifier *classifier_new(char *name)
classif->mlp_model_filename = NULL;
classif->mlp_struct_filename = NULL;
classif->mlp = NULL;
classif->fplm = NULL;
classif->d_lookup = NULL;
classif->fplm_filename = NULL;
classif->d_lookup_filename = NULL;
classif->d_features_filename = NULL;
classif->d_tapes_filename = NULL;
......@@ -161,6 +165,12 @@ void classifier_print_desc_file(char *filename, classifier *classif)
fprintf(f,"%%MLP_STRUCT %s\n", classif->mlp_struct_filename);
}
if(classif->fplm)
fprintf(f,"%%FPLM %s\n", classif->fplm_filename);
if(classif->d_lookup)
fprintf(f,"%%D_LOOKUP %s\n", classif->d_lookup_filename);
if(classif->oracle_name)
fprintf(f,"%%ORACLE_TYPE %s\n", classif->oracle_name);
}
......@@ -256,7 +266,7 @@ classifier *classifier_read_full(char *filename, char *absolute_path, dico_vec *
if(verbose)
fprintf(stderr, "ORACLE_TYPE = %s\n", name);
if(classif->type != classifier::Type::Classifier){
if(classif->type == classifier::Type::Forced){
fprintf(stderr, "ERROR %s : classifier type '%s' must not use ORACLE_TYPE\n", __func__, type2string(classif->type));
exit(1);
}
......@@ -265,6 +275,34 @@ classifier *classifier_read_full(char *filename, char *absolute_path, dico_vec *
continue;
}
if(sscanf(buffer, "%%FPLM %s", name)){
if(verbose)
fprintf(stderr, "FPLM = %s\n", name);
if(classif->type != classifier::Type::Lookup){
fprintf(stderr, "ERROR %s : classifier type '%s' must not use FPLM\n", __func__, type2string(classif->type));
exit(1);
}
classif->fplm = fplm_load_file(name, verbose);
classif->fplm_filename = strdup(name);
continue;
}
if(sscanf(buffer, "%%D_LOOKUP %s", name)){
if(verbose)
fprintf(stderr, "D_LOOKUP = %s\n", name);
if(classif->type != classifier::Type::Lookup){
fprintf(stderr, "ERROR %s : classifier type '%s' must not use D_LOOKUP\n", __func__, type2string(classif->type));
exit(1);
}
classif->d_lookup = dico_read(name, 0.5);
classif->d_lookup_filename = strdup(name);
continue;
}
if(sscanf(buffer, "%%MLP_MODEL %s", name)){
if(verbose)
fprintf(stderr, "MLP_MODEL = %s\n", name);
......@@ -381,243 +419,45 @@ classifier *classifier_read_full(char *filename, char *absolute_path, dico_vec *
}
classif->output_tagset = mvt_tagset_lemmatizer_rules(d_labels);
}
else {
fprintf(stderr, "ERROR (%s) : unknown input tagset \'%s\'\n", __func__, classif->output_tagset_name);
exit(1);
}
if(classif->mlp_model_filename && classif->mlp_struct_filename){
classifier_init_mlp(classif, new Mlp(classif->mlp_model_filename, classif->mlp_struct_filename, absolute_path));
}
return classif;
}
classifier *classifier_read_full_backup(char *filename, char *absolute_path, dico_vec *d_tapes, int verbose)
{
FILE *f = NULL;
char buffer[1000]; /* ugly */
int line_number = 0;
char name[1000];
char absolute_name[1000];
classifier *classif = NULL;
dico *d_labels;
f = myfopen_no_exit(filename, "r", verbose);
if(f == NULL){
strcpy(absolute_name, absolute_path);
strcat(absolute_name, filename);
f = myfopen(absolute_name, "r");
}
if(verbose) fprintf(stderr, "----------------------------------------\n");
if(verbose) fprintf(stderr, "loading classifier %s\n", filename);
while(fgets(buffer, 1000, f)){
//printf("buffer = <%s>\n", buffer);
line_number++;
if(feof(f)) break;
if((buffer[0] == '\n') || (buffer[0] == '#')) continue;
if(sscanf(buffer, "%%NAME %s", name)){ /* NAME must come before other fields */
if(verbose) fprintf(stderr, "classifier name = %s\n", name);
classif = classifier_new(name);
classif->filename = strdup(filename);
classif->d_tapes = d_tapes;
continue;
}
#if 0
if(sscanf(buffer, "%%D_TAPES %s", name)){
if(verbose) fprintf(stderr, "D_TAPES file= %s\n", name);
if(strcmp(name, "NULL")){
classif->d_tapes = dico_vec_read(name, 0.5);
if(classif->d_tapes == NULL){/* cannot open the tape alphabets file try the absolute filename */
strcpy(absolute_name, absolute_path);
strcat(absolute_name, name);
if(verbose)
fprintf(stderr, "classifier : cannot open D_TAPES file \t %s\n\t trying \t%s\n", name, absolute_name);
classif->d_tapes = dico_vec_read(absolute_name, 0.5);
if(classif->d_tapes == NULL){
fprintf(stderr, "classifier : cannot open D_TAPES file %s, aborting\n", name);
exit(1);
}
}
classif->d_tapes_filename = strdup(name);
}
continue;
}
#endif
if(sscanf(buffer, "%%D_FEATURES %s", name)){
if(verbose) fprintf(stderr, "D_FEATURE file = %s\n", name);
if(strcmp(name, "NULL")){
classif->d_features = dico_read(name, 0.5);
if(classif->d_features == NULL){/* cannot open the feature dictionnary file try the absolute filename */
strcpy(absolute_name, absolute_path);
strcat(absolute_name, name);
if(verbose)
fprintf(stderr, "classifier : cannot open D_FEATURES file %s\n\t trying %s\n", name, absolute_name);
classif->d_features = dico_read(absolute_name, 0.5);
if(classif->d_features == NULL){
fprintf(stderr, "classifier : cannot open D_FEATURES file %s, aborting\n", name);
exit(1);
}
}
classif->d_features_filename = strdup(name);
}
continue;
}
if(sscanf(buffer, "%%OUTPUT_TAGSET %s", name)){
if(verbose) fprintf(stderr, "OUTPUT_TAGSET name = %s\n", name);
classif->output_tagset_name = strdup(name);
continue;
}
if(sscanf(buffer, "%%FEAT_MODEL %s", name)){
if(verbose) fprintf(stderr, "FEAT_MODEL file= %s\n", name);
if(strcmp(name, "NULL")){
classif->fm = feat_model_read(name, feat_lib_build(), verbose);
if(classif->fm == NULL){/* cannot open the feature model file try the absolute filename */
strcpy(absolute_name, absolute_path);
strcat(absolute_name, name);
classif->fm = feat_model_read(absolute_name, feat_lib_build(), verbose);
if(classif->fm == NULL){
fprintf(stderr, "classifier : cannot open FEAT_MODEL file %s, aborting\n", name);
exit(1);
}
}
classif->fm_filename = strdup(name);
classif->fv = feat_vec_new(classif->fm->nbelem);
}
continue;
}
if(sscanf(buffer, "%%ORACLE_TYPE %s", name)){
if(verbose) fprintf(stderr, "ORACLE_TYPE = %s\n", name);
classif->oracle_name = strdup(name);
continue;
}
if(sscanf(buffer, "%%MLP_MODEL %s", name)){
if(verbose) fprintf(stderr, "MLP_MODEL = %s\n", name);
classif->mlp_model_filename = strdup(name);
continue;
}
if(sscanf(buffer, "%%MLP_STRUCT %s", name)){
if(verbose) fprintf(stderr, "MLP_STRUCT = %s\n", name);
classif->mlp_struct_filename = strdup(name);
continue;
}
if(sscanf(buffer, "%%WEIGHT_MATRIX %s", name)){
if(verbose) fprintf(stderr, "WEIGHT MATRIX file= %s\n", name);
if(strcmp(name, "NULL")){
classif->ft = feature_table_load(name, verbose);
if(classif->ft == NULL){ /* cannot open the features table file try the absolute filename */
strcpy(absolute_name, absolute_path);
strcat(absolute_name, name);
classif->ft = feature_table_load(absolute_name, verbose);
if(classif->ft == NULL){
fprintf(stderr, "classifier : cannot open WEIGHT MATRIX file %s, aborting\n", name);
exit(1);
}
}
classif->ft_filename = strdup(name);
}
continue;
}
}
if(!strcmp(classif->output_tagset_name, "parser")){
if(classif->d_tapes == NULL){
fprintf(stderr, "cannot build output tagset, tape alphabets must be specified in file %s\n", filename);
exit(1);
}
d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "LABEL");
if(d_labels == NULL){
fprintf(stderr, "cannot find syntactic labels in file %s\n", classif->d_tapes_filename);
exit(1);
}
classif->output_tagset = mvt_tagset_parser(d_labels);
}
else if(!strcmp(classif->output_tagset_name, "tagger")){
if(classif->d_tapes == NULL){
fprintf(stderr, "cannot build output tagset, tape alphabets must be specified in file %s\n", filename);
exit(1);
}
d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "POS");
if(d_labels == NULL){
fprintf(stderr, "cannot find pos tags in file %s\n", classif->d_tapes_filename);
exit(1);
}
classif->output_tagset = mvt_tagset_tagger(d_labels);
}
else if(!strcmp(classif->output_tagset_name, "morpho")){
else if(!strcmp(classif->output_tagset_name, "lemmatizer_lookup")){
if(classif->d_tapes == NULL){
fprintf(stderr, "cannot build output tagset, tape alphabets must be specified in file %s\n", filename);
exit(1);
}
d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "FEATS");
dico *d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "LEMMA");
if(d_labels == NULL){
fprintf(stderr, "cannot find feats in file %s\n", classif->d_tapes_filename);
exit(1);
}
classif->output_tagset = mvt_tagset_morpho(d_labels);
}
else if(!strcmp(classif->output_tagset_name, "backtracker")){
if(classif->d_tapes == NULL){
fprintf(stderr, "cannot build output tagset, tape alphabets must be specified in file %s\n", filename);
fprintf(stderr, "cannot find LEMMA in file %s\n", classif->d_tapes_filename);
exit(1);
}
d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "LABEL");
if(d_labels == NULL){
fprintf(stderr, "cannot find syntactic labels in file %s\n", classif->d_tapes_filename);
exit(1);
}
classif->output_tagset = mvt_tagset_backtracker(d_labels);
}
else if(!strcmp(classif->output_tagset_name, "forward")){
if(classif->d_tapes == NULL){
fprintf(stderr, "cannot build output tagset, tape alphabets must be specified in file %s\n", filename);
exit(1);
}
d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "LABEL");
if(d_labels == NULL){
fprintf(stderr, "cannot find syntactic labels in file %s\n", classif->d_tapes_filename);
exit(1);
}
classif->output_tagset = mvt_tagset_forward(d_labels);
}
else if(!strcmp(classif->output_tagset_name, "lemmatizer_rules")){
if(classif->d_tapes == NULL){
fprintf(stderr, "cannot build output tagset, tape alphabets must be specified in file %s\n", filename);
exit(1);
}
d_labels = dico_vec_get_dico(classif->d_tapes, (char *) "d_rules");
if(d_labels == NULL){
fprintf(stderr, "cannot find d_rules in file %s\n", classif->d_tapes_filename);
exit(1);
}
classif->output_tagset = mvt_tagset_lemmatizer_rules(d_labels);
classif->output_tagset = mvt_tagset_lookup(d_labels);
}
else {
fprintf(stderr, "ERROR (%s) : unknown input tagset \'%s\'\n", __func__, classif->output_tagset_name);
exit(1);
}
if(!classif->oracle_name){
classif->oracle_name = strdup("NULL");
}
if(classif->mlp_model_filename && classif->mlp_struct_filename){
classifier_init_mlp(classif, new Mlp(classif->mlp_model_filename, classif->mlp_struct_filename, absolute_path));
}
return classif;
if(classif->type == classifier::Type::Lookup){
if(!classif->fplm){
fprintf(stderr, "ERROR (%s) : classifier type 'LOOKUP' require fplm\n", __func__);
exit(1);
}
if(!classif->d_lookup){
classif->d_lookup = dico_new("d_lookup", 20000);
}
if(!classif->d_lookup_filename){
char d_name[1024];
strcpy(d_name, classif->name);
strcat(d_name, "_lookup.dic");
classif->d_lookup_filename = strdup(d_name);
}
}
return classif;
}
classifier *classifier_read(char *filename, char *absolute_path, int verbose)
{
......
maca_trans_parser/src/classifier.h
+ 5
− 0
View file @ 66b28aac
......@@ -14,6 +14,7 @@
#include"feat_model.h"
#include"feature_table.h"
#include"mvt_tagset.h"
#include"fplm.h"
struct classifier{
enum Type{Classifier, Lookup, Forced, Unknown};
......@@ -28,6 +29,10 @@ struct classifier{
feature_table *ft; /* weight matrix */
feat_vec *fv; /* a feature vector of the right size (useful for class prediction) */
Mlp *mlp; /* Multi Layers Perceptron */
fplm_struct *fplm; /* Used for training lookup classifiers */
char *fplm_filename; /* Its filename */
dico *d_lookup; /* The dictionnary used by Lookup classifiers */
char *d_lookup_filename; /* Its filename */
char *output_tagset_name; /* name of the movement tagset */
char *d_features_filename; /* name of the file that stores the feature dictionnary */
char *d_tapes_filename; /* name of the file that stores the tape alphabets */
......
maca_trans_parser/src/maca_tm_mcf2scf.c
+ 12
− 4
View file @ 66b28aac
......@@ -8,6 +8,7 @@
#include"oracle_parser_arc_eager.h"
#include"oracle_tagger.h"
#include"oracle_lemmatizer_rules.h"
#include"oracle_lemmatizer_lookup.h"
#include"feat_fct.h"
#include"context.h"
#include"feat_vec.h"
......@@ -120,6 +121,7 @@ void generate_scf_file(context *ctx)
dico *d_form = dico_vec_get_dico(tm_get_d_tapes(ctx->machine), (char *)"FORM");
dico *d_lemma = dico_vec_get_dico(tm_get_d_tapes(ctx->machine), (char *)"LEMMA");
dico *d_rules = dico_vec_get_dico(tm_get_d_tapes(ctx->machine), (char *)"d_rules");
dico *d_pos = dico_vec_get_dico(tm_get_d_tapes(ctx->machine), (char *)"POS");
while(!word_buffer_end(config_get_buffer(c)) && (sentence_nb < ctx->sent_nb)){
current_state = machine->state_array[c->current_state_nb];
......@@ -156,7 +158,13 @@ void generate_scf_file(context *ctx)
mvt_code = 0;
}
else if(classif->type == classifier::Type::Lookup){
mvt_code = -1; //TODO utiliser un oracle ici
if(!strcmp("LEMMATIZER_LOOKUP", classifier_get_oracle_name(classif))){
mvt_code = oracle_lemmatizer_lookup(c, classifier_get_output_tagset(classif), d_form, d_lemma, d_pos, classif->fplm);
}
else{
fprintf(stderr, "do not know which oracle to use for state %s, oracle_name = %s\n", current_state->name, classifier_get_oracle_name(classif));
exit(1);
}
}
else{
fprintf(stderr, "ERROR %s : classifier '%s' wrong type '%s'\n", __func__, classif->name, type2string(classif->type));
......@@ -165,7 +173,7 @@ void generate_scf_file(context *ctx)
mvt_type = mvt_tagset_get_type(classifier_get_output_tagset(classif), mvt_code);
if(classif->type == classifier::Type::Classifier)
if(classif->type == classifier::Type::Classifier || classif->type == classifier::Type::Lookup)
config2feat_vec_cff(classif->fm, c, classif->d_features, classif->fv, ctx->mode);
if(ctx->debug_mode){
......@@ -184,7 +192,7 @@ void generate_scf_file(context *ctx)
word_buffer_move_left(ref);
}
if(classif->type == classifier::Type::Classifier){
if(classif->type == classifier::Type::Classifier || classif->type == classifier::Type::Lookup){
fprintf(output_file, "%d", current_state->classifier_nb);
fprintf(output_file, "\t%d", mvt_code);
feat_vec_print(output_file, classif->fv);
......
maca_trans_parser/src/maca_tm_train.c
+ 10
− 3
View file @ 66b28aac
......@@ -43,12 +43,19 @@ void train_perceptron(context * ctx){
for(classif_nb = 0; classif_nb < classif_vec->nb; classif_nb++){
classif = classif_vec->array[classif_nb];
if(classif->type != classifier::Type::Classifier)
continue;
if(ctx->verbose)
fprintf(stderr, "training classifier %d / %d : %s\n", classif_nb + 1, classif_vec->nb, classif->name);
if(classif->type == classifier::Type::Lookup){
//TODO ici finir
classifier_print_desc_file(classif->filename, classif);
dico_print(classif->d_lookup_filename, classif->d_lookup);
continue;
}
if(classif->type == classifier::Type::Forced)
continue;
strcpy(cfw_filename, classif->name);
strcat(cfw_filename, ".cfw");
......
maca_trans_parser/src/mvt_tagset.c
+ 25
− 2
View file @ 66b28aac
......@@ -94,6 +94,12 @@ void mvt_tagset_print_mvt(FILE *f, mvt_tagset *t, int mvt_code)
case MVT_LEMMATIZER_RULES :
fprintf(f, "LEMMA_RULES %s", mvt_label_str);
break;
case MVT_LOOKUP_FOUND :
fprintf(f, "LOOKUP_FOUND %s", mvt_label_str);
break;
case MVT_LOOKUP_NOTFOUND :
fprintf(f, "LOOKUP_NOTFOUND");
break;
}
}
int mvt_tagset_type_string2int(mvt_tagset *t, char *mvt_type_str)
......@@ -111,6 +117,8 @@ int mvt_tagset_type_string2int(mvt_tagset *t, char *mvt_type_str)
if(!strcmp(mvt_type_str, "BKWD")) return MVT_BKWD;
if(!strcmp(mvt_type_str, "MORPHO")) return MVT_MORPHO;
if(!strcmp(mvt_type_str, "LEMMATIZER_RULES")) return MVT_LEMMATIZER_RULES;
if(!strcmp(mvt_type_str, "LOOKUP_FOUND")) return MVT_LOOKUP_FOUND;
if(!strcmp(mvt_type_str, "LOOKUP_NOTFOUND")) return MVT_LOOKUP_NOTFOUND;
return -1;
}
......@@ -127,6 +135,9 @@ void mvt_tagset_print(FILE * f, mvt_tagset *t)
fprintf(f, "CPOS %d %d\n", t->start[MVT_CPOS], t->end[MVT_CPOS]);
fprintf(f, "BKWD %d %d\n", t->start[MVT_BKWD], t->end[MVT_BKWD]);
fprintf(f, "MORPHO %d %d\n", t->start[MVT_MORPHO], t->end[MVT_MORPHO]);
fprintf(f, "LEMMATIZER_RULES %d %d\n", t->start[MVT_LEMMATIZER_RULES], t->end[MVT_LEMMATIZER_RULES]);
fprintf(f, "LOOKUP_NOTFOUND %d %d\n", t->start[MVT_LOOKUP_NOTFOUND], t->end[MVT_LOOKUP_NOTFOUND]);
fprintf(f, "LOOKUP_FOUND %d %d\n", t->start[MVT_LOOKUP_FOUND], t->end[MVT_LOOKUP_FOUND]);
}
mvt_tagset *mvt_tagset_parser(dico *d_labels)
......@@ -186,7 +197,7 @@ mvt_tagset *mvt_tagset_lemmatizer_rules(dico *d_labels)
mvt_tagset *mvt_tagset_backtracker(dico *d_labels)
{
mvt_tagset *t = mvt_tagset_new((char *)"std", d_labels);
mvt_tagset *t = mvt_tagset_new((char *)"backtracker", d_labels);
t->start[MVT_BKWD] = t->end[MVT_BKWD] = t->nbelem++;
//mvt_tagset_print(stderr, t);
......@@ -195,13 +206,25 @@ mvt_tagset *mvt_tagset_backtracker(dico *d_labels)
mvt_tagset *mvt_tagset_forward(dico *d_labels)
{
mvt_tagset *t = mvt_tagset_new((char *)"std", d_labels);
mvt_tagset *t = mvt_tagset_new((char *)"forward", d_labels);
t->start[MVT_FWD] = t->end[MVT_FWD] = t->nbelem++;
//mvt_tagset_print(stderr, t);
return t;
}
mvt_tagset *mvt_tagset_lookup(dico *d_labels)
{
mvt_tagset *t = mvt_tagset_new((char *)"lookup", d_labels);
t->start[MVT_LOOKUP_NOTFOUND] = t->end[MVT_LOOKUP_NOTFOUND] = t->nbelem++;
t->start[MVT_LOOKUP_FOUND] = t->nbelem;
t->nbelem += d_labels->nbelem;
t->end[MVT_LOOKUP_FOUND] = t->nbelem - 1;
//mvt_tagset_print(stderr, t);
return t;
}
mvt_tagset *mvt_tagset_std(void)
{
mvt_tagset *t = mvt_tagset_new((char *)"std", NULL);
......
maca_trans_parser/src/mvt_tagset.h
+ 4
− 1
View file @ 66b28aac
......@@ -13,8 +13,10 @@
#define MVT_BKWD 9
#define MVT_MORPHO 10
#define MVT_LEMMATIZER_RULES 11
#define MVT_LOOKUP_NOTFOUND 12
#define MVT_LOOKUP_FOUND 13
#define MVT_TYPES_NB 12
#define MVT_TYPES_NB 14
#include"dico.h"
......@@ -38,6 +40,7 @@ mvt_tagset *mvt_tagset_parser(dico *d_labels);
mvt_tagset *mvt_tagset_tagger(dico *d_labels);
mvt_tagset *mvt_tagset_morpho(dico *d_labels);
mvt_tagset *mvt_tagset_lemmatizer_rules(dico *d_labels);
mvt_tagset *mvt_tagset_lookup(dico *d_labels);
mvt_tagset *mvt_tagset_backtracker(dico *d_labels);
mvt_tagset *mvt_tagset_forward(dico *d_labels);
mvt_tagset *mvt_tagset_std(void);
......
maca_trans_parser/src/oracle_lemmatizer_lookup.c 0 → 100644
+ 27
− 0
View file @ 66b28aac
#include"oracle_lemmatizer_lookup.h"
int oracle_lemmatizer_lookup(config *c, mvt_tagset *tagset, dico *d_form, dico *d_lemma, dico *d_pos, fplm_struct *fplm)
{
static char lower_form[1024];
int form = word_get_form(word_buffer_b0(config_get_buffer(c)));
int pos = word_get_pos(word_buffer_b0(config_get_buffer(c)));
char *form_str = dico_int2string(d_form, form);
char *pos_str = dico_int2string(d_pos, pos);
strcpy(lower_form, form_str);
to_lower_string(lower_form);
char *lemma_str = fplm_lookup_lemma(fplm, form_str, pos_str, 0);
if(!strcmp(lemma_str, form_str) || !strcmp(lemma_str, lower_form))
return tagset->start[MVT_LOOKUP_NOTFOUND];
int lemma = dico_string2int(d_lemma, lemma_str);
int mvt_code = mvt_tagset_get_code(tagset, MVT_LOOKUP_FOUND, lemma);
return mvt_code;
}
maca_trans_parser/src/oracle_lemmatizer_lookup.h 0 → 100644
+ 14
− 0
View file @ 66b28aac
#ifndef __ORACLE_LEMMATIZER_LOOKUP__
#define __ORACLE_LEMMATIZER_LOOKUP__
#include<stdio.h>
#include<stdlib.h>
#include"config.h"
#include"mvt_tagset.h"
#include"dico.h"
#include"fplm.h"
int oracle_lemmatizer_lookup(config *c, mvt_tagset *tagset, dico *d_form, dico *d_lemma, dico *d_pos, fplm_struct *fplm);
#endif
maca_trans_parser/src/oracle_lemmatizer_rules.c
+ 4
− 5
View file @ 66b28aac
......@@ -19,14 +19,13 @@ int oracle_lemmatizer_rules(config *c, mvt_tagset *tagset, dico *d_form, dico *d
to_lower_string(lower_lemma);
char *rule_str = compute_l_rule(lower_lemma, lower_form, strict);
bool current_dico_add_policy = dico_get_add_unknown_strings();
dico_unset_add_unknown_strings();
int rule_code = dico_string2int(d_rules, rule_str);
if(current_dico_add_policy)
dico_set_add_unknown_strings();
if(rule_code == -1){
/*
fprintf(stderr, "ERROR : %s unknown rule form=%s lemma=%s rule=%s\n", __func__, form_str, lemma_str, rule_str);
exit(1);
*/
rule_code = 0;
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment