Commit fcfe6c1c authored by Alexis Nasr's avatar Alexis Nasr
Browse files

modified some english tokenization rules

parent 61309227
......@@ -29,7 +29,8 @@ extern char *token;
[0-9]+\.[0-9]+ {maca_tokenizer_segment(yytext, yytext);}
don't printf("do\nnot\n");
it's printf("it\nis\n");
isn't printf("is\nnot\n");
don’t printf("do\nnot");
doesn't printf("does\nnot");
doesn’t printf("does\nnot");
......@@ -37,8 +38,8 @@ won't printf("will\nnot");
won’t printf("will\nnot");
cannot printf("can\nnot");
wanna printf("want\nto");
's printf("\n's");
’s printf("\n's");
's {maca_tokenizer_segment((char *)"'s", yytext);}
’s {maca_tokenizer_segment((char *)"’s", yytext);}
\n+ printf("\n");
. {maca_tokenizer_add_char_to_token(yytext[0]);}
......
......@@ -13,19 +13,11 @@
void cff2fann_help_message(context *ctx)
{
context_general_help_message(ctx);
context_mode_help_message(ctx);
context_sent_nb_help_message(ctx);
fprintf(stderr, "INPUT\n");
context_conll_help_message(ctx);
fprintf(stderr, "IN TEST MODE\n");
context_vocabs_help_message(ctx);
fprintf(stderr, "OUTPUT\n");
printf("\nInput:\n");
context_cff_help_message(ctx);
fprintf(stderr, "IN TRAIN MODE\n");
context_features_model_help_message(ctx);
context_vocabs_help_message(ctx);
}
void cff2fann_check_options(context *ctx)
......@@ -36,9 +28,8 @@ void cff2fann_check_options(context *ctx)
if(!ctx->cff_filename
|| ctx->help
|| !ctx->mcd_filename
|| !ctx->vocabs_filename
|| !ctx->features_model_filename
/* || !(ctx->cff_filename || ctx->fann_filename) */
){
cff2fann_help_message(ctx);
exit(1);
......@@ -63,7 +54,7 @@ void print_header(mcd *m, feat_model *fm)
for(i=0; i <fm->nbelem; i++){
fd = fm->array[i];
if(fd->nbelem > 1){
printf("feature %d is a complex feature, skipping it\n", i);
fprintf(stderr, "feature %d is a complex feature, skipping it\n", i);
}
else{
sfd = fd->array[0];
......@@ -76,7 +67,7 @@ void print_header(mcd *m, feat_model *fm)
for(i=0; i <fm->nbelem; i++){
fd = fm->array[i];
if(fd->nbelem > 1){
printf("feature %d is a complex feature, skipping it\n", i);
fprintf(stderr, "feature %d is a complex feature, skipping it\n", i);
}
else{
sfd = fd->array[0];
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment