fixed bug in tokenization rules for french

maca_tokenizer/src/fr_tok_rules.l

@@ -29,15 +29,15 @@ nosepar [^ \t\n]
 
 \<[^\>]*\>   {maca_tokenizer_segment((char *)"", yytext);}
 {separ}+     {maca_tokenizer_segment((char *)"", yytext);}
-{separ}*\.   {maca_tokenizer_segment((char *)".", yytext);}
-{separ}*\?   {maca_tokenizer_segment((char *)"?", yytext);}
-{separ}*\!   {maca_tokenizer_segment((char *)"!", yytext);}
-{separ}*,    {maca_tokenizer_segment((char *)",", yytext);}
-{separ}*:    {maca_tokenizer_segment((char *)":", yytext);}
-{separ}*;    {maca_tokenizer_segment((char *)";", yytext);}
-{separ}*…    {maca_tokenizer_segment((char *)"…", yytext);}
-{separ}*\)   {maca_tokenizer_segment((char *)")", yytext);}
-{separ}*»    {maca_tokenizer_segment((char *)"»", yytext);}
+\.   {maca_tokenizer_segment((char *)".", yytext);}
+\?   {maca_tokenizer_segment((char *)"?", yytext);}
+\!   {maca_tokenizer_segment((char *)"!", yytext);}
+,    {maca_tokenizer_segment((char *)",", yytext);}
+:    {maca_tokenizer_segment((char *)":", yytext);}
+;    {maca_tokenizer_segment((char *)";", yytext);}
+…    {maca_tokenizer_segment((char *)"…", yytext);}
+\)   {maca_tokenizer_segment((char *)")", yytext);}
+»    {maca_tokenizer_segment((char *)"»", yytext);}
 \(           {maca_tokenizer_segment((char *)"(", yytext);}
 \"           {maca_tokenizer_segment((char *)"\"", yytext);}
 «            {maca_tokenizer_segment((char *)"«", yytext);}

maca_tokenizer/src/maca_tokenizer_functions_for_lex.c

@@ -28,7 +28,7 @@ void maca_tokenizer_segment(char *separator, char *text_matched){
     if(print_offset)
       printf("\t%d", offset);
     if(print_token_length)
-      printf("\t%d", (int) strlen(separator));
+      printf("\t%d", (int) utf8_strlen(separator));
     printf("\n");
   }
   offset += utf8_strlen(text_matched);