From 821ec273e559704ace30af1b48668113b642744c Mon Sep 17 00:00:00 2001
From: Alexis Nasr <alexis.nasr@lif.univ-mrs.fr>
Date: Wed, 4 Jan 2017 22:20:30 +0100
Subject: [PATCH] added some rules in french tokenizer
---
maca_tokenizer/src/fr_tok_rules.l | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/maca_tokenizer/src/fr_tok_rules.l b/maca_tokenizer/src/fr_tok_rules.l
index 7e7fba8..56b0026 100644
--- a/maca_tokenizer/src/fr_tok_rules.l
+++ b/maca_tokenizer/src/fr_tok_rules.l
@@ -15,9 +15,13 @@ extern int defait_amalgames;
[0-9]+,[0-9]* printf("%s", yytext);
[ \t]+ printf("\n");
-\. printf("\n.");
-\, printf("\n,");
-… printf("\n…");
+[ ]*\. printf("\n.");
+[ ]*\? printf("\n?");
+[ ]*\! printf("\n!");
+[ ]*, printf("\n,");
+[ ]*: printf("\n:");
+[ ]*; printf("\n;");
+[ ]*… printf("\n…");
' printf("'\n");
’ printf("'\n");
-je printf("\n-je");
--
GitLab