From 363c1bdea21fd6cc3afa6c1f653f77a46633dfe7 Mon Sep 17 00:00:00 2001
From: "johannes.heinecke" <johannes.heinecke@orange.com>
Date: Thu, 5 Jan 2017 09:29:33 +0100
Subject: [PATCH] ajout de regles de tokenisation
---
maca_tokenizer/src/fr_tok_rules.l | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/maca_tokenizer/src/fr_tok_rules.l b/maca_tokenizer/src/fr_tok_rules.l
index 56b0026..d0f391d 100644
--- a/maca_tokenizer/src/fr_tok_rules.l
+++ b/maca_tokenizer/src/fr_tok_rules.l
@@ -22,8 +22,13 @@ extern int defait_amalgames;
[ ]*: printf("\n:");
[ ]*; printf("\n;");
[ ]*… printf("\n…");
+[ ]*\) printf("\n)");
+[ ]*» printf("\n»");
+\( printf("(\n");
' printf("'\n");
’ printf("'\n");
+\" printf("\"\n");
+« printf("»\n");
-je printf("\n-je");
-tu printf("\n-tu");
-on printf("\n-on");
--
GitLab