From 363c1bdea21fd6cc3afa6c1f653f77a46633dfe7 Mon Sep 17 00:00:00 2001
From: "johannes.heinecke" <johannes.heinecke@orange.com>
Date: Thu, 5 Jan 2017 09:29:33 +0100
Subject: [PATCH] ajout de regles de tokenisation

---
 maca_tokenizer/src/fr_tok_rules.l | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/maca_tokenizer/src/fr_tok_rules.l b/maca_tokenizer/src/fr_tok_rules.l
index 56b0026..d0f391d 100644
--- a/maca_tokenizer/src/fr_tok_rules.l
+++ b/maca_tokenizer/src/fr_tok_rules.l
@@ -22,8 +22,13 @@ extern int defait_amalgames;
 [ ]*:       printf("\n:");
 [ ]*;       printf("\n;");
 [ ]*…        printf("\n…");
+[ ]*\)        printf("\n)");
+[ ]*»        printf("\n»");
+\(        printf("(\n");
 '        printf("'\n");
 ’        printf("'\n");
+\"	printf("\"\n");
+«	printf("»\n");
 -je      printf("\n-je");
 -tu      printf("\n-tu");
 -on      printf("\n-on");
-- 
GitLab