From 363c1bdea21fd6cc3afa6c1f653f77a46633dfe7 Mon Sep 17 00:00:00 2001 From: "johannes.heinecke" <johannes.heinecke@orange.com> Date: Thu, 5 Jan 2017 09:29:33 +0100 Subject: [PATCH] ajout de regles de tokenisation --- maca_tokenizer/src/fr_tok_rules.l | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/maca_tokenizer/src/fr_tok_rules.l b/maca_tokenizer/src/fr_tok_rules.l index 56b0026..d0f391d 100644 --- a/maca_tokenizer/src/fr_tok_rules.l +++ b/maca_tokenizer/src/fr_tok_rules.l @@ -22,8 +22,13 @@ extern int defait_amalgames; [ ]*: printf("\n:"); [ ]*; printf("\n;"); [ ]*… printf("\n…"); +[ ]*\) printf("\n)"); +[ ]*» printf("\n»"); +\( printf("(\n"); ' printf("'\n"); ’ printf("'\n"); +\" printf("\"\n"); +« printf("»\n"); -je printf("\n-je"); -tu printf("\n-tu"); -on printf("\n-on"); -- GitLab