From b74a8e24b594528dfc789ff62b457b4914f18f7e Mon Sep 17 00:00:00 2001
From: Franck Dary <franck.dary@lis-lab.fr>
Date: Fri, 3 Jul 2020 17:33:13 +0200
Subject: [PATCH] Give more transitions to tokenizer

---
 UD_any/data/Makefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/UD_any/data/Makefile b/UD_any/data/Makefile
index dd2c6c9..e869f10 100644
--- a/UD_any/data/Makefile
+++ b/UD_any/data/Makefile
@@ -27,6 +27,11 @@ tokenizer.ts: all_no_test.conllu
 	echo "SPLIT 6" >> $@
 	echo "SPLIT 7" >> $@
 	echo "ADDCHARTOWORD 1" >> $@
+	echo "ADDCHARTOWORD 2" >> $@
+	echo "ADDCHARTOWORD 3" >> $@
+	echo "ADDCHARTOWORD 4" >> $@
+	echo "ADDCHARTOWORD 5" >> $@
+	echo "ADDCHARTOWORD 6" >> $@
 	echo "IGNORECHAR" >> $@
 	sed -i -e 's/^/<tokenizer> /' $@
 	sed -i -e 's/^/<tokenizer> /' splitwords.ts
-- 
GitLab