Forced scripts output to utf8

tools/conll2text.py

@@ -7,10 +7,13 @@ def printUsageAndExit() :
   exit(1)
 
 if __name__ == "__main__" :
+
+  sys.stdout = open(1, 'w', encoding='utf-8', closefd=False)
+
   if len(sys.argv) != 3 :
     printUsageAndExit()
 
-  for line in open(sys.argv[1]) :
+  for line in open(sys.argv[1], encoding='utf-8') :
     if len(line.strip()) < 2 :
       continue
 

tools/conllu2fplm.py

@@ -21,13 +21,16 @@ def readMCD(mcdFilename) :
   return mcd
 
 if __name__ == "__main__" :
+
+  sys.stdout = open(1, 'w', encoding='utf-8', closefd=False)
+
   if len(sys.argv) != 3 :
     printUsageAndExit()
 
   conllMCD = readMCD(sys.argv[2])
   conllMCDr = {v: k for k, v in conllMCD.items()} 
 
-  for line in open(sys.argv[1], "r") :
+  for line in open(sys.argv[1], "r", encoding="utf8") :
     if len(line.strip()) < 3 :
       continue
     if line.strip()[0] == '#' :

tools/conllu2splits.py

@@ -47,6 +47,9 @@ def computeRules(sentence) :
         rules[word[1]][rule] = 1
 
 def main() :
+
+  sys.stdout = open(1, 'w', encoding='utf-8', closefd=False)
+
   if len(sys.argv) != 3 :
     printUsageAndExit()
 
@@ -58,7 +61,7 @@ def main() :
 
   sentence = []
 
-  for line in open(sys.argv[1], "r") :
+  for line in open(sys.argv[1], "r", encoding="utf8") :
     if len(line.strip()) < 2 or line[0] == '#' :
       if len(sentence) > 0 :
         computeRules(sentence)

tools/conlluShuffleAndMakeDev.py

@@ -8,6 +8,9 @@ def printUsageAndExit() :
   exit(1)
 
 if __name__ == "__main__" :
+
+  sys.stdout = open(1, 'w', encoding='utf-8', closefd=False)
+
   if len(sys.argv) != 3 and len(sys.argv) != 4 :
     printUsageAndExit()
 
@@ -16,7 +19,7 @@ if __name__ == "__main__" :
 
   sentences = []
 
-  for line in open(inputFile, "r") :
+  for line in open(inputFile, "r", encoding="utf8") :
     if len(line.strip()) < 3 :
       continue
     if line.strip().split('=')[0] == "# sent_id " :
@@ -33,7 +36,7 @@ if __name__ == "__main__" :
   if len(sys.argv) == 3 :
     exit(0)
 
-  outputRest = open(sys.argv[3], "w")
+  outputRest = open(sys.argv[3], "w", encoding="utf8")
   for sentence in sentences[int(len(sentences)*float(ratio))+1:] :
     for word in sentence :
       print(word, file=outputRest)

tools/fplm2fP.py

@@ -2,6 +2,8 @@
 
 import sys
 
+sys.stdout = open(1, 'w', encoding='utf-8', closefd=False)
+
 def getLineAsList(inputFile) :
   line = inputFile.readline()
 
@@ -13,7 +15,6 @@ def getLineAsList(inputFile) :
 
   return line
 
-#fplm = open(sys.argv[1], "r", encoding="ISO-8859-1")
 fplm = open(sys.argv[1], "r", encoding="utf8")
 
 line = []