Added script to clean w2v file of long words

scripts/cleanW2V.py

+#! /usr/bin/env python3
+
+import sys
+
+def printUsageAndExit() :
+  print("USAGE: %s w2vFilename wordSizeLimit"%sys.argv[0], file=sys.stderr)
+  exit(1)
+
+if __name__ == "__main__" :
+  if len(sys.argv) != 3 :
+    printUsageAndExit()
+
+  nbWords = None
+  embSize = None
+  lineNumber = 0
+  deleted = set()
+
+  for line in open(sys.argv[1], "r") :
+    lineNumber += 1
+    line = line.strip()
+    splited = line.split()
+    if nbWords is None :
+      nbWords = int(splited[0])
+      embSize = int(splited[1])
+      deleted.add(lineNumber)
+      continue
+    wordLen = int(len(splited[0]))
+    numValues = int(len(splited))-1
+    if numValues != embSize :
+      print("ERROR: in line %d expected %d values got %d instead"%(lineNumber, embSize, numValues), file=sys.stderr)
+      continue
+    if wordLen > int(sys.argv[2]) :
+      print("deleting line %d: word length = %d (%s...)"%(lineNumber, wordLen,splited[0][:10]), file=sys.stderr)
+      deleted.add(lineNumber)
+
+  newNb = lineNumber - 1 - len(deleted)
+  print(newNb, embSize)
+  lineNumber = 0
+  
+  for line in open(sys.argv[1], "r") :
+    lineNumber += 1
+    if lineNumber not in deleted :
+      print(line, end="")
+