Merge branch 'master' of https://gitlab.lis-lab.fr/franck.dary/macaon_data

pecially if it merges an updated upstream into a topic branch.

UD_any/launchForCluster.py

@@ -4,6 +4,9 @@ import sys
 import os
 import time
 
+cluster = "LIS"
+#cluster = "MESOCENTRE"
+
 prefix = "#SBATCH "
 account = "-A b177"
 shell = "#! /bin/bash"
@@ -16,7 +19,7 @@ stdoutStr = "-o %x.stdout"
 stderrStr = "-e %x.stderr"
 modules = ["userspace/all", "boost/gcc72/1.65.1"]
 configPrefix = "UD_ROOT=~/ud/ud-treebanks-all/"
-commandStr = "./train.sh . %s %s --interactive 0 --printTime %s"
+commandStr = "time ./train.sh . %s %s --interactive 0 --printTime %s --dictCapacity 100000"
 
 def waitUntilDirExists(dirPath) :
   time.sleep(2.0)
@@ -28,18 +31,20 @@ def waitUntilFileExists(filePath) :
   while not os.path.isfile(filePath) :
     time.sleep(2.0)
 
-def launchExperiment(jobName,lang,template,nbCpus=1,partition='skylake',time='2-00',moreArgs='') :
+def launchExperiment(langChanged, jobName,lang,template,nbCpus=1,partition='skylake',time='2-00',moreArgs='') :
 
+  if langChanged :
     print(configPrefix+lang+'/', file=open('config', 'w'))
 
     os.system("cd data && make clean && make && cd ..")
 
     waitUntilFileExists('data/tagger.as')
 
-  slurmFile = open('script.slurm', 'w')
-
   jobName += '_' + lang
 
+  if cluster == "MESOCENTRE" :
+    slurmFile = open('script.slurm', 'w')
+  
     print(shell, file=slurmFile)
     print(prefix+account, file=slurmFile)
     print(prefix+jobNameStr%jobName, file=slurmFile)
@@ -61,6 +66,19 @@ def launchExperiment(jobName,lang,template,nbCpus=1,partition='skylake',time='2-
     slurmFile.close()
   
     os.system("sbatch script.slurm")
+  elif cluster == "LIS" :
+    command = "oarsub"
+    command += " -t besteffort"
+    command += " -n %s"%jobName
+    command += " -E %s.stderr"%jobName
+    command += " -O %s.stdout"%jobName
+    command += " -p \"gpu IS NULL\""
+    command += " -l walltime=96:00:00"
+    command += " \"" + commandStr%(template,jobName,moreArgs) + "\""
+    os.system(command)
+  else :
+    print("ERROR : not a valid cluster \'%s\'"%cluster)
+    exit(1)
 
   waitUntilDirExists('bin/'+jobName+'/data/')
 
@@ -77,8 +95,11 @@ if __name__ == "__main__" :
 
   langs = ["UD_French-GSD", "UD_Hebrew-HTB", "UD_Chinese-GSD", "UD_English-EWT"]
 
-  for experience in templatesExperiments :
   for lang in langs :
+    langChanged = True
+    for experience in templatesExperiments :
       experience['lang'] = lang
+      experience['langChanged'] = langChanged
       launchExperiment(**experience)
+      langChanged = False