Allowing multiline commands to be launched from launchBatches.py

UD_any/launchBatches.py

@@ -21,7 +21,7 @@ def prepareExperiment(lang, template, expName) :
 def addNamesAndCommandsTrain(names, commands, mode, expName, arguments, seed) :
   names.append("\""+expName+"\"")
   
-  commands.append("\""+"./train.sh {} bin/{} {} --silent --seed {}".format(mode, expName,arguments,seed)+"\"")
+  commands.append("'"+"./train.sh {} bin/{} {} --silent --seed {}".format(mode, expName,arguments,seed)+"'")
 ###############################################################################
 
 ###############################################################################
@@ -63,13 +63,31 @@ module load python/3.7.5
 names=({})
 commands=({})
 
-mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out" ${{names[$SLURM_ARRAY_TASK_ID]}}".stdout"
-mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err" ${{names[$SLURM_ARRAY_TASK_ID]}}".stderr"
-${{commands[$SLURM_ARRAY_TASK_ID]}}
-""".format(len(names), limit, mode, jobName, "qos_gpu-t4" if nbHours > 20 else "qos_gpu-t3", nbHours, " ".join(names), " ".join(commands)), file=sFile)
+newOut=${{names[$SLURM_ARRAY_TASK_ID]}}".stdout"
+newErr=${{names[$SLURM_ARRAY_TASK_ID]}}".stderr"
+oldOut=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out"
+oldErr=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err"
+tmpFile=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".tmp"
+
+touch $newOut
+
+cp $newOut $tmpFile
+mv $oldOut $newOut
+cat $tmpFile >> $newOut
+
+touch $newErr
+
+cp $newErr $tmpFile
+mv $oldErr $newErr
+cat $tmpFile >> $newErr
+
+rm $tmpFile
+
+eval "${{commands[$SLURM_ARRAY_TASK_ID]}}"
+""".format(len(names)-1, limit, mode, jobName, "qos_gpu-t4" if nbHours > 20 else "qos_gpu-t3", nbHours, " ".join(names), " ".join(commands)), file=sFile)
     sFile.close()
   elif hostname == "sms.liscluster" :
-    print("""#! /usr/bin/env bash
+    print('''#! /usr/bin/env bash
 
 #SBATCH --array=0-{}%{}
 #SBATCH --job-name={}:{}
@@ -88,10 +106,28 @@ module purge
 names=({})
 commands=({})
 
-mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out" ${{names[$SLURM_ARRAY_TASK_ID]}}".stdout"
-mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err" ${{names[$SLURM_ARRAY_TASK_ID]}}".stderr"
-${{commands[$SLURM_ARRAY_TASK_ID]}}
-""".format(len(names), limit, mode, jobName, "cpu" if device == "cpu" else "gpu\n#SBATCH --gres=gpu", nbHours, " ".join(names), commandList), file=sFile)
+newOut=${{names[$SLURM_ARRAY_TASK_ID]}}".stdout"
+newErr=${{names[$SLURM_ARRAY_TASK_ID]}}".stderr"
+oldOut=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out"
+oldErr=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err"
+tmpFile=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".tmp"
+
+touch $newOut
+
+cp $newOut $tmpFile
+mv $oldOut $newOut
+cat $tmpFile >> $newOut
+
+touch $newErr
+
+cp $newErr $tmpFile
+mv $oldErr $newErr
+cat $tmpFile >> $newErr
+
+rm $tmpFile
+
+eval "${{commands[$SLURM_ARRAY_TASK_ID]}}"
+'''.format(len(names)-1, limit, mode, jobName, "cpu" if device == "cpu" else "gpu\n#SBATCH --gres=gpu", nbHours, " ".join(names), commandList), file=sFile)
     sFile.close()
   else :
     print("ERROR : Unknown hostname \'%s\'"%hostname)