diff --git a/UD_any/launchBatches.py b/UD_any/launchBatches.py index ba00da1b2815c65dae5fc8dd49d88a1001fd6dc1..f8a74279df9a822533e01c1d0527ac30d0238fc5 100755 --- a/UD_any/launchBatches.py +++ b/UD_any/launchBatches.py @@ -21,7 +21,7 @@ def prepareExperiment(lang, template, expName) : def addNamesAndCommandsTrain(names, commands, mode, expName, arguments, seed) : names.append("\""+expName+"\"") - commands.append("\""+"./train.sh {} bin/{} {} --silent --seed {}".format(mode, expName,arguments,seed)+"\"") + commands.append("'"+"./train.sh {} bin/{} {} --silent --seed {}".format(mode, expName,arguments,seed)+"'") ############################################################################### ############################################################################### @@ -63,13 +63,31 @@ module load python/3.7.5 names=({}) commands=({}) -mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out" ${{names[$SLURM_ARRAY_TASK_ID]}}".stdout" -mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err" ${{names[$SLURM_ARRAY_TASK_ID]}}".stderr" -${{commands[$SLURM_ARRAY_TASK_ID]}} -""".format(len(names), limit, mode, jobName, "qos_gpu-t4" if nbHours > 20 else "qos_gpu-t3", nbHours, " ".join(names), " ".join(commands)), file=sFile) +newOut=${{names[$SLURM_ARRAY_TASK_ID]}}".stdout" +newErr=${{names[$SLURM_ARRAY_TASK_ID]}}".stderr" +oldOut=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out" +oldErr=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err" +tmpFile=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".tmp" + +touch $newOut + +cp $newOut $tmpFile +mv $oldOut $newOut +cat $tmpFile >> $newOut + +touch $newErr + +cp $newErr $tmpFile +mv $oldErr $newErr +cat $tmpFile >> $newErr + +rm $tmpFile + +eval "${{commands[$SLURM_ARRAY_TASK_ID]}}" +""".format(len(names)-1, limit, mode, jobName, "qos_gpu-t4" if nbHours > 20 else "qos_gpu-t3", nbHours, " ".join(names), " ".join(commands)), file=sFile) sFile.close() elif hostname == "sms.liscluster" : - print("""#! /usr/bin/env bash + print('''#! /usr/bin/env bash #SBATCH --array=0-{}%{} #SBATCH --job-name={}:{} @@ -88,10 +106,28 @@ module purge names=({}) commands=({}) -mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out" ${{names[$SLURM_ARRAY_TASK_ID]}}".stdout" -mv $SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err" ${{names[$SLURM_ARRAY_TASK_ID]}}".stderr" -${{commands[$SLURM_ARRAY_TASK_ID]}} -""".format(len(names), limit, mode, jobName, "cpu" if device == "cpu" else "gpu\n#SBATCH --gres=gpu", nbHours, " ".join(names), commandList), file=sFile) +newOut=${{names[$SLURM_ARRAY_TASK_ID]}}".stdout" +newErr=${{names[$SLURM_ARRAY_TASK_ID]}}".stderr" +oldOut=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out" +oldErr=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err" +tmpFile=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".tmp" + +touch $newOut + +cp $newOut $tmpFile +mv $oldOut $newOut +cat $tmpFile >> $newOut + +touch $newErr + +cp $newErr $tmpFile +mv $oldErr $newErr +cat $tmpFile >> $newErr + +rm $tmpFile + +eval "${{commands[$SLURM_ARRAY_TASK_ID]}}" +'''.format(len(names)-1, limit, mode, jobName, "cpu" if device == "cpu" else "gpu\n#SBATCH --gres=gpu", nbHours, " ".join(names), commandList), file=sFile) sFile.close() else : print("ERROR : Unknown hostname \'%s\'"%hostname)