From 265848ed2c3f729848bd8b37d75ee88358281725 Mon Sep 17 00:00:00 2001
From: Franck Dary <franck.dary@lis-lab.fr>
Date: Mon, 15 Mar 2021 16:33:01 +0100
Subject: [PATCH] Using more cpu when launching on slurm
---
scripts/launchSlurmArray.py | 50 ++++++++++++++++++++++++++++++++++---
1 file changed, 47 insertions(+), 3 deletions(-)
diff --git a/scripts/launchSlurmArray.py b/scripts/launchSlurmArray.py
index 69d9eb1..7de4c91 100644
--- a/scripts/launchSlurmArray.py
+++ b/scripts/launchSlurmArray.py
@@ -12,7 +12,52 @@ def launchSlurmArray(names, commands, jobName, device, nbHours, limit) :
commandList = " ".join(commands)
- if "jean-zay" in hostname :
+ if ".cluster" in hostname :
+ print("""#! /usr/bin/env bash
+
+#SBATCH --array=0-{}%{}
+#SBATCH --job-name={}
+#SBATCH --output=%A_%a.out
+#SBATCH --error=%A_%a.err
+#SBATCH --open-mode=append
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --partition=skylake
+#SBATCH -A b250
+#SBATCH --time={}:00:00
+
+module purge
+module load userspace/all
+module load gcc/10.2.0
+module load python3/3.8.6
+
+names=({})
+commands=({})
+
+newOut=${{names[$SLURM_ARRAY_TASK_ID]}}".stdout"
+newErr=${{names[$SLURM_ARRAY_TASK_ID]}}".stderr"
+oldOut=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".out"
+oldErr=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".err"
+tmpFile=$SLURM_ARRAY_JOB_ID"_"$SLURM_ARRAY_TASK_ID".tmp"
+
+touch $newOut
+
+cp $newOut $tmpFile
+mv $oldOut $newOut
+cat $tmpFile >> $newOut
+
+touch $newErr
+
+cp $newErr $tmpFile
+mv $oldErr $newErr
+cat $tmpFile >> $newErr
+
+rm $tmpFile
+
+eval "${{commands[$SLURM_ARRAY_TASK_ID]}}"
+""".format(len(names)-1, limit, jobName, nbHours, " ".join(names), " ".join(commands)), file=sFile)
+ sFile.close()
+ elif "jean-zay" in hostname :
print("""#! /usr/bin/env bash
#SBATCH --array=0-{}%{}
@@ -66,8 +111,7 @@ eval "${{commands[$SLURM_ARRAY_TASK_ID]}}"
#SBATCH --error=%A_%a.err
#SBATCH --open-mode=append
#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --hint=nomultithread
+#SBATCH --cpus-per-task=4
#SBATCH --partition={}
#SBATCH --time={}:00:00
--
GitLab