diff --git a/python/tffpy/experiments/yafe_slurm.py b/python/tffpy/experiments/yafe_slurm.py index 08d662d4c6b08d159538b95b812973995bc3ddd1..593ec83b67f3d737e8998992d28a57139ed570bd 100644 --- a/python/tffpy/experiments/yafe_slurm.py +++ b/python/tffpy/experiments/yafe_slurm.py @@ -85,22 +85,22 @@ def generate_slurm_script(script_file_path, xp_var_name, task_ids=None, # script_name = script_path.stem # Generate job script - + log_dir = xp.xp_path / 'logs' + log_dir.mkdir(exist_ok=True) script = '#!/bin/sh\n' - # define parameters script += '#SBATCH --job-name={}\n'.format(xp.name) script += '#SBATCH --array={}%{}\n'.format( ','.join(str(i) for i in task_ids), n_simultaneous_jobs) - script += '#SBATCH --output={}/stdout_%A_%a.slurm\n'.format(xp.xp_path) - script += '#SBATCH --error={}/stderr_%A_%a.slurm\n'.format(xp.xp_path) + script += '#SBATCH --output={}/stdout_%A_%a.slurm\n'.format(log_dir) + script += '#SBATCH --error={}/stderr_%A_%a.slurm\n'.format(log_dir) script += '#SBATCH --time={}\n'.format(slurm_walltime) # if use_gpu: # script += '#SBATCH -p gpu IS NOT NULL\n' # else: # script += '#SBATCH -p gpu IS NULL\n' - script += 'srun -N1 -n1 run_$SLURM_ARRAY_TASK_ID.sh' + script += 'srun -N1 -n1 {}/run_$SLURM_ARRAY_TASK_ID.sh'.format(script_dir) # script += 'echo "OAR_JOB_ID: $OAR_JOB_ID"\n' # script += 'echo "OAR_ARRAY_ID: $OAR_ARRAY_ID"\n' # script += 'echo "SLURM_ARRAY_TASK_ID: $SLURM_ARRAY_TASK_ID"\n' @@ -112,7 +112,7 @@ def generate_slurm_script(script_file_path, xp_var_name, task_ids=None, os.chmod(script_path, status.st_mode | stat.S_IXUSR) # Generate a script of each array element - for i_elt, idt in enumerate(task_ids): + for idt in task_ids: script = '#!/bin/sh\n' # activate the virtual env @@ -120,15 +120,15 @@ def generate_slurm_script(script_file_path, xp_var_name, task_ids=None, script += '{}\n'.format(activate_env_command) # python command - script += 'echo "Running {}.launch_experiment(task_ids={})"\n'\ + script += 'echo "Running {}.launch_experiment(task_ids=[{}])"\n'\ .format(xp_var_name, idt) script += 'python -c "import sys; sys.path.append(\'{0}\'); ' \ 'from {1} import {2}; ' \ - '{2}.launch_experiment(task_ids={3})"\n'.format( + '{2}.launch_experiment(task_ids=[{3}])"\n'.format( script_dir, script_name, xp_var_name, idt) script += 'exit $?' - script_i_path = script_dir / 'run_{}.sh'.format(i_elt) + script_i_path = script_dir / 'run_{}.sh'.format(idt) with script_i_path.open('w') as file: file.write(script) status = os.stat(script_i_path)