diff --git a/python/tffpy/experiments/exp_variance.py b/python/tffpy/experiments/exp_variance.py
index 03acde350c1cf66d00bd523fca501fe060bbc0bb..b6b3d74a6d7afeb5dcc2e737c6c3d41aec7fbf1e 100644
--- a/python/tffpy/experiments/exp_variance.py
+++ b/python/tffpy/experiments/exp_variance.py
@@ -53,7 +53,7 @@ class VarianceExperiment(SolveTffExperiment):
or_mask=True,
crop=None,
fig_dir=None)
- solver_params = dict(tol_subregions=1e-5,
+ solver_params = dict(tol_subregions=[None, 1e-5],
tolerance_arrf=1e-3,
proba_arrf=1 - 1e-4,
rand_state=np.arange(100))
diff --git a/python/tffpy/experiments/tests/test_exp_variance.py b/python/tffpy/experiments/tests/test_exp_variance.py
new file mode 100644
index 0000000000000000000000000000000000000000..8918c5805b25d982b0ba0529349bb2035786f787
--- /dev/null
+++ b/python/tffpy/experiments/tests/test_exp_variance.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# ######### COPYRIGHT #########
+# Credits
+# #######
+#
+# Copyright(c) 2020-2020
+# ----------------------
+#
+# * Laboratoire d'Informatique et Systèmes <http://www.lis-lab.fr/>
+# * Université d'Aix-Marseille <http://www.univ-amu.fr/>
+# * Centre National de la Recherche Scientifique <http://www.cnrs.fr/>
+# * Université de Toulon <http://www.univ-tln.fr/>
+#
+# Contributors
+# ------------
+#
+# * `Valentin Emiya <mailto:valentin.emiya@lis-lab.fr>`_
+# * `Ama Marina Krémé <mailto:ama-marina.kreme@lis-lab.fr>`_
+#
+# This package has been created thanks to the joint work with Florent Jaillet
+# and Ronan Hamon on other packages.
+#
+# Description
+# -----------
+#
+# Time frequency fading using Gabor multipliers
+#
+# Version
+# -------
+#
+# * tffpy version = 0.1.3
+#
+# Licence
+# -------
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# ######### COPYRIGHT #########
+"""Test of the module :module:`tffpy.experiments.exp_variance`
+
+.. moduleauthor:: Valentin Emiya
+"""
+import unittest
+
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+mpl.rcParams['figure.max_open_warning'] = 40
+
+from tffpy.experiments.exp_variance import \
+ VarianceExperiment, create_and_run_light_experiment
+from tffpy.tests.ci_config import create_config_files
+
+
+class TestVarianceExperiment(unittest.TestCase):
+ def setUp(self):
+ create_config_files()
+
+ def test_light_experiment(self):
+ create_and_run_light_experiment()
+
+ light_exp = VarianceExperiment.get_experiment(
+ setting='light', force_reset=False)
+ for idt in range(light_exp.n_tasks):
+ light_exp.plot_task(idt=idt, fontsize=16)
+ plt.close('all')
+ light_exp.plot_results()
+ plt.close('all')
+
+ def test_create_full_experiment(self):
+ experiment = VarianceExperiment.get_experiment(
+ setting='full', force_reset=False)
+ experiment.display_status()
diff --git a/python/tffpy/experiments/yafe_slurm.py b/python/tffpy/experiments/yafe_slurm.py
new file mode 100644
index 0000000000000000000000000000000000000000..08d662d4c6b08d159538b95b812973995bc3ddd1
--- /dev/null
+++ b/python/tffpy/experiments/yafe_slurm.py
@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+"""
+
+.. moduleauthor:: Valentin Emiya
+"""
+
+import importlib
+import os
+from pathlib import Path
+import stat
+import sys
+
+
+def generate_slurm_script(script_file_path, xp_var_name, task_ids=None,
+ n_simultaneous_jobs=10, slurm_walltime='02:00:00',
+ activate_env_command=None, use_gpu=False):
+ """Generate a script to launch an experiment using Slurm.
+
+ Tasks are divided into batches that are executed by oar jobs.
+
+ The resulting script is written in the experiment folder, and the command
+ to launch the jobs with Slurm is displayed in the terminal.
+
+ An example script illustrating how to use
+ :func:`yafe.utils.generate_slurm_script` is available in the corresponding
+ :ref:`tutorial <tutorial_oar>`.
+
+ Parameters
+ ----------
+ script_file_path : str
+ File path to the script that defines the experiment.
+ xp_var_name : str
+ Name of the variable containing the experiment in the script.
+ task_ids : list
+ List of tasks ids to run.
+ If ``task_ids`` is ``None``, the list of pending tasks of the
+ experiment is used.
+ batch_size : int
+ Number of tasks run in each batch.
+ slurm_walltime : str
+ Wall time for each Slurm job ('HH:MM:SS').
+ activate_env_command : str or None
+ Optional command that must be run to activate a Python virtual
+ environment before launching the experiment.
+ Typically, this is a command of the form
+ ``source some_virtual_env/bin/activate`` when using virtualenv and
+ ``source activate some_conda_env`` when using conda.
+ If ``activate_env_command`` is ``None``, no virtual environment is
+ activated.
+ use_gpu : bool
+ Flag specifying if a gpu ressource is needed when running the
+ experiment.
+ """
+ # TODO update documentation
+ script_file_path = Path(script_file_path)
+ script_dir = script_file_path.parent
+ script_name = script_file_path.stem
+
+ sys.path.append(str(script_dir))
+ mod = importlib.import_module(script_name)
+ xp = getattr(mod, xp_var_name)
+ script_dir = xp.xp_path / 'job_scripts'
+ script_dir.mkdir(exist_ok=True)
+ for f in script_dir.glob('*.sh'):
+ os.remove(f)
+ # script_dir.rm
+
+ if task_ids is None:
+ task_ids = xp.get_pending_task_ids()
+
+ # split and save the tasks
+ # task_ids = list(map(str, task_ids))
+ # batches = [
+ # task_ids[i:(i + batch_size)]
+ # for i in range(0, len(task_ids), batch_size)
+ # ]
+ # file_path = xp.xp_path / 'listoftasks.txt'
+ #
+ # with open(str(file_path), 'wt') as fout:
+ # fout.write('\n'.join(map(lambda batch: ','.join(batch), batches)))
+
+ # generate and save script
+ # script_path = Path(os.path.abspath(script_file_path))
+ # script_dir = script_path.parent
+ # script_name = script_path.stem
+
+ # Generate job script
+
+ script = '#!/bin/sh\n'
+
+ # define parameters
+ script += '#SBATCH --job-name={}\n'.format(xp.name)
+ script += '#SBATCH --array={}%{}\n'.format(
+ ','.join(str(i) for i in task_ids), n_simultaneous_jobs)
+ script += '#SBATCH --output={}/stdout_%A_%a.slurm\n'.format(xp.xp_path)
+ script += '#SBATCH --error={}/stderr_%A_%a.slurm\n'.format(xp.xp_path)
+ script += '#SBATCH --time={}\n'.format(slurm_walltime)
+ # if use_gpu:
+ # script += '#SBATCH -p gpu IS NOT NULL\n'
+ # else:
+ # script += '#SBATCH -p gpu IS NULL\n'
+
+ script += 'srun -N1 -n1 run_$SLURM_ARRAY_TASK_ID.sh'
+ # script += 'echo "OAR_JOB_ID: $OAR_JOB_ID"\n'
+ # script += 'echo "OAR_ARRAY_ID: $OAR_ARRAY_ID"\n'
+ # script += 'echo "SLURM_ARRAY_TASK_ID: $SLURM_ARRAY_TASK_ID"\n'
+
+ script_path = script_dir / 'script_slurm.sh'
+ with script_path.open('w') as file:
+ file.write(script)
+ status = os.stat(script_path)
+ os.chmod(script_path, status.st_mode | stat.S_IXUSR)
+
+ # Generate a script of each array element
+ for i_elt, idt in enumerate(task_ids):
+ script = '#!/bin/sh\n'
+
+ # activate the virtual env
+ if activate_env_command is not None and len(activate_env_command) > 0:
+ script += '{}\n'.format(activate_env_command)
+
+ # python command
+ script += 'echo "Running {}.launch_experiment(task_ids={})"\n'\
+ .format(xp_var_name, idt)
+ script += 'python -c "import sys; sys.path.append(\'{0}\'); ' \
+ 'from {1} import {2}; ' \
+ '{2}.launch_experiment(task_ids={3})"\n'.format(
+ script_dir, script_name, xp_var_name, idt)
+ script += 'exit $?'
+
+ script_i_path = script_dir / 'run_{}.sh'.format(i_elt)
+ with script_i_path.open('w') as file:
+ file.write(script)
+ status = os.stat(script_i_path)
+ os.chmod(script_i_path, status.st_mode | stat.S_IXUSR)
+
+ print('*' * 80)
+ print('Submit the job array using:')
+ print('sbatch {}'.format(str(script_path)))
+ print('*' * 80)
diff --git a/python/tffpy/scripts/script_exp_variance.py b/python/tffpy/scripts/script_exp_variance.py
index e4d339e85e77e97b8e63083435a9906ab765c93f..502d8e6c6c3d7a3502098a59eea395eb3f9ae5bb 100644
--- a/python/tffpy/scripts/script_exp_variance.py
+++ b/python/tffpy/scripts/script_exp_variance.py
@@ -52,13 +52,11 @@ Run this script to handle the main experiment :class:`SolveTffExperiment`.
.. moduleauthor:: Valentin Emiya
"""
-from yafe.utils import generate_oar_script
-
import matplotlib.pyplot as plt
from tffpy.experiments.exp_variance import \
VarianceExperiment, create_and_run_light_experiment
-
+from tffpy.experiments.yafe_slurm import generate_slurm_script
try:
experiment = VarianceExperiment.get_experiment(setting='full',
@@ -74,7 +72,7 @@ if __name__ == '__main__':
input_msg = '\n'.join(['1 - Create and run light experiment',
'2 - Display results of light experiment',
'3 - Full experiment: create full experiment',
- '4 - Generate OAR script',
+ '4 - Generate Slurm script',
'5 - Full experiment: collect results',
'6 - Full experiment: download results',
'7 - Full experiment: display results',
@@ -98,13 +96,15 @@ if __name__ == '__main__':
experiment.display_status()
elif answer == 4:
experiment.display_status()
- batch_size = int(input('Batch size (#tasks per job)?'))
- generate_oar_script(script_file_path=__file__,
- xp_var_name='experiment',
- batch_size=batch_size,
- oar_walltime='01:00:00',
- activate_env_command='source activate py36',
- use_gpu=False)
+ n_simultaneous_jobs = int(
+ input('Max number of simultaneous jobs?'))
+ experiment.display_status()
+ generate_slurm_script(script_file_path=__file__,
+ xp_var_name='experiment',
+ n_simultaneous_jobs=n_simultaneous_jobs,
+ slurm_walltime='02:00:00',
+ activate_env_command='source activate py36',
+ use_gpu=False)
elif answer == 5:
experiment.collect_results()
experiment.display_status()