all files for v1.0.0

requirements/doc.txt

+--index-url https://pypi.python.org/simple/
+
+nbsphinx
+numpydoc
+sphinx

requirements/xarray.txt

+--index-url https://pypi.python.org/simple/
+
+xarray>=0.10

setup.cfg

+[tool:pytest]
+testpaths = yafe
+addopts = --verbose
+          --cov-report=term-missing
+          --cov-report=html
+          --cov=yafe
+          --doctest-modules

setup.py

+#!/usr/bin/env python
+import os
+from setuptools import setup, find_packages
+import sys
+
+NAME = 'yafe'
+DESCRIPTION = 'Yet Another Framework for Experiments'
+LICENSE = 'GNU General Public License v3 (GPLv3)'
+URL = 'https://gitlab.lis-lab.fr/skmad-suite/{}'.format(NAME)
+AUTHOR = 'Ronan Hamon, Valentin Emiya, and Florent Jaillet'
+AUTHOR_EMAIL = ('ronan.hamon@lis-lab.fr, valentin.emiya@lis-lab.fr, '
+                'florent.jaillet@lis-lab.fr')
+INSTALL_REQUIRES = ['numpy']
+CLASSIFIERS = [
+    'Development Status :: 5 - Production/Stable',
+    'Intended Audience :: Developers',
+    'Intended Audience :: End Users/Desktop',
+    'Intended Audience :: Science/Research',
+    'Topic :: Scientific/Engineering :: Mathematics',
+    'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+    'Natural Language :: English',
+    'Operating System :: MacOS :: MacOS X ',
+    'Operating System :: POSIX :: Linux',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6']
+PYTHON_REQUIRES = '>=3.5'
+EXTRAS_REQUIRE = {
+    'xarray': ['xarray'],
+    'dev': ['coverage', 'pytest', 'pytest-cov', 'pytest-randomly'],
+    'doc': ['nbsphinx', 'numpydoc', 'sphinx']}
+PROJECT_URLS = {'Bug Reports': URL + '/issues',
+                'Source': URL}
+KEYWORDS = 'audio, data structures, missing data'
+
+###############################################################################
+if sys.argv[-1] == 'setup.py':
+    print("To install, run 'python setup.py install'\n")
+
+if sys.version_info[:2] < (3, 5):
+    errmsg = '{} requires Python 3.5 or later ({[0]:d}.{[1]:d} detected).'
+    print(errmsg.format(NAME, sys.version_info[:2]))
+    sys.exit(-1)
+
+
+def get_version():
+    v_text = open('VERSION').read().strip()
+    v_text_formted = '{"' + v_text.replace('\n', '","').replace(':', '":"')
+    v_text_formted += '"}'
+    v_dict = eval(v_text_formted)
+    return v_dict[NAME]
+
+
+def set_version(path, VERSION):
+    filename = os.path.join(path, '__init__.py')
+    buf = ""
+    for line in open(filename, "rb"):
+        if not line.decode("utf8").startswith("__version__ ="):
+            buf += line.decode("utf8")
+    f = open(filename, "wb")
+    f.write(buf.encode("utf8"))
+    f.write(('__version__ = "%s"\n' % VERSION).encode("utf8"))
+
+
+def setup_package():
+    """Setup function"""
+    # set version
+    VERSION = get_version()
+
+    here = os.path.abspath(os.path.dirname(__file__))
+    with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
+        long_description = f.read()
+
+    mod_dir = NAME
+    set_version(mod_dir, get_version())
+    setup(name=NAME,
+          version=VERSION,
+          description=DESCRIPTION,
+          long_description=long_description,
+          url=URL,
+          author=AUTHOR,
+          author_email=AUTHOR_EMAIL,
+          license=LICENSE,
+          classifiers=CLASSIFIERS,
+          keywords=KEYWORDS,
+          packages=find_packages(exclude=['doc', 'dev']),
+          install_requires=INSTALL_REQUIRES,
+          python_requires=PYTHON_REQUIRES,
+          extras_require=EXTRAS_REQUIRE,
+          project_urls=PROJECT_URLS)
+
+
+if __name__ == "__main__":
+    setup_package()

yafe/__init__.py

+"""Experiments as combinations of data, problems and solvers.
+
+.. moduleauthor:: Ronan Hamon
+.. moduleauthor:: Valentin Emiya
+.. moduleauthor:: Florent Jaillet
+"""
+from .base import Experiment
+
+__all__ = ['Experiment']
+__version__ = "1.0.0"

yafe/tests/script_for_generate_oar_script.py

+# -*- coding: utf-8 -*-
+"""Example script to test yafe.utils.generate_oar_script()
+
+.. moduleauthor:: Ronan Hamon
+.. moduleauthor:: Valentin Emiya
+.. moduleauthor:: Florent Jaillet
+"""
+from pathlib import Path
+import tempfile
+
+import numpy as np
+
+from yafe import Experiment
+
+
+def get_sine_data(f0, signal_len=1000):
+    return {'signal': np.sin(2*np.pi*f0*np.arange(signal_len))}
+
+
+class SimpleDenoisingProblem:
+    def __init__(self, snr_db):
+        self.snr_db = snr_db
+
+    def __call__(self, signal):
+        random_state = np.random.RandomState(0)
+        noise = random_state.randn(*signal.shape)
+        observation = signal + 10 ** (-self.snr_db / 20) * noise \
+            / np.linalg.norm(noise) * np.linalg.norm(signal)
+        problem_data = {'observation': observation}
+        solution_data = {'signal': signal}
+        return problem_data, solution_data
+
+    def __str__(self):
+        return 'SimpleDenoisingProblem(snr_db={})'.format(self.snr_db)
+
+
+class SmoothingSolver:
+    def __init__(self, filter_len):
+        self.filter_len = filter_len
+
+    def __call__(self, observation):
+        smoothing_filter = np.hamming(self.filter_len)
+        smoothing_filter /= np.sum(smoothing_filter)
+        return {'reconstruction': np.convolve(observation, smoothing_filter,
+                                              mode='same')}
+
+    def __str__(self):
+        return 'SmoothingSolver(filter_len={})'.format(self.filter_len)
+
+
+def measure(solution_data, solved_data, task_params=None, source_data=None,
+            problem_data=None):
+    euclidian_distance = np.linalg.norm(solution_data['signal']
+                                        - solved_data['reconstruction'])
+    sdr = 20 * np.log10(np.linalg.norm(solution_data['signal'])
+                        / euclidian_distance)
+    inf_distance = np.linalg.norm(solution_data['signal']
+                                  - solved_data['reconstruction'], ord=np.inf)
+    return {'sdr': sdr,
+            'euclidian_distance': euclidian_distance,
+            'inf_distance': inf_distance}
+
+
+def create_tasks(exp):
+    data_params = {'f0': np.arange(0.01, 0.1, 0.01), 'signal_len': [1000]}
+    problem_params = {'snr_db': [-10, 0, 30]}
+    solver_params = {'filter_len': 2**np.arange(6, step=2)}
+    exp.add_tasks(data_params=data_params,
+                  problem_params=problem_params,
+                  solver_params=solver_params)
+    exp.generate_tasks()
+
+
+temp_data_path = tempfile.gettempdir() / Path('yafe_temp_test_data')
+if not temp_data_path.exists():
+    temp_data_path.mkdir()
+
+
+experiment = Experiment(name='test_generate_oar_script',
+                        get_data=get_sine_data,
+                        get_problem=SimpleDenoisingProblem,
+                        get_solver=SmoothingSolver,
+                        measure=measure,
+                        data_path=temp_data_path,
+                        log_to_file=False,
+                        log_to_console=False)
+
+
+if __name__ == '__main__':
+    create_tasks(experiment)

yafe/tests/test_utils.py

+# -*- coding: utf-8 -*-
+"""Test of the module :module:`yafe.utils`
+
+.. moduleauthor:: Ronan Hamon
+.. moduleauthor:: Valentin Emiya
+.. moduleauthor:: Florent Jaillet
+"""
+from contextlib import redirect_stdout
+from io import StringIO
+import json
+import logging
+import os
+from pathlib import Path
+from shutil import rmtree
+import runpy
+import stat
+import tempfile
+import unittest
+
+import numpy as np
+
+from yafe.utils import get_logger, ConfigParser, generate_oar_script
+from .test_base import create_temp_test_config, get_data_path, MockExperiment
+
+
+class TestGetLogger(unittest.TestCase):
+
+    def setUp(self):
+        self._temp_dir = create_temp_test_config()
+
+    def tearDown(self):
+        rmtree(self._temp_dir)
+
+    def test_get_logger(self):
+
+        logger_path = ConfigParser().get_path('LOGGER', 'PATH', False)
+
+        # no saving to file
+        logger = get_logger(name='test', to_file=False, to_console=False)
+
+        logger.info('test')
+        self.assertEqual(logging.getLevelName(logger.level), 'DEBUG')
+        self.assertEqual(len(logger.handlers), 0)
+
+        # saving to file with path from config file
+        self.assertFalse(logger_path.exists())
+
+        logger = get_logger(name='test')
+
+        self.assertTrue(logger_path.exists())
+
+        self.assertEqual(logging.getLevelName(logger.level), 'DEBUG')
+        self.assertEqual(len(logger.handlers), 2)
+
+        self.assertEqual(
+            logging.getLevelName(logger.handlers[0].level), 'DEBUG')
+        self.assertEqual(logger.handlers[0].stream.name, str(logger_path))
+
+        self.assertEqual(
+            logging.getLevelName(logger.handlers[1].level), 'INFO')
+
+        # saving to file with specified path
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        tmp_file.close()
+        try:
+            logger = get_logger(name='test',
+                                logger_path=tmp_file.name)
+
+            self.assertEqual(logging.getLevelName(logger.level), 'DEBUG')
+            self.assertEqual(len(logger.handlers), 2)
+
+            self.assertEqual(
+                logging.getLevelName(logger.handlers[0].level), 'DEBUG')
+            self.assertEqual(logger.handlers[0].stream.name, tmp_file.name)
+
+            self.assertEqual(
+                logging.getLevelName(logger.handlers[1].level), 'INFO')
+        finally:
+            os.remove(tmp_file.name)
+
+
+class TestConfigParser(unittest.TestCase):
+
+    def setUp(self):
+        self._temp_dir = create_temp_test_config()
+
+    def tearDown(self):
+        rmtree(self._temp_dir)
+
+    def test_generate_config_and_get_path(self):
+        original_config_path = ConfigParser._config_path
+        temp_dir = tempfile.mkdtemp(prefix='yafe_')
+        try:
+            ConfigParser._config_path = temp_dir / Path('yafe.log')
+            ConfigParser().generate_config()
+
+            # Check that the config file has been created
+            self.assertTrue(ConfigParser._config_path.exists())
+
+            config = ConfigParser()
+            # Check that the USER section exists with an empty data_path
+            user_data_path = config.get_path('USER', 'data_path')
+            self.assertIs(user_data_path, None)
+            # Check that the LOGGER section exists with an empty path
+            logger_path = config.get_path('LOGGER', 'path')
+            self.assertIs(logger_path, None)
+
+            # Further check that get_path() works as expected
+            config['TEST_SECTION'] = {'good_path': temp_dir,
+                                      'bad_path': '/non_existing_path'}
+            with ConfigParser._config_path.open('w') as configfile:
+                config.write(configfile)
+            good_path = config.get_path('TEST_SECTION', 'good_path')
+            self.assertEqual(temp_dir, str(good_path))
+            with self.assertRaises(IOError):
+                bad_path = config.get_path('TEST_SECTION', 'bad_path')
+
+        finally:
+            ConfigParser._config_path = original_config_path
+            rmtree(temp_dir)
+
+
+class TestGenerateOarScript(unittest.TestCase):
+
+    def tearDown(self):
+        rmtree(tempfile.gettempdir() / Path('yafe_temp_test_data'))
+
+    def test_generate_oar_script(self):
+        script_file_path = (Path(__file__).parent
+                            / 'script_for_generate_oar_script.py')
+        xp_var_name = 'experiment'
+        temp_dir = tempfile.gettempdir()
+        xp_path = (temp_dir / Path('yafe_temp_test_data')
+                   / 'test_generate_oar_script')
+
+        runpy.run_path(str(script_file_path), run_name='__main__')
+
+        tmp_stdout = StringIO()
+        with redirect_stdout(tmp_stdout):
+            generate_oar_script(script_file_path, xp_var_name)
+
+        out = tmp_stdout.getvalue().strip()
+        expected_out = ('oarsub -S {}/yafe_temp_test_data/test_generate_oar'
+                        '_script/script_oar.sh'.format(temp_dir))
+        self.assertEqual(out, expected_out)
+
+        task_file_path = xp_path / "listoftasks.txt"
+        self.assertTrue(task_file_path.exists())
+        task_data = np.loadtxt(task_file_path, dtype=np.int64)
+        np.testing.assert_array_equal(np.arange(81), task_data)
+
+        tasks = [1, 3]
+        generate_oar_script(script_file_path, xp_var_name, tasks)
+
+        task_file_path = xp_path / "listoftasks.txt"
+        self.assertTrue(task_file_path.exists())
+        task_data = np.loadtxt(task_file_path, dtype=np.int64)
+        np.testing.assert_array_equal(tasks, task_data)
+
+        script_path = xp_path / "script_oar.sh"
+        self.assertTrue(script_path.exists())
+        # Check that the script is executable
+        self.assertTrue(os.stat(script_path).st_mode & stat.S_IXUSR)
+        # Check the content of the script
+        with script_path.open() as script_file:
+            script_content = script_file.readlines()
+        self.assertEqual(len(script_content), 13)
+        expected_content = [
+            '#!/bin/sh\n',
+            '#OAR -n test_generate_oar_script\n',
+            '#OAR --array-param-file {}/yafe_temp_test_data/'
+            'test_generate_oar_script/listoftasks.txt\n'.format(temp_dir),
+            '#OAR -O {}/yafe_temp_test_data/'
+            'test_generate_oar_script/%jobid%.out\n'.format(temp_dir),
+            '#OAR -E {}/yafe_temp_test_data/'
+            'test_generate_oar_script/%jobid%.err\n'.format(temp_dir),
+            '#OAR -l walltime=02:00:00\n',
+            '#OAR -p gpu IS NULL\n',
+            'echo "OAR_JOB_ID: $OAR_JOB_ID"\n',
+            'echo "OAR_ARRAY_ID: $OAR_ARRAY_ID"\n',
+            'echo "OAR_ARRAY_INDEX: $OAR_ARRAY_INDEX"\n',
+            'echo "Running experiment.launch_experiment(task_ids=\'$1\')"\n',
+            'python -c "import sys; sys.path.append(\'{}\'); '
+            'from script_for_generate_oar_script import experiment; '
+            'experiment.launch_experiment(task_ids=\'$1\')"\n'.format(
+                Path(__file__).parent),
+            'exit $?']
+        self.assertEqual(script_content, expected_content)
+
+        activate_env_command = 'source activate test_env'
+        generate_oar_script(script_file_path, xp_var_name,
+                            activate_env_command=activate_env_command)
+        self.assertTrue(script_path.exists())
+        with script_path.open() as script_file:
+            script_content = script_file.readlines()
+        self.assertEqual(len(script_content), 14)
+        expected_content.insert(10, '{}\n'.format(activate_env_command))
+        self.assertEqual(script_content, expected_content)
+
+        generate_oar_script(script_file_path, xp_var_name,
+                            use_gpu=True)
+        self.assertTrue(script_path.exists())
+        with script_path.open() as script_file:
+            script_content = script_file.readlines()
+        self.assertEqual(len(script_content), 13)
+        expected_content.pop(10)
+        expected_content[6] = '#OAR -p gpu IS NOT NULL\n'
+        self.assertEqual(script_content, expected_content)

yafe/utils.py

+# -*- coding: utf-8 -*-
+"""Utils classes and functions for yafe.
+
+.. moduleauthor:: Ronan Hamon
+.. moduleauthor:: Valentin Emiya
+.. moduleauthor:: Florent Jaillet
+"""
+import configparser
+import importlib
+import logging
+import os
+from pathlib import Path
+import stat
+import sys
+
+
+class ConfigParser(configparser.ConfigParser):
+    """Configuration file parser for yafe.
+
+    This class inherits from ConfigParser in the configparser module.
+
+    It enables reading the yafe configuration file ``$HOME/.config/yafe.conf``
+    at initialization of an experiment. It also provides a method to properly
+    read a path in the configuration file, and a static method to generate a
+    basic empty configuration file.
+    """
+    _config_path = Path(os.path.expanduser('~')) / '.config' / 'yafe.conf'
+
+    def __init__(self):
+        super().__init__()
+        loaded_files = super().read(self._config_path)
+        if not len(loaded_files):
+            print('Generation of the config file.')
+            ConfigParser.generate_config()
+            print('Please update config file at {}'.format(self._config_path))
+            super().read(self._config_path)
+
+    def get_path(self, section, option, check_exists=True):
+        """Get the path filled in a given option of a given section.
+
+        Parameters
+        ----------
+        section : str
+            Name of the section.
+        option : str
+            Name of the option.
+        check_exists : bool, optional
+            Indicates if the existence of the path is checked.
+
+        Returns
+        -------
+        pathlib.Path or None
+            Path if the option is defined, None otherwise.
+
+        Raises
+        ------
+        IOError
+            If the parameter ``check_exists`` is set to ``True`` and the path
+            does not exist.
+        """
+        path = super().get(section, option)
+
+        if path == '':
+            return None
+
+        path = Path(path)
+
+        if check_exists and not path.exists():
+            errmsg = 'Path {} does not exist.'
+            raise IOError(errmsg.format(path))
+
+        return path
+
+    @staticmethod
+    def generate_config():
+        """Generate an empty configuration file.
+
+        The generated configuration file is stored in
+        ``$HOME/.config/yafe.conf``.
+        """
+
+        config = configparser.ConfigParser(allow_no_value=True)
+
+        config.add_section('USER')
+        config.set('USER', '# user directory (data generated by experiments)')
+        config.set('USER', 'data_path', '')
+
+        config.add_section('LOGGER')
+        config.set('LOGGER', '# path to the log file')
+        config.set('LOGGER', 'path', '')
+
+        with open(str(ConfigParser._config_path), 'w') as config_file:
+            config.write(config_file)
+
+
+def get_logger(name='', to_file=True, to_console=True, logger_path=None):
+    """Return a yafe logger with the given name.
+
+    Parameters
+    ----------
+    name : str
+        Name of the logger.
+    to_file : bool
+        Indicates if the logger writes processing and debug information in a
+        file.
+    to_console : bool
+        Indicates if the logger displays processing information in the console.
+    logger_path : None or str or pathlib.Path
+        Full file path where the data must be written by the logger when
+        ``to_file == True``.
+        If ``logger_path`` is ``None`` the logger path given in the yafe
+        configuration file is used.
+
+    Returns
+    -------
+    logging.Logger
+        A logger with the given name prefixed by ``yafe.``.
+
+    Notes
+    -----
+    The name of the logger is automatically prefixed by ``yafe.``.
+    """
+    logger = logging.getLogger('yafe.{}'.format(name))
+    logger.handlers = []
+    logger.setLevel(logging.DEBUG)
+
+    if to_file:
+        if logger_path is None:
+            config = ConfigParser()
+            logger_path = config.get_path('LOGGER', 'PATH', check_exists=False)
+        else:
+            logger_path = Path(logger_path)
+        if not logger_path.exists():
+            logger_path.touch()
+        file_handler = logging.FileHandler(str(logger_path))
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(
+            logging.Formatter(
+                '%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+
+        logger.addHandler(file_handler)
+
+    if to_console:
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(logging.INFO)
+        console_handler.setFormatter(
+            logging.Formatter(
+                '%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+
+        logger.addHandler(console_handler)
+
+    return logger
+
+
+def generate_oar_script(script_file_path, xp_var_name, task_ids=None,
+                        batch_size=1, oar_walltime='02:00:00',
+                        activate_env_command=None, use_gpu=False):
+    """Generate a script to launch an experiment using OAR.
+
+    Tasks are divided into batches that are executed by oar jobs.
+
+    The resulting script is written in the experiment folder, and the command
+    to launch the jobs with OAR is displayed in the terminal.
+
+    An example script illustrating how to use
+    :func:`yafe.utils.generate_oar_script` is available in the corresponding
+    :ref:`tutorial <tutorial_oar>`.
+
+    Parameters
+    ----------
+    script_file_path : str
+        File path to the script that defines the experiment.
+    xp_var_name : str
+        Name of the variable containing the experiment in the script.
+    task_ids : list
+        List of tasks ids to run.
+        If ``task_ids`` is ``None``, the list of pending tasks of the
+        experiment is used.
+    batch_size : int
+        Number of tasks run in each batch.
+    oar_walltime : str
+        Wall time for each OAR job ('HH:MM:SS').
+    activate_env_command : str or None
+        Optional command that must be run to activate a Python virtual
+        environment before launching the experiment.
+        Typically, this is a command of the form
+        ``source some_virtual_env/bin/activate`` when using virtualenv and
+        ``source activate some_conda_env`` when using conda.
+        If ``activate_env_command`` is ``None``, no virtual environment is
+        activated.
+    use_gpu : bool
+        Flag specifying if a gpu ressource is needed when running the
+        experiment.
+    """
+    script_file_path = Path(script_file_path)
+    script_dir = script_file_path.parent
+    script_name = script_file_path.stem
+
+    sys.path.append(str(script_dir))
+    mod = importlib.import_module(script_name)
+    xp = getattr(mod, xp_var_name)
+
+    if task_ids is None:
+        task_ids = xp.get_pending_task_ids()
+
+    # split and save the tasks
+    task_ids = list(map(str, task_ids))
+    batches = [
+        task_ids[i:(i + batch_size)]
+        for i in range(0, len(task_ids), batch_size)
+    ]
+    file_path = xp.xp_path / 'listoftasks.txt'
+
+    with open(str(file_path), 'wt') as fout:
+        fout.write('\n'.join(map(lambda batch: ','.join(batch), batches)))
+
+    # generate and save script
+    script_path = Path(os.path.abspath(script_file_path))
+    script_dir = script_path.parent
+    script_name = script_path.stem
+
+    script = '#!/bin/sh\n'
+
+    # define the OAR parameters
+    script += '#OAR -n {}\n'.format(xp.name)
+    script += '#OAR --array-param-file {}\n'.format(str(file_path))
+    script += '#OAR -O {}/%jobid%.out\n'.format(xp.xp_path)
+    script += '#OAR -E {}/%jobid%.err\n'.format(xp.xp_path)
+    script += '#OAR -l walltime={}\n'.format(oar_walltime)
+    if use_gpu:
+        script += '#OAR -p gpu IS NOT NULL\n'
+    else:
+        script += '#OAR -p gpu IS NULL\n'
+
+    script += 'echo "OAR_JOB_ID: $OAR_JOB_ID"\n'
+    script += 'echo "OAR_ARRAY_ID: $OAR_ARRAY_ID"\n'
+    script += 'echo "OAR_ARRAY_INDEX: $OAR_ARRAY_INDEX"\n'
+
+    # activate the virtual env
+    if activate_env_command is not None and len(activate_env_command) > 0:
+        script += '{}\n'.format(activate_env_command)
+
+    # python command
+    script += 'echo "Running {}.launch_experiment(task_ids=\'$1\')"\n'.format(
+        xp_var_name)
+    script += 'python -c "import sys; sys.path.append(\'{0}\'); ' \
+        'from {1} import {2}; ' \
+        '{2}.launch_experiment(task_ids=\'$1\')"\n'.format(
+            script_dir, script_name, xp_var_name)
+    script += 'exit $?'
+
+    script_path = xp.xp_path / 'script_oar.sh'
+    with script_path.open('w') as file:
+        file.write(script)
+    status = os.stat(script_path)
+    os.chmod(script_path, status.st_mode | stat.S_IXUSR)
+
+    print('oarsub -S {}'.format(str(script_path)))