diff --git a/batchoar.py b/batchoar.py index 015fc00dc637357d324d53bdd6988e0485927fab..0f5c442fc88a19185d4781a7636f25c557cac37b 100644 --- a/batchoar.py +++ b/batchoar.py @@ -6,6 +6,7 @@ import yaml import itertools import logging from collections import deque +import os import oargen @@ -14,8 +15,6 @@ def argparser(): parser = argparse.ArgumentParser() parser.add_argument('job_file', help="YAML file which contains all the jobs to launch") - parser.add_argument('destination', - help="New directory to create used to store outputs") parser.add_argument('-b', '--besteffort', action="store_true", help="Launch job in besteffort mode") parser.add_argument('-t', '--time', default="10", @@ -35,6 +34,8 @@ def argparser(): help="Print each individual command") parser.add_argument('-r', '--run', action="store_true", help="Run the command") + parser.add_argument('-d', '--directory', + help="Creates/specifies a directory and stores oarsub outputs in it.") parser.add_argument('-l', '--logger', default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help="Logging level: DEBUG, INFO (default), WARNING, ERROR") @@ -115,6 +116,14 @@ def extract_job_id(cmd_output): return None +def create_directory(directory, fake_run=False): + if directory is None or fake_run: + return + if os.path.isdir(directory): + return + os.mkdir(directory) + + def main(): args = argparser() @@ -123,6 +132,9 @@ def main(): anteriors = deque() fake_id_counter = 0 + + create_directory(args.directory, fake_run=not args.run) + for job_name, job in jobs: if len(anteriors) < args.max_jobs: anterior = None @@ -130,8 +142,9 @@ def main(): anterior = anteriors.pop() oar_command = oargen.prepare_oarsub(args.gpu, args.host, args.core, args.time, - command=job, - name=job_name, besteffort=args.besteffort, + command=job, name=job_name, + output_directory=args.directory, + besteffort=args.besteffort, checkpoint=args.checkpoint, anterior=anterior) cmd_output = oargen.run_oarsub(oar_command, print_cmd=args.print_commands, diff --git a/oargen.py b/oargen.py index dcc1d2ea542e58652334642959cbb414a3849993..74ab3e579ebb2ba0e4bdcc6f1ac30dd54c844a37 100755 --- a/oargen.py +++ b/oargen.py @@ -4,6 +4,7 @@ from __future__ import print_function from __future__ import unicode_literals +import os import argparse import subprocess @@ -16,6 +17,8 @@ def argparser(): help="Command to use for the job (in passive mode)") parser.add_argument('-n', '--name', help="Name to give to the job") + parser.add_argument('-d', '--directory', + help="Directory in which will be stores oarsub outputs") parser.add_argument('-b', '--besteffort', action="store_true", help="Launch job in besteffort mode") parser.add_argument('-t', '--time', default="10", @@ -42,7 +45,8 @@ def argparser(): def prepare_oarsub(gpu, host, core, time, command=None, argument=None, interactive=False, - name=None, besteffort=False, + name=None, output_directory=None, + besteffort=False, checkpoint=None, anterior=None): oar_cmd = ["oarsub"] oar_cmd.append("-p") @@ -67,10 +71,11 @@ def prepare_oarsub(gpu, host, core, time, if name is not None: oar_cmd.append("-n") oar_cmd.append(name) + directory = output_directory + "/" if output_directory is not None else "" oar_cmd.append("-O") - oar_cmd.append("{}.%jobid%.stdout".format(name)) + oar_cmd.append("{}{}.%jobid%.stdout".format(directory, name)) oar_cmd.append("-E") - oar_cmd.append("{}.%jobid%.stderr".format(name)) + oar_cmd.append("{}{}.%jobid%.stderr".format(directory, name)) if besteffort: oar_cmd.extend(["-t", "besteffort", "-t", "idempotent"]) @@ -103,9 +108,13 @@ def run_oarsub(command, print_cmd=False, fake_run=False, return_output=False): def main(): args = argparser() + if args.directory is not None and not os.path.isdir(args.directory) and args.run: + raise RuntimeError("'{}' is not a directory!".format(args.directory)) + oar_command = prepare_oarsub(args.gpu, args.host, args.core, args.time, command=args.command, argument=args.argument, - interactive=args.interactive, name=args.name, + interactive=args.interactive, + name=args.name, output_directory=args.directory, besteffort=args.besteffort, checkpoint=args.checkpoint, anterior=args.anterior)