Skip to content
Snippets Groups Projects
Commit 0e005d38 authored by Jeremy Auguste's avatar Jeremy Auguste
Browse files

Added a way to specify oar output directories

parent 48dcbaf1
Branches
No related tags found
No related merge requests found
......@@ -6,6 +6,7 @@ import yaml
import itertools
import logging
from collections import deque
import os
import oargen
......@@ -14,8 +15,6 @@ def argparser():
parser = argparse.ArgumentParser()
parser.add_argument('job_file',
help="YAML file which contains all the jobs to launch")
parser.add_argument('destination',
help="New directory to create used to store outputs")
parser.add_argument('-b', '--besteffort', action="store_true",
help="Launch job in besteffort mode")
parser.add_argument('-t', '--time', default="10",
......@@ -35,6 +34,8 @@ def argparser():
help="Print each individual command")
parser.add_argument('-r', '--run', action="store_true",
help="Run the command")
parser.add_argument('-d', '--directory',
help="Creates/specifies a directory and stores oarsub outputs in it.")
parser.add_argument('-l', '--logger', default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help="Logging level: DEBUG, INFO (default), WARNING, ERROR")
......@@ -115,6 +116,14 @@ def extract_job_id(cmd_output):
return None
def create_directory(directory, fake_run=False):
if directory is None or fake_run:
return
if os.path.isdir(directory):
return
os.mkdir(directory)
def main():
args = argparser()
......@@ -123,6 +132,9 @@ def main():
anteriors = deque()
fake_id_counter = 0
create_directory(args.directory, fake_run=not args.run)
for job_name, job in jobs:
if len(anteriors) < args.max_jobs:
anterior = None
......@@ -130,8 +142,9 @@ def main():
anterior = anteriors.pop()
oar_command = oargen.prepare_oarsub(args.gpu, args.host, args.core, args.time,
command=job,
name=job_name, besteffort=args.besteffort,
command=job, name=job_name,
output_directory=args.directory,
besteffort=args.besteffort,
checkpoint=args.checkpoint, anterior=anterior)
cmd_output = oargen.run_oarsub(oar_command, print_cmd=args.print_commands,
......
......@@ -4,6 +4,7 @@
from __future__ import print_function
from __future__ import unicode_literals
import os
import argparse
import subprocess
......@@ -16,6 +17,8 @@ def argparser():
help="Command to use for the job (in passive mode)")
parser.add_argument('-n', '--name',
help="Name to give to the job")
parser.add_argument('-d', '--directory',
help="Directory in which will be stores oarsub outputs")
parser.add_argument('-b', '--besteffort', action="store_true",
help="Launch job in besteffort mode")
parser.add_argument('-t', '--time', default="10",
......@@ -42,7 +45,8 @@ def argparser():
def prepare_oarsub(gpu, host, core, time,
command=None, argument=None,
interactive=False,
name=None, besteffort=False,
name=None, output_directory=None,
besteffort=False,
checkpoint=None, anterior=None):
oar_cmd = ["oarsub"]
oar_cmd.append("-p")
......@@ -67,10 +71,11 @@ def prepare_oarsub(gpu, host, core, time,
if name is not None:
oar_cmd.append("-n")
oar_cmd.append(name)
directory = output_directory + "/" if output_directory is not None else ""
oar_cmd.append("-O")
oar_cmd.append("{}.%jobid%.stdout".format(name))
oar_cmd.append("{}{}.%jobid%.stdout".format(directory, name))
oar_cmd.append("-E")
oar_cmd.append("{}.%jobid%.stderr".format(name))
oar_cmd.append("{}{}.%jobid%.stderr".format(directory, name))
if besteffort:
oar_cmd.extend(["-t", "besteffort", "-t", "idempotent"])
......@@ -103,9 +108,13 @@ def run_oarsub(command, print_cmd=False, fake_run=False, return_output=False):
def main():
args = argparser()
if args.directory is not None and not os.path.isdir(args.directory) and args.run:
raise RuntimeError("'{}' is not a directory!".format(args.directory))
oar_command = prepare_oarsub(args.gpu, args.host, args.core, args.time,
command=args.command, argument=args.argument,
interactive=args.interactive, name=args.name,
interactive=args.interactive,
name=args.name, output_directory=args.directory,
besteffort=args.besteffort, checkpoint=args.checkpoint,
anterior=args.anterior)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment