Skip to content
Snippets Groups Projects
Commit 0e005d38 authored by Jeremy Auguste's avatar Jeremy Auguste
Browse files

Added a way to specify oar output directories

parent 48dcbaf1
No related branches found
No related tags found
No related merge requests found
...@@ -6,6 +6,7 @@ import yaml ...@@ -6,6 +6,7 @@ import yaml
import itertools import itertools
import logging import logging
from collections import deque from collections import deque
import os
import oargen import oargen
...@@ -14,8 +15,6 @@ def argparser(): ...@@ -14,8 +15,6 @@ def argparser():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('job_file', parser.add_argument('job_file',
help="YAML file which contains all the jobs to launch") help="YAML file which contains all the jobs to launch")
parser.add_argument('destination',
help="New directory to create used to store outputs")
parser.add_argument('-b', '--besteffort', action="store_true", parser.add_argument('-b', '--besteffort', action="store_true",
help="Launch job in besteffort mode") help="Launch job in besteffort mode")
parser.add_argument('-t', '--time', default="10", parser.add_argument('-t', '--time', default="10",
...@@ -35,6 +34,8 @@ def argparser(): ...@@ -35,6 +34,8 @@ def argparser():
help="Print each individual command") help="Print each individual command")
parser.add_argument('-r', '--run', action="store_true", parser.add_argument('-r', '--run', action="store_true",
help="Run the command") help="Run the command")
parser.add_argument('-d', '--directory',
help="Creates/specifies a directory and stores oarsub outputs in it.")
parser.add_argument('-l', '--logger', default='INFO', parser.add_argument('-l', '--logger', default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help="Logging level: DEBUG, INFO (default), WARNING, ERROR") help="Logging level: DEBUG, INFO (default), WARNING, ERROR")
...@@ -115,6 +116,14 @@ def extract_job_id(cmd_output): ...@@ -115,6 +116,14 @@ def extract_job_id(cmd_output):
return None return None
def create_directory(directory, fake_run=False):
if directory is None or fake_run:
return
if os.path.isdir(directory):
return
os.mkdir(directory)
def main(): def main():
args = argparser() args = argparser()
...@@ -123,6 +132,9 @@ def main(): ...@@ -123,6 +132,9 @@ def main():
anteriors = deque() anteriors = deque()
fake_id_counter = 0 fake_id_counter = 0
create_directory(args.directory, fake_run=not args.run)
for job_name, job in jobs: for job_name, job in jobs:
if len(anteriors) < args.max_jobs: if len(anteriors) < args.max_jobs:
anterior = None anterior = None
...@@ -130,8 +142,9 @@ def main(): ...@@ -130,8 +142,9 @@ def main():
anterior = anteriors.pop() anterior = anteriors.pop()
oar_command = oargen.prepare_oarsub(args.gpu, args.host, args.core, args.time, oar_command = oargen.prepare_oarsub(args.gpu, args.host, args.core, args.time,
command=job, command=job, name=job_name,
name=job_name, besteffort=args.besteffort, output_directory=args.directory,
besteffort=args.besteffort,
checkpoint=args.checkpoint, anterior=anterior) checkpoint=args.checkpoint, anterior=anterior)
cmd_output = oargen.run_oarsub(oar_command, print_cmd=args.print_commands, cmd_output = oargen.run_oarsub(oar_command, print_cmd=args.print_commands,
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import os
import argparse import argparse
import subprocess import subprocess
...@@ -16,6 +17,8 @@ def argparser(): ...@@ -16,6 +17,8 @@ def argparser():
help="Command to use for the job (in passive mode)") help="Command to use for the job (in passive mode)")
parser.add_argument('-n', '--name', parser.add_argument('-n', '--name',
help="Name to give to the job") help="Name to give to the job")
parser.add_argument('-d', '--directory',
help="Directory in which will be stores oarsub outputs")
parser.add_argument('-b', '--besteffort', action="store_true", parser.add_argument('-b', '--besteffort', action="store_true",
help="Launch job in besteffort mode") help="Launch job in besteffort mode")
parser.add_argument('-t', '--time', default="10", parser.add_argument('-t', '--time', default="10",
...@@ -42,7 +45,8 @@ def argparser(): ...@@ -42,7 +45,8 @@ def argparser():
def prepare_oarsub(gpu, host, core, time, def prepare_oarsub(gpu, host, core, time,
command=None, argument=None, command=None, argument=None,
interactive=False, interactive=False,
name=None, besteffort=False, name=None, output_directory=None,
besteffort=False,
checkpoint=None, anterior=None): checkpoint=None, anterior=None):
oar_cmd = ["oarsub"] oar_cmd = ["oarsub"]
oar_cmd.append("-p") oar_cmd.append("-p")
...@@ -67,10 +71,11 @@ def prepare_oarsub(gpu, host, core, time, ...@@ -67,10 +71,11 @@ def prepare_oarsub(gpu, host, core, time,
if name is not None: if name is not None:
oar_cmd.append("-n") oar_cmd.append("-n")
oar_cmd.append(name) oar_cmd.append(name)
directory = output_directory + "/" if output_directory is not None else ""
oar_cmd.append("-O") oar_cmd.append("-O")
oar_cmd.append("{}.%jobid%.stdout".format(name)) oar_cmd.append("{}{}.%jobid%.stdout".format(directory, name))
oar_cmd.append("-E") oar_cmd.append("-E")
oar_cmd.append("{}.%jobid%.stderr".format(name)) oar_cmd.append("{}{}.%jobid%.stderr".format(directory, name))
if besteffort: if besteffort:
oar_cmd.extend(["-t", "besteffort", "-t", "idempotent"]) oar_cmd.extend(["-t", "besteffort", "-t", "idempotent"])
...@@ -103,9 +108,13 @@ def run_oarsub(command, print_cmd=False, fake_run=False, return_output=False): ...@@ -103,9 +108,13 @@ def run_oarsub(command, print_cmd=False, fake_run=False, return_output=False):
def main(): def main():
args = argparser() args = argparser()
if args.directory is not None and not os.path.isdir(args.directory) and args.run:
raise RuntimeError("'{}' is not a directory!".format(args.directory))
oar_command = prepare_oarsub(args.gpu, args.host, args.core, args.time, oar_command = prepare_oarsub(args.gpu, args.host, args.core, args.time,
command=args.command, argument=args.argument, command=args.command, argument=args.argument,
interactive=args.interactive, name=args.name, interactive=args.interactive,
name=args.name, output_directory=args.directory,
besteffort=args.besteffort, checkpoint=args.checkpoint, besteffort=args.besteffort, checkpoint=args.checkpoint,
anterior=args.anterior) anterior=args.anterior)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment