diff --git a/.gitignore b/.gitignore index fc07282..6cbfbde 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ codeship.aes .cache __pycache__ README.rst +.pytest_cache +.remote diff --git a/pyccc/engines/base.py b/pyccc/engines/base.py index 55db4df..773063f 100644 --- a/pyccc/engines/base.py +++ b/pyccc/engines/base.py @@ -25,6 +25,14 @@ class EngineBase(object): This class defines the implementation only - you intantiate one of its subclasses """ + USES_IMAGES = None + "bool: subclasses should set this to indicate whether they use the `job.image` field" + + ABSPATHS = None + """bool: subclasses should set this to indicate whether files can + be referenced via absolute path""" + + hostname = 'not specified' # this should be overidden in subclass init methods def __call__(self, *args, **kwargs): @@ -59,6 +67,23 @@ def launch(self, image, command, **kwargs): else: return Job(self, image, command, **kwargs) + def get_job(self, jobid): + """ Return a Job object for this job. + + The returned object will be suitable for retrieving output, but depending on the engine, + may not populate all fields used at launch time (such as `job.inputs`, `job.commands`, etc.) + + Args: + jobid (Any): job id object + + Returns: + pyccc.job.Job: job object for this job id + + Raises: + pyccc.exceptions.JobNotFound: if no job could be located for this jobid + """ + raise NotImplementedError() + def submit(self, job): """ submit job to engine diff --git a/pyccc/engines/dockerengine.py b/pyccc/engines/dockerengine.py index d7120c3..d8d2fe0 100644 --- a/pyccc/engines/dockerengine.py +++ b/pyccc/engines/dockerengine.py @@ -19,15 +19,19 @@ import subprocess -import docker +import docker.errors + from .. import docker_utils as du, DockerMachineError -from .. import utils, files, status +from .. import utils, files, status, exceptions from . import EngineBase class Docker(EngineBase): """ A compute engine - uses a docker server to run jobs """ + USES_IMAGES = True + ABSPATHS = True + def __init__(self, client=None, workingdir='/workingdir'): """ Initialization: @@ -61,6 +65,51 @@ def test_connection(self): version = self.client.version() return version + def get_job(self, jobid): + """ Return a Job object for the requested job id. + + The returned object will be suitable for retrieving output, but depending on the engine, + may not populate all fields used at launch time (such as `job.inputs`, `job.commands`, etc.) + + Args: + jobid (str): container id + + Returns: + pyccc.job.Job: job object for this container + + Raises: + pyccc.exceptions.JobNotFound: if no job could be located for this jobid + """ + import shlex + from pyccc.job import Job + + job = Job(engine=self) + job.jobid = job.rundata.containerid = jobid + try: + jobdata = self.client.inspect_container(job.jobid) + except docker.errors.NotFound: + raise exceptions.JobNotFound( + 'The daemon could not find containter "%s"' % job.jobid) + + cmd = jobdata['Config']['Cmd'] + entrypoint = jobdata['Config']['Entrypoint'] + + if len(cmd) == 3 and cmd[0:2] == ['sh', '-c']: + cmd = cmd[2] + elif entrypoint is not None: + cmd = entrypoint + cmd + + if isinstance(cmd, list): + cmd = ' '.join(shlex.quote(x) for x in cmd) + + job.command = cmd + job.env = jobdata['Config']['Env'] + job.workingdir = jobdata['Config']['WorkingDir'] + job.rundata.container = jobdata + + return job + + def submit(self, job): """ Submit job to the engine @@ -76,10 +125,10 @@ def submit(self, job): container_args = self._generate_container_args(job) - job.container = self.client.create_container(job.imageid, **container_args) - self.client.start(job.container) - job.containerid = job.container['Id'] - job.jobid = job.containerid + job.rundata.container = self.client.create_container(job.imageid, **container_args) + self.client.start(job.rundata.container) + job.rundata.containerid = job.rundata.container['Id'] + job.jobid = job.rundata.containerid def _generate_container_args(self, job): container_args = dict(command="sh -c '%s'" % job.command, @@ -104,7 +153,6 @@ def _generate_container_args(self, job): bind = '%s:%s:%s' % (volume, mountpoint, mode) else: mountpoint = mount - mode = None bind = '%s:%s' % (volume, mountpoint) volumes.append(mountpoint) @@ -117,17 +165,17 @@ def _generate_container_args(self, job): return container_args def wait(self, job): - stat = self.client.wait(job.container) + stat = self.client.wait(job.rundata.container) if isinstance(stat, int): # i.e., docker<3 return stat else: # i.e., docker>=3 return stat['StatusCode'] def kill(self, job): - self.client.kill(job.container) + self.client.kill(job.rundata.container) def get_status(self, job): - inspect = self.client.inspect_container(job.containerid) + inspect = self.client.inspect_container(job.rundata.containerid) if inspect['State']['Running']: return status.RUNNING else: @@ -135,11 +183,11 @@ def get_status(self, job): def get_directory(self, job, path): docker_host = du.kwargs_from_client(self.client) - remotedir = files.DockerArchive(docker_host, job.containerid, path) + remotedir = files.DockerArchive(docker_host, job.rundata.containerid, path) return remotedir def _list_output_files(self, job): - docker_diff = self.client.diff(job.container) + docker_diff = self.client.diff(job.rundata.container) if docker_diff is None: return {} @@ -159,11 +207,11 @@ def _list_output_files(self, job): else: relative_path = filename - remotefile = files.LazyDockerCopy(docker_host, job.containerid, filename) + remotefile = files.LazyDockerCopy(docker_host, job.rundata.containerid, filename) output_files[relative_path] = remotefile return output_files def _get_final_stds(self, job): - stdout = self.client.logs(job.container, stdout=True, stderr=False) - stderr = self.client.logs(job.container, stdout=False, stderr=True) + stdout = self.client.logs(job.rundata.container, stdout=True, stderr=False) + stderr = self.client.logs(job.rundata.container, stdout=False, stderr=True) return stdout.decode('utf-8'), stderr.decode('utf-8') diff --git a/pyccc/engines/subproc.py b/pyccc/engines/subproc.py index de51d2c..6c28dc8 100644 --- a/pyccc/engines/subproc.py +++ b/pyccc/engines/subproc.py @@ -20,7 +20,7 @@ import subprocess import locale -from pyccc import utils as utils, files +from pyccc import utils as utils, files, exceptions from . import EngineBase, status @@ -29,13 +29,15 @@ class Subprocess(EngineBase): For now, don't return anything until job completes""" hostname = 'local' + USES_IMAGES = False + ABSPATHS = False def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.term_encoding = locale.getpreferredencoding() def get_status(self, job): - if job.subproc.poll() is None: + if job.rundata.subproc.poll() is None: return status.RUNNING else: return status.FINISHED @@ -50,37 +52,38 @@ def get_engine_description(self, job): """ Return a text description for the UI """ - return 'Local subprocess %s' % job.subproc.pid + return 'Local subprocess %s' % job.rundata.subproc def launch(self, image=None, command=None, **kwargs): if command is None: command = image return super(Subprocess, self).launch('no_image', command, **kwargs) + def get_job(self, jobid): + raise NotImplementedError("Cannot retrieve jobs with the subprocess engine") + def submit(self, job): self._check_job(job) - if job.workingdir is None: - job.workingdir = utils.make_local_temp_dir() + job.rundata.localdir = utils.make_local_temp_dir() - assert os.path.isabs(job.workingdir) + assert os.path.isabs(job.rundata.localdir) if job.inputs: for filename, f in job.inputs.items(): - targetpath = self._check_file_is_under_workingdir(filename, job.workingdir) + targetpath = self._check_file_is_under_workingdir(filename, job.rundata.localdir) f.put(targetpath) subenv = os.environ.copy() subenv['PYTHONIOENCODING'] = 'utf-8' if job.env: subenv.update(job.env) - job.subproc = subprocess.Popen(job.command, - shell=True, - cwd=job.workingdir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=subenv) - job.jobid = job.subproc.pid - job._started = True - return job.subproc.pid + job.rundata.subproc = subprocess.Popen(job.command, + shell=True, + cwd=job.rundata.localdir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=subenv) + job.jobid = job.rundata.subproc.pid + return job.rundata.subproc.pid @staticmethod def _check_file_is_under_workingdir(filename, wdir): @@ -93,22 +96,22 @@ def _check_file_is_under_workingdir(filename, wdir): wdir = os.path.realpath(wdir) common = os.path.commonprefix([wdir, targetpath]) if len(common) < len(wdir): - raise ValueError( + raise exceptions.PathError( "The subprocess engine does not support input files with absolute paths") return p def kill(self, job): - job.subproc.terminate() + job.rundata.subproc.terminate() def wait(self, job): - return job.subproc.wait() + return job.rundata.subproc.wait() def get_directory(self, job, path): - targetpath = self._check_file_is_under_workingdir(path, job.workingdir) + targetpath = self._check_file_is_under_workingdir(path, job.rundata.localdir) return files.LocalDirectoryReference(targetpath) def _list_output_files(self, job, dir=None): - if dir is None: dir = job.workingdir + if dir is None: dir = job.rundata.localdir filenames = {} for fname in os.listdir(dir): abs_path = '%s/%s' % (dir, fname) @@ -124,6 +127,6 @@ def _list_output_files(self, job, dir=None): def _get_final_stds(self, job): strings = [] - for fileobj in (job.subproc.stdout, job.subproc.stderr): + for fileobj in (job.rundata.subproc.stdout, job.rundata.subproc.stderr): strings.append(fileobj.read().decode('utf-8')) return strings diff --git a/pyccc/exceptions.py b/pyccc/exceptions.py index a0bb244..0828701 100644 --- a/pyccc/exceptions.py +++ b/pyccc/exceptions.py @@ -76,4 +76,12 @@ def __init__(self, engine): class DockerMachineError(Exception): """ Failures related to connecting to docker machines + """ + +class PathError(Exception): + """ The engine can't fulfill the requested input or output filesystem path + """ + +class JobNotFound(Exception): + """ The requested job was not found """ \ No newline at end of file diff --git a/pyccc/job.py b/pyccc/job.py index 1525bb6..80fa377 100644 --- a/pyccc/job.py +++ b/pyccc/job.py @@ -22,6 +22,8 @@ import fnmatch +from mdtcollections import DotDict + import pyccc from pyccc import files, status from pyccc.utils import * @@ -30,7 +32,9 @@ def exports(o): __all__.append(o.__name__) return o -__all__ = [] + + +__all__ = ['Job'] class EngineFunction(object): @@ -95,10 +99,11 @@ def __init__(self, engine=None, self.command = if_not_none(command, '') self.engine_options = if_not_none(engine_options, {}) self.workingdir = workingdir - self.env = env + self.rundata = DotDict() + self.env = if_not_none(env, {}) - self.inputs = inputs - if self.inputs is not None: # translate strings into file objects + self.inputs = if_not_none(inputs, {}) + if self.inputs: # translate strings into file objects for filename, fileobj in inputs.items(): if isinstance(fileobj, basestring): self.inputs[filename] = files.StringContainer(fileobj) @@ -118,7 +123,6 @@ def __init__(self, engine=None, def _reset(self): self._submitted = False - self._started = False self._final_stdout = None self._final_stderr = None self._finished = False @@ -136,7 +140,7 @@ def _reset(self): def __str__(self): desc = ['Job "%s" status:%s' % (self.name, self.status)] - if self.jobid: desc.append('jobid:%s' % self.jobid) + if self.jobid: desc.append('jobid:%s' % (self.jobid,) ) if self.engine: desc.append('engine:%s' % type(self.engine).__name__) return ' '.join(desc) diff --git a/pyccc/tests/test_engine_features.py b/pyccc/tests/test_engine_features.py new file mode 100644 index 0000000..b93180a --- /dev/null +++ b/pyccc/tests/test_engine_features.py @@ -0,0 +1,94 @@ +import os +import pytest +from .engine_fixtures import subprocess_engine, local_docker_engine + +""" +Tests of features and quirks specific to engines +""" + +@pytest.fixture +def set_env_var(): + import os + assert 'NULL123' not in os.environ, "Bleeding environment" + os.environ['NULL123'] = 'nullabc' + yield + del os.environ['NULL123'] + + +def test_subprocess_environment_preserved(subprocess_engine, set_env_var): + job = subprocess_engine.launch(command='echo $NULL123', image='python:2.7-slim') + job.wait() + assert job.stdout.strip() == 'nullabc' + + +def test_readonly_docker_volume_mount(local_docker_engine): + engine = local_docker_engine + mountdir = '/tmp' + job = engine.launch(image='docker', + command='echo blah > /mounted/blah', + engine_options={'volumes': + {mountdir: ('/mounted', 'ro')}}) + job.wait() + assert isinstance(job.exitcode, int) + assert job.exitcode != 0 + + +def test_set_workingdir_docker(local_docker_engine): + engine = local_docker_engine + job = engine.launch(image='docker', command='pwd', workingdir='/testtest-dir-test') + job.wait() + assert job.stdout.strip() == '/testtest-dir-test' + + +def test_docker_volume_mount(local_docker_engine): + """ + Note: + The test context is not necessarily the same as the bind mount context! + These tests will run in containers themselves, so we can't assume + that any directories accessible to the tests are bind-mountable. + + Therefore we just test a named volume here. + """ + import subprocess, uuid + engine = local_docker_engine + key = uuid.uuid4() + volname = 'my-mounted-volume-%s' % key + + # Create a named volume with a file named "keyfile" containing a random uuid4 + subprocess.check_call(('docker volume rm {vn}; docker volume create {vn}; ' + 'docker run -v {vn}:/mounted alpine sh -c "echo {k} > /mounted/keyfile"') + .format(vn=volname, k=key), + shell=True) + + job = engine.launch(image='docker', + command='cat /mounted/keyfile', + engine_options={'volumes': {volname: '/mounted'}}) + job.wait() + result = job.stdout.strip() + assert result == str(key) + + +@pytest.mark.skipif('CI_PROJECT_ID' in os.environ, + reason="Can't mount docker socket in codeship") +def test_docker_socket_mount_with_engine_option(local_docker_engine): + engine = local_docker_engine + + job = engine.launch(image='docker', + command='docker ps -q --no-trunc', + engine_options={'mount_docker_socket': True}) + job.wait() + running = job.stdout.strip().splitlines() + assert job.jobid in running + + +@pytest.mark.skipif('CI_PROJECT_ID' in os.environ, + reason="Can't mount docker socket in codeship") +def test_docker_socket_mount_withdocker_option(local_docker_engine): + engine = local_docker_engine + + job = engine.launch(image='docker', + command='docker ps -q --no-trunc', + withdocker=True) + job.wait() + running = job.stdout.strip().splitlines() + assert job.jobid in running diff --git a/pyccc/tests/test_job_types.py b/pyccc/tests/test_engines.py similarity index 78% rename from pyccc/tests/test_job_types.py rename to pyccc/tests/test_engines.py index f136b7a..9d40fbb 100644 --- a/pyccc/tests/test_job_types.py +++ b/pyccc/tests/test_engines.py @@ -1,4 +1,24 @@ # -*- coding: utf-8 -*- +""" +Basic test battery for regular and python jobs on all underlying engines + +This can be used to test external engines (in a hacky, somewhat brittle way right now): + +```python +import pytest +from pyccc.tests import engine_fixtures + +@pytest.fixture(scope='module') +def my_engine(): + return MyEngine() + +engine_fixtures.fixture_types['engine'] = ['my_engine'] +from pyccc.tests.test_engines import * # imports all the tests +``` + +A less hacky way to via a parameterized test strategy similar to testscenarios: +https://docs.pytest.org/en/latest/example/parametrize.html#a-quick-port-of-testscenarios +""" import os import sys import pytest @@ -6,13 +26,10 @@ from .engine_fixtures import * from . import function_tests -"""Basic test battery for regular and python jobs on all underlying engines""" - PYVERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor) PYIMAGE = 'python:%s-slim' % PYVERSION THISDIR = os.path.dirname(__file__) - ######################## # Python test objects # ######################## @@ -28,6 +45,7 @@ def _raise_valueerror(msg): def test_hello_world(fixture, request): engine = request.getfuncargvalue(fixture) job = engine.launch('alpine', 'echo hello world') + print(job.rundata) job.wait() assert job.stdout.strip() == 'hello world' @@ -38,6 +56,7 @@ def test_job_status(fixture, request): job = engine.launch('alpine', 'sleep 3', submit=False) assert job.status.lower() == 'unsubmitted' job.submit() + print(job.rundata) assert job.status.lower() in ('queued', 'running', 'downloading') job.wait() assert job.status.lower() == 'finished' @@ -47,6 +66,7 @@ def test_job_status(fixture, request): def test_file_glob(fixture, request): engine = request.getfuncargvalue(fixture) job = engine.launch('alpine', 'touch a.txt b c d.txt e.gif') + print(job.rundata) job.wait() assert set(job.get_output().keys()) <= set('a.txt b c d.txt e.gif'.split()) @@ -60,6 +80,7 @@ def test_input_ouput_files(fixture, request): command='cat a.txt b.txt > out.txt', inputs={'a.txt': 'a', 'b.txt': pyccc.StringContainer('b')}) + print(job.rundata) job.wait() assert job.get_output('out.txt').read().strip() == 'ab' @@ -68,6 +89,7 @@ def test_input_ouput_files(fixture, request): def test_sleep_raises_jobstillrunning(fixture, request): engine = request.getfuncargvalue(fixture) job = engine.launch('alpine', 'sleep 5; echo done') + print(job.rundata) with pytest.raises(pyccc.JobStillRunning): job.stdout job.wait() @@ -79,6 +101,7 @@ def test_python_function(fixture, request): engine = request.getfuncargvalue(fixture) pycall = pyccc.PythonCall(function_tests.fn, 5) job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) + print(job.rundata) job.wait() assert job.result == 6 @@ -89,6 +112,7 @@ def test_python_instance_method(fixture, request): obj = function_tests.Cls() pycall = pyccc.PythonCall(obj.increment, by=2) job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) + print(job.rundata) job.wait() assert job.result == 2 @@ -100,6 +124,7 @@ def test_python_reraises_exception(fixture, request): engine = request.getfuncargvalue(fixture) pycall = pyccc.PythonCall(_raise_valueerror, 'this is my message') job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) + print(job.rundata) job.wait() with pytest.raises(ValueError): @@ -112,6 +137,7 @@ def test_builtin_imethod(fixture, request): mylist = [3, 2, 1] fn = pyccc.PythonCall(mylist.sort) job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) + print(job.rundata) job.wait() assert job.result is None # since sort doesn't return anything @@ -169,6 +195,7 @@ def test_bash_exitcode(fixture, request): command='sleep 5 && exit 35', engine=engine, submit=True) + print(job.rundata) with pytest.raises(pyccc.JobStillRunning): job.exitcode job.wait() @@ -176,26 +203,12 @@ def test_bash_exitcode(fixture, request): assert job.exitcode == 35 -@pytest.fixture -def set_env_var(): - import os - assert 'NULL123' not in os.environ, "Bleeding environment" - os.environ['NULL123'] = 'nullabc' - yield - del os.environ['NULL123'] - - -def test_subprocess_environment_preserved(subprocess_engine, set_env_var): - job = subprocess_engine.launch(command='echo $NULL123', image='python:2.7-slim') - job.wait() - assert job.stdout.strip() == 'nullabc' - - @pytest.mark.parametrize('fixture', fixture_types['engine']) def test_python_exitcode(fixture, request): engine = request.getfuncargvalue(fixture) fn = pyccc.PythonCall(function_tests.sleep_then_exit_38) job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) + print(job.rundata) with pytest.raises(pyccc.JobStillRunning): job.exitcode @@ -229,6 +242,7 @@ def test_persistence_assumptions(fixture, request): # First the control experiment - references are NOT persisted job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) + print(job.rundata) job.wait() result = job.result assert result is not testobj @@ -247,6 +261,7 @@ def test_persist_references_flag(fixture, request): # With the right flag, references ARE now persisted job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION, persist_references=True) + print(job.rundata) job.wait() result = job.result assert result is testobj @@ -268,6 +283,7 @@ def test_persistent_and_nonpersistent_mixture(fixture, request): pycall = pyccc.PythonCall(testobj.identity) job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION, persist_references=True) + print(job.rundata) job.wait() result = job.result assert result is not testobj @@ -285,6 +301,7 @@ def _callback(job): job = engine.launch(image=PYIMAGE, command='echo hello world > out.txt', when_finished=_callback) + print(job.rundata) job.wait() assert job.result == 'hello world' @@ -295,6 +312,7 @@ def test_unicode_stdout_and_return(fixture, request): engine = request.getfuncargvalue(fixture) fn = pyccc.PythonCall(function_tests.fn_prints_unicode) job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) + print(job.rundata) job.wait() assert job.result == u'¶' assert job.stdout.strip() == u'Å' @@ -309,6 +327,7 @@ def _callback(job): fn = pyccc.PythonCall(function_tests.fn, 3.0) engine = request.getfuncargvalue(fixture) job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION, when_finished=_callback) + print(job.rundata) job.wait() assert job.function_result == 4.0 @@ -327,6 +346,7 @@ def _callback(job): engine = request.getfuncargvalue(fixture) job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION, when_finished=_callback, persist_references=True) + print(job.rundata) job.wait() assert job.function_result is testobj @@ -344,103 +364,28 @@ def _runcall(fixture, request, function, *args, **kwargs): engine = request.getfuncargvalue(fixture) fn = pyccc.PythonCall(function, *args, **kwargs) job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) + print(job.rundata) job.wait() return job.result -@pytest.mark.skipif('CI_PROJECT_ID' in os.environ, - reason="Can't mount docker socket in codeship") -def test_docker_socket_mount_with_engine_option(local_docker_engine): - engine = local_docker_engine - - job = engine.launch(image='docker', - command='docker ps -q --no-trunc', - engine_options={'mount_docker_socket': True}) - job.wait() - running = job.stdout.strip().splitlines() - assert job.jobid in running - - -@pytest.mark.skipif('CI_PROJECT_ID' in os.environ, - reason="Can't mount docker socket in codeship") -def test_docker_socket_mount_withdocker_option(local_docker_engine): - engine = local_docker_engine - - job = engine.launch(image='docker', - command='docker ps -q --no-trunc', - withdocker=True) - job.wait() - running = job.stdout.strip().splitlines() - assert job.jobid in running - - -def test_docker_volume_mount(local_docker_engine): - """ - Note: - The test context is not necessarily the same as the bind mount context! - These tests will run in containers themselves, so we can't assume - that any directories accessible to the tests are bind-mountable. - - Therefore we just test a named volume here. - """ - import subprocess, uuid - engine = local_docker_engine - key = uuid.uuid4() - volname = 'my-mounted-volume-%s' % key - - # Create a named volume with a file named "keyfile" containing a random uuid4 - subprocess.check_call(('docker volume rm {vn}; docker volume create {vn}; ' - 'docker run -v {vn}:/mounted alpine sh -c "echo {k} > /mounted/keyfile"') - .format(vn=volname, k=key), - shell=True) - - job = engine.launch(image='docker', - command='cat /mounted/keyfile', - engine_options={'volumes': {volname: '/mounted'}}) - job.wait() - result = job.stdout.strip() - assert result == str(key) - - -def test_readonly_docker_volume_mount(local_docker_engine): - engine = local_docker_engine - mountdir = '/tmp' - job = engine.launch(image='docker', - command='echo blah > /mounted/blah', - engine_options={'volumes': - {mountdir: ('/mounted', 'ro')}}) - job.wait() - assert isinstance(job.exitcode, int) - assert job.exitcode != 0 - - -def test_set_workingdir_docker(local_docker_engine): - engine = local_docker_engine - job = engine.launch(image='docker', command='pwd', workingdir='/testtest-dir-test') - job.wait() - assert job.stdout.strip() == '/testtest-dir-test' - - -def test_set_workingdir_subprocess(subprocess_engine, tmpdir): - engine = subprocess_engine - job = engine.launch(image=None, command='pwd', workingdir=str(tmpdir)) - job.wait() - assert job.stdout.strip() == str(tmpdir) - - @pytest.mark.parametrize('fixture', fixture_types['engine']) def test_clean_working_dir(fixture, request): """ Because of some weird results that seemed to indicate the wrong run dir """ engine = request.getfuncargvalue(fixture) job = engine.launch(image='alpine', command='ls') + print(job.rundata) job.wait() assert job.stdout.strip() == '' -class no_context(): # context manager that does nothing that can be used conditionaly +class no_context(): + """context manager that does nothing -- useful if we need to conditionally apply a context + """ def __enter__(self): return None + def __exit__(self, exc_type, exc_value, traceback): return False @@ -448,11 +393,11 @@ def __exit__(self, exc_type, exc_value, traceback): @pytest.mark.parametrize('fixture', fixture_types['engine']) def test_abspath_input_files(fixture, request): engine = request.getfuncargvalue(fixture) - with pytest.raises(ValueError) if isinstance(engine, pyccc.Subprocess) else no_context(): - # this is OK with docker but should fail with a subprocess + with no_context() if engine.ABSPATHS else pytest.raises(pyccc.PathError): job = engine.launch(image='alpine', command='cat /opt/a', inputs={'/opt/a': pyccc.LocalFile(os.path.join(THISDIR, 'data', 'a'))}) - if not isinstance(engine, pyccc.Subprocess): + if engine.ABSPATHS: + print(job.rundata) job.wait() assert job.exitcode == 0 assert job.stdout.strip() == 'a' @@ -465,6 +410,7 @@ def test_directory_input(fixture, request): job = engine.launch(image='alpine', command='cat data/a data/b', inputs={'data': pyccc.LocalDirectoryReference(os.path.join(THISDIR, 'data'))}) + print(job.rundata) job.wait() assert job.exitcode == 0 assert job.stdout.strip() == 'a\nb' @@ -475,11 +421,13 @@ def test_passing_files_between_jobs(fixture, request): engine = request.getfuncargvalue(fixture) job1 = engine.launch(image='alpine', command='echo hello > world') + print('job1:', job1.rundata) job1.wait() assert job1.exitcode == 0 job2 = engine.launch(image='alpine', command='cat helloworld', inputs={'helloworld': job1.get_output('world')}) + print('job2:', job2.rundata) job2.wait() assert job2.exitcode == 0 assert job2.stdout.strip() == 'hello' @@ -492,6 +440,27 @@ def test_job_env_vars(fixture, request): job = engine.launch(image='alpine', command='echo ${AA} ${BB}', env={'AA': 'hello', 'BB':'world'}) + print(job.rundata) job.wait() assert job.exitcode == 0 assert job.stdout.strip() == 'hello world' + + +@pytest.mark.parametrize('fixture', fixture_types['engine']) +def test_get_job(fixture, request): + engine = request.getfuncargvalue(fixture) + job = engine.launch(image='alpine', + command='sleep 1 && echo nice nap') + + try: + newjob = engine.get_job(job.jobid) + except NotImplementedError: + pytest.skip('get_job raised NotImplementedError for %s' % fixture) + + assert job.jobid == newjob.jobid + job.wait() + assert newjob.status == job.status + + assert newjob.stdout == job.stdout + assert newjob.stderr == job.stderr + diff --git a/requirements.txt b/requirements.txt index 01f03e9..fdae1d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ docker >=3.2.1 funcsigs ; python_version < '3.3' future requests +mdtcollections tblib