Merge pull request #19 from Autodesk/volume_mounts

Volume mounts

pyccc/engines/dockerengine.py

@@ -41,7 +41,6 @@ def __init__(self, client=None, workingdir='/default_wdir'):
         self.default_wdir = workingdir
         self.hostname = self.client.base_url
 
-
     def connect_to_docker(self, client=None):
         if isinstance(client, basestring):
             client = du.get_docker_apiclient(client)
@@ -74,16 +73,48 @@ def submit(self, job):
             job.workingdir = self.default_wdir
         job.imageid = du.create_provisioned_image(self.client, job.image,
                                                   job.workingdir, job.inputs)
-        cmdstring = "sh -c '%s'" % job.command
 
-        job.container = self.client.create_container(job.imageid,
-                                                     command=cmdstring,
-                                                     working_dir=job.workingdir,
-                                                     environment={'PYTHONIOENCODING':'utf-8'})
+        container_args = self._generate_container_args(job)
+
+        job.container = self.client.create_container(job.imageid, **container_args)
         self.client.start(job.container)
         job.containerid = job.container['Id']
         job.jobid = job.containerid
 
+    def _generate_container_args(self, job):
+        container_args = dict(command="sh -c '%s'" % job.command,
+                              working_dir=job.workingdir,
+                              environment={'PYTHONIOENCODING':'utf-8'})
+
+        if job.engine_options:
+            volumes = []
+            binds = []
+
+            # mount the docker socket into the container
+            if job.engine_options.get('mount_docker_socket', False):
+                volumes.append('/var/run/docker.sock')
+                binds.append('/var/run/docker.sock:/var/run/docker.sock:rw')
+
+            # handle other mounted volumes
+            for volume, mount in job.engine_options.get('volumes', {}).items():
+                if isinstance(mount, (list, tuple)):
+                    mountpoint, mode = mount
+                    bind = '%s:%s:%s' % (volume, mountpoint, mode)
+                else:
+                    mountpoint = mount
+                    mode = None
+                    bind = '%s:%s' % (volume, mountpoint)
+
+                volumes.append(mountpoint)
+                binds.append(bind)
+
+            if volumes or binds:
+                container_args['volumes'] = volumes
+                container_args['host_config'] = self.client.create_host_config(binds=binds)
+
+        return container_args
+
+
     def wait(self, job):
         return self.client.wait(job.container)
 

pyccc/job.py

@@ -68,6 +68,7 @@ class Job(object):
             locally once, when this job completes
         numcpus (int): number of CPUs required (default:1)
         runtime (int): kill job if the runtime exceeds this value (in seconds) (default: 1 hour)`
+        engine_options (dict): additional engine-specific options
     """
     def __init__(self, engine=None,
                  image=None,
@@ -79,12 +80,14 @@ def __init__(self, engine=None,
                  numcpus=1,
                  runtime=3600,
                  on_status_update=None,
-                 when_finished=None):
+                 when_finished=None,
+                 engine_options=None):
 
         self.name = name
         self.engine = engine
         self.image = image
         self.command = if_not_none(command, '')
+        self.engine_options = engine_options
 
         self.inputs = inputs
         if self.inputs is not None:  # translate strings into file objects

pyccc/tests/test_job_types.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 import sys
 import pytest
 import pyccc
@@ -344,3 +345,55 @@ def _runcall(fixture, request, function, *args, **kwargs):
     job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION)
     job.wait()
     return job.result
+
+@pytest.mark.skipif('CI_PROJECT_ID' in os.environ,
+                    reason="Can't mount docker socket in codeship")
+def test_docker_socket_mount(local_docker_engine):
+    engine = local_docker_engine
+
+    job = engine.launch(image='docker',
+                        command='docker ps -q --no-trunc',
+                        engine_options={'mount_docker_socket': True})
+    job.wait()
+    running = job.stdout.strip().splitlines()
+    assert job.jobid in running
+
+
+def test_docker_volume_mount(local_docker_engine):
+    """
+    Note:
+        The test context is not necessarily the same as the bind mount context!
+        These tests will run in containers themselves, so we can't assume
+        that any directories accessible to the tests are bind-mountable.
+
+        Therefore we just test a named volume here.
+    """
+    import subprocess, uuid
+    engine = local_docker_engine
+    key = uuid.uuid4()
+    volname = 'my-mounted-volume-%s' % key
+
+    # Create a named volume with a file named "keyfile" containing a random uuid4
+    subprocess.check_call(('docker volume rm {vn}; docker volume create {vn}; '
+                          'docker run -v {vn}:/mounted alpine sh -c "echo {k} > /mounted/keyfile"')
+                          .format(vn=volname, k=key),
+                          shell=True)
+
+    job = engine.launch(image='docker',
+                        command='cat /mounted/keyfile',
+                        engine_options={'volumes': {volname: '/mounted'}})
+    job.wait()
+    result = job.stdout.strip()
+    assert result == str(key)
+
+
+def test_readonly_docker_volume_mount(local_docker_engine):
+    engine = local_docker_engine
+    mountdir = '/tmp'
+    job = engine.launch(image='docker',
+                        command='echo blah > /mounted/blah',
+                        engine_options={'volumes':
+                                            {mountdir: ('/mounted', 'ro')}})
+    job.wait()
+    assert isinstance(job.exitcode, int)
+    assert job.exitcode != 0

requirements.txt

@@ -1,5 +1,5 @@
 chardet
-docker > 2.0
+docker >=2.0,<3.0
 funcsigs ; python_version < '3.3'
 future
 requests