From 0add734ee5c98ad6aeb5d175b0100ec4c96e5225 Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 16 May 2017 11:52:37 -0700 Subject: [PATCH 1/3] Fix some pickling bugs and ui issues --- prep_release.py | 69 ---------------------------------- pyccc/python.py | 2 +- pyccc/source_inspections.py | 31 +++++++++++---- pyccc/static/run_job.py | 36 +++++++++++++----- pyccc/tests/__init__.py | 0 pyccc/tests/engine_fixtures.py | 40 ++++++++++++++++++++ pyccc/tests/test_job_types.py | 67 ++++++++++++++------------------- pyccc/ui.py | 5 ++- 8 files changed, 124 insertions(+), 126 deletions(-) delete mode 100755 prep_release.py create mode 100644 pyccc/tests/__init__.py create mode 100644 pyccc/tests/engine_fixtures.py diff --git a/prep_release.py b/prep_release.py deleted file mode 100755 index e2b4c58..0000000 --- a/prep_release.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -""" This script builds and runs automated tests for a new moldesign release. -It's manual for now, will be replaced by Travis or Jenkins soon. - -You shouldn't run this unless you know exactly what you're doing and why you're doing it -""" - -import os -import sys -import subprocess -import atexit - -version = sys.argv[1] - - -def run(cmd, display=True): - print '\n>>> %s' % cmd - if display: - return subprocess.check_call(cmd, shell=True) - else: - return subprocess.check_output(cmd, shell=True) - - -tags = set(run('git tag --list', display=False).splitlines()) -rootdir = run('git rev-parse --show-toplevel', display=False).strip() - -assert os.path.abspath(rootdir) == os.path.abspath(os.path.curdir), \ - "This command must be run at root of the repository directory" - -# check that tag is valid -assert version not in tags, "Tag %s already exists!" % version -major, minor, patch = map(int, version.split('.')) - - -# Set the tag! -run('git tag %s' % version) -print 'Tag set: %s' % version - - -def untag(): - if not untag.success: - print 'Failed. Removing version tag %s' % version - run('git tag -d %s' % version) - -untag.success = False - -atexit.register(untag) - -# Check that it propagated to the python package -import pyccc -assert os.path.abspath(os.path.join(pyccc.__path__[0],'..')) == os.path.abspath(rootdir) -assert pyccc.__version__ == version, 'Package has incorrect version: %s' % pyccc.__version__ - -untag.success = True - -# This is the irreversible part - so do it manually -print """ -This LOOKS ready for release! Do the following to create version %s. -If you're ready, run these commands: -1. python setup.py register -r pypi -2. python setup.py sdist upload -r pypi -3. git push origin master --tags - -Finally, mark it as release "v%s" in GitHub. - -TO ABORT, RUN: -git tag -d %s -""" % (version, version, version) - diff --git a/pyccc/python.py b/pyccc/python.py index c74595d..2df4d63 100644 --- a/pyccc/python.py +++ b/pyccc/python.py @@ -238,7 +238,7 @@ class PackagedFunction(native.object): """ def __init__(self, function_call): func = function_call.function - self.is_imethod = hasattr(func, '__self__') + self.is_imethod = getattr(func, '__self__', None) is not None if self.is_imethod: self.obj = func.__self__ self.imethod_name = func.__name__ diff --git a/pyccc/source_inspections.py b/pyccc/source_inspections.py index 0889f32..ce17313 100644 --- a/pyccc/source_inspections.py +++ b/pyccc/source_inspections.py @@ -15,6 +15,7 @@ Source code inspections for sending python code to workers """ from __future__ import print_function, unicode_literals, absolute_import, division + from future import standard_library, builtins standard_library.install_aliases() from future.builtins import * @@ -23,7 +24,6 @@ import linecache import re import string -from collections import namedtuple __author__ = 'aaronvirshup' @@ -48,7 +48,7 @@ def get_global_vars(func): for name, value in closure['global'].items(): if inspect.ismodule(value): # TODO: deal FUNCTIONS from closure globalvars['modules'][name] = value.__name__ - elif inspect.isfunction(value): + elif inspect.isfunction(value) or inspect.ismethod(value): globalvars['functions'][name] = value else: globalvars['vars'][name] = value @@ -69,6 +69,9 @@ def getsource(classorfunc): Returns: str: source code (without any decorators) """ + if _isbuiltin(classorfunc): + return '' + try: source = inspect.getsource(classorfunc) except TypeError: # raised if defined in __main__ - use fallback to get the source instead @@ -195,13 +198,18 @@ def getclosurevars(func): if inspect.ismethod(func): func = func.__func__ - if not inspect.isfunction(func): + elif not inspect.isroutine(func): raise TypeError("'{!r}' is not a Python function".format(func)) - code = func.__code__ + # AMVMOD: deal with python 2 builtins that don't define these + code = getattr(func, '__code__', None) + closure = getattr(func, '__closure__', None) + co_names = getattr(code, 'co_names', ()) + glb = getattr(func, '__globals__', {}) + # Nonlocal references are named in co_freevars and resolved # by looking them up in __closure__ by positional index - if func.__closure__ is None: + if closure is None: nonlocal_vars = {} else: nonlocal_vars = {var: cell.cell_contents @@ -209,14 +217,14 @@ def getclosurevars(func): # Global and builtin references are named in co_names and resolved # by looking them up in __globals__ or __builtins__ - global_ns = func.__globals__ + global_ns = glb builtin_ns = global_ns.get("__builtins__", builtins.__dict__) if inspect.ismodule(builtin_ns): builtin_ns = builtin_ns.__dict__ global_vars = {} builtin_vars = {} unbound_names = set() - for name in code.co_names: + for name in co_names: if name in ("None", "True", "False"): # Because these used to be builtins instead of keywords, they # may still show up as name references. We ignore them. @@ -233,3 +241,12 @@ def getclosurevars(func): 'global': global_vars, 'builtin': builtin_vars, 'unbound': unbound_names} + + +def _isbuiltin(obj): + if inspect.isbuiltin(obj): + return True + elif obj.__module__ in ('builtins', '__builtin__'): + return True + else: + return False diff --git a/pyccc/static/run_job.py b/pyccc/static/run_job.py index 132928b..5dc3c84 100644 --- a/pyccc/static/run_job.py +++ b/pyccc/static/run_job.py @@ -34,6 +34,14 @@ RENAMETABLE = {'pyccc.python': 'source', '__main__': 'source'} +if sys.version_info.major == 2: + PYVERSION = 2 + import __builtin__ as BUILTINS +else: + assert sys.version_info.major == 3 + PYVERSION = 3 + import builtins as BUILTINS + def main(): os.environ['IS_PYCCC_JOB'] = '1' @@ -58,7 +66,10 @@ def load_job(): funcpkg = MappedUnpickler(pf).load() if hasattr(funcpkg, 'func_name'): - func = getattr(source, funcpkg.func_name) + try: + func = getattr(source, funcpkg.func_name) + except AttributeError: + func = getattr(BUILTINS, funcpkg.func_name) else: func = None @@ -103,17 +114,24 @@ def find_class(self, module, name): # can't use ``super`` here (not 2/3 compatible) klass = pickle.Unpickler.find_class(self, modname, name) - except ImportError: - if hasattr(source, name): - newmod = types.ModuleType(modname) - sys.modules[modname] = newmod - setattr(newmod, name, getattr(source, name)) - klass = self.find_class(modname, name) - - klass.__module__ = module + except (ImportError, RuntimeError): + definition = getattr(source, name) + newmod = _makemod(modname) + sys.modules[modname] = newmod + setattr(newmod, name, definition) + klass = pickle.Unpickler.find_class(self, newmod.__name__, name) + klass.__module__ = module return klass +def _makemod(name): + fields = name.split('.') + for i in range(0, len(fields)): + tempname = str('.'.join(fields[0:i])) + sys.modules[tempname] = types.ModuleType(tempname) + return sys.modules[tempname] + + if __name__ == '__main__': main() diff --git a/pyccc/tests/__init__.py b/pyccc/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyccc/tests/engine_fixtures.py b/pyccc/tests/engine_fixtures.py new file mode 100644 index 0000000..220dfe6 --- /dev/null +++ b/pyccc/tests/engine_fixtures.py @@ -0,0 +1,40 @@ +import pytest +import pyccc + +__all__ = 'typedfixture fixture_types subprocess_engine public_ccc_engine local_docker_engine'.split() + +fixture_types = {} + + +def typedfixture(*types, **kwargs): + """This is a decorator that lets us associate fixtures with one or more arbitrary types. + This makes it easy, later, to run the same test on all fixtures of a given type""" + + def fixture_wrapper(func): + for t in types: + fixture_types.setdefault(t, []).append(func.__name__) + return pytest.fixture(**kwargs)(func) + + return fixture_wrapper + + +################### +# Fixtures # +################### + +@typedfixture('engine') +def subprocess_engine(): + return pyccc.Subprocess() + + +@typedfixture('engine') +def public_ccc_engine(): + if not pytest.config.getoption("--testccc"): + pytest.skip("need --testccc option to run") + else: + return pyccc.CloudComputeCannon('cloudcomputecannon.bionano.autodesk.com:9000') + + +@typedfixture('engine') +def local_docker_engine(): + return pyccc.Docker() diff --git a/pyccc/tests/test_job_types.py b/pyccc/tests/test_job_types.py index 67b17fc..5b5668b 100644 --- a/pyccc/tests/test_job_types.py +++ b/pyccc/tests/test_job_types.py @@ -1,28 +1,13 @@ -from itertools import product - import sys import pytest import pyccc +from .engine_fixtures import * """Basic test battery for regular and python jobs on all underlying engines""" REMOTE_PYTHON_VERSIONS = {2: '2.7', 3: '3.6'} PYVERSION = REMOTE_PYTHON_VERSIONS[sys.version_info.major] -fixture_types = {} - - - -def typedfixture(*types, **kwargs): - """This is a decorator that lets us associate fixtures with one or more arbitrary types. - This makes it easy, later, to run the same test on all fixtures of a given type""" - - def fixture_wrapper(func): - for t in types: - fixture_types.setdefault(t, []).append(func.__name__) - return pytest.fixture(**kwargs)(func) - - return fixture_wrapper ######################## @@ -45,27 +30,6 @@ def _raise_valueerror(msg): raise ValueError(msg) -################### -# Fixtures # -################### - -@typedfixture('engine') -def subprocess_engine(): - return pyccc.Subprocess() - - -@typedfixture('engine') -def public_ccc_engine(): - if not pytest.config.getoption("--testccc"): - pytest.skip("need --testccc option to run") - else: - return pyccc.CloudComputeCannon('cloudcomputecannon.bionano.autodesk.com:9000') - - -@typedfixture('engine') -def local_docker_engine(): - return pyccc.Docker() - ################### # Tests # @@ -113,7 +77,7 @@ def test_python_instance_method(fixture, request): engine = request.getfuncargvalue(fixture) obj = _TestCls() pycall = pyccc.PythonCall(obj.increment, by=2) - job = engine.launch('python:%s-slim'%PYVERSION, pycall, interpreter=PYVERSION) + job = engine.launch('python:%s-slim' % PYVERSION, pycall, interpreter=PYVERSION) job.wait() assert job.result == 2 @@ -124,9 +88,34 @@ def test_python_instance_method(fixture, request): def test_python_reraises_exception(fixture, request): engine = request.getfuncargvalue(fixture) pycall = pyccc.PythonCall(_raise_valueerror, 'this is my message') - job = engine.launch('python:%s-slim'%PYVERSION, pycall, interpreter=PYVERSION) + job = engine.launch('python:%s-slim' % PYVERSION, pycall, interpreter=PYVERSION) job.wait() with pytest.raises(ValueError): job.result + +@pytest.mark.parametrize('fixture', fixture_types['engine']) +def test_builtin_imethod(fixture, request): + engine = request.getfuncargvalue(fixture) + mylist = [3, 2, 1] + fn = pyccc.PythonCall(mylist.sort) + job = engine.launch(image='python:2.7-slim', command=fn, interpreter=PYVERSION) + job.wait() + + assert job.result is None # since sort doesn't return anything + assert job.updated_object == [1, 2, 3] + + +@pytest.mark.parametrize('fixture', fixture_types['engine']) +def test_builtin_function(fixture, request): + engine = request.getfuncargvalue(fixture) + mylist = [3, 2, 1] + fn = pyccc.PythonCall(sorted, mylist) + job = engine.launch(image='python:2.7-slim', command=fn, interpreter=PYVERSION) + job.wait() + + assert job.result == [1,2,3] + + + diff --git a/pyccc/ui.py b/pyccc/ui.py index 57cd44f..3129142 100644 --- a/pyccc/ui.py +++ b/pyccc/ui.py @@ -125,7 +125,10 @@ def __init__(self, fileobj, **kwargs): self.download_button.on_click(self.handle_download_click) # if it's file-like, get the _contents elif hasattr(fileobj, 'read'): - self._string = fileobj.read() + try: + self._string = fileobj.read() + except UnicodeDecodeError: + self._string = '[NOT SHOWN - UNABLE TO DECODE FILE]' self.render_string() # Just display a string else: From 8ee7c5491d2d450bdf233afb8ca159ea588401d1 Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 16 May 2017 12:13:52 -0700 Subject: [PATCH 2/3] Get everything working in 2/3 --- pyccc/python.py | 19 +++++++++++++++---- pyccc/tests/test_job_types.py | 12 ++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pyccc/python.py b/pyccc/python.py index 2df4d63..8dc8332 100644 --- a/pyccc/python.py +++ b/pyccc/python.py @@ -42,6 +42,14 @@ def exports(o): DEFAULT_INTERPRETER = 'python%s.%s' % sys.version_info[:2] PICKLE_PROTOCOL = 2 # required for 2/3 compatibile pickle objects +if sys.version_info.major == 2: + PYVERSION = 2 + import __builtin__ as BUILTINS +else: + assert sys.version_info.major == 3 + PYVERSION = 3 + import builtins as BUILTINS + @exports class PythonCall(object): @@ -51,12 +59,14 @@ def __init__(self, function, *args, **kwargs): self.kwargs = kwargs try: - temp = function.__self__.__class__ + cls = function.__self__.__class__ except AttributeError: self.is_instancemethod = False else: - self.is_instancemethod = True - + if function.__self__ == BUILTINS or function.__self__ is None: + self.is_instancemethod = False + else: + self.is_instancemethod = True @exports class PythonJob(job.Job): @@ -111,6 +121,7 @@ def _get_python_files(self): python_files['function.pkl'] = pyccc.BytesContainer( pickle.dumps(remote_function, protocol=PICKLE_PROTOCOL), name='function.pkl') + self._remote_function = remote_function sourcefile = StringContainer(self._get_source(), name='source.py') @@ -238,7 +249,7 @@ class PackagedFunction(native.object): """ def __init__(self, function_call): func = function_call.function - self.is_imethod = getattr(func, '__self__', None) is not None + self.is_imethod = function_call.is_instancemethod if self.is_imethod: self.obj = func.__self__ self.imethod_name = func.__name__ diff --git a/pyccc/tests/test_job_types.py b/pyccc/tests/test_job_types.py index 5b5668b..323175f 100644 --- a/pyccc/tests/test_job_types.py +++ b/pyccc/tests/test_job_types.py @@ -7,7 +7,7 @@ REMOTE_PYTHON_VERSIONS = {2: '2.7', 3: '3.6'} PYVERSION = REMOTE_PYTHON_VERSIONS[sys.version_info.major] - +PYIMAGE = 'python:%s-slim' % PYVERSION ######################## @@ -67,7 +67,7 @@ def test_sleep_raises_jobstillrunning(fixture, request): def test_python_function(fixture, request): engine = request.getfuncargvalue(fixture) pycall = pyccc.PythonCall(_testfun, 5) - job = engine.launch('python:%s-slim'%PYVERSION, pycall, interpreter=PYVERSION) + job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) job.wait() assert job.result == 6 @@ -77,7 +77,7 @@ def test_python_instance_method(fixture, request): engine = request.getfuncargvalue(fixture) obj = _TestCls() pycall = pyccc.PythonCall(obj.increment, by=2) - job = engine.launch('python:%s-slim' % PYVERSION, pycall, interpreter=PYVERSION) + job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) job.wait() assert job.result == 2 @@ -88,7 +88,7 @@ def test_python_instance_method(fixture, request): def test_python_reraises_exception(fixture, request): engine = request.getfuncargvalue(fixture) pycall = pyccc.PythonCall(_raise_valueerror, 'this is my message') - job = engine.launch('python:%s-slim' % PYVERSION, pycall, interpreter=PYVERSION) + job = engine.launch(PYIMAGE, pycall, interpreter=PYVERSION) job.wait() with pytest.raises(ValueError): @@ -100,7 +100,7 @@ def test_builtin_imethod(fixture, request): engine = request.getfuncargvalue(fixture) mylist = [3, 2, 1] fn = pyccc.PythonCall(mylist.sort) - job = engine.launch(image='python:2.7-slim', command=fn, interpreter=PYVERSION) + job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) job.wait() assert job.result is None # since sort doesn't return anything @@ -112,7 +112,7 @@ def test_builtin_function(fixture, request): engine = request.getfuncargvalue(fixture) mylist = [3, 2, 1] fn = pyccc.PythonCall(sorted, mylist) - job = engine.launch(image='python:2.7-slim', command=fn, interpreter=PYVERSION) + job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) job.wait() assert job.result == [1,2,3] From 8181c670c1b9dc1f3227a39710062192be7953ac Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 16 May 2017 12:17:46 -0700 Subject: [PATCH 3/3] Update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5d4e11b..84d6316 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ A high-level python interface for running computational jobs using a variety of ## Installation Normal installation: +`pyccc` works with Python 2.6 and 3.5+. You'll probably want a local version of [docker](https://www.docker.com/get-docker) running. + ```shell pip install pyccc ```