Merge branch 'release-0.14.2'

plumpy/lang.py

@@ -4,7 +4,13 @@
 
 from __future__ import absolute_import
 import functools
-import inspect
+from inspect import stack, currentframe
+from six import PY2
+
+if PY2:
+    from inspect import getargspec as get_arg_spec
+else:
+    from inspect import getfullargspec as get_arg_spec
 
 
 def protected(check=False):
@@ -13,18 +19,18 @@ def wrap(func):
         if isinstance(func, property):
             raise RuntimeError("Protected must go after @property decorator")
 
-        args = inspect.getargspec(func)[0]
+        args = get_arg_spec(func)[0]  # pylint: disable=deprecated-method
         if len(args) == 0:
             raise RuntimeError("Can only use the protected decorator on member functions")
 
         # We can only perform checks if the interpreter is capable of giving
         # us the stack i.e. currentframe() produces a valid object
-        if check and inspect.currentframe() is not None:
+        if check and currentframe() is not None:
 
             @functools.wraps(func)
             def wrapped_fn(self, *args, **kwargs):
                 try:
-                    calling_class = inspect.stack()[1][0].f_locals['self']
+                    calling_class = stack()[1][0].f_locals['self']
                     assert self is calling_class
                 except (KeyError, AssertionError):
                     raise RuntimeError("Cannot access protected function {} from outside"
@@ -45,7 +51,7 @@ def wrap(func):
         if isinstance(func, property):
             raise RuntimeError("Override must go after @property decorator")
 
-        args = inspect.getargspec(func)[0]
+        args = get_arg_spec(func)[0]  # pylint: disable=deprecated-method
         if len(args) == 0:
             raise RuntimeError("Can only use the override decorator on member functions")
 

plumpy/ports.py

@@ -2,13 +2,17 @@
 """Module for process ports"""
 from __future__ import absolute_import
 import abc
-import collections
 import copy
 import json
 import logging
 import six
 
-from plumpy.utils import is_mutable_property
+from plumpy.utils import is_mutable_property, type_check
+
+if six.PY2:
+    import collections
+else:
+    import collections.abc as collections
 
 _LOGGER = logging.getLogger(__name__)
 UNSPECIFIED = ()
@@ -356,6 +360,7 @@ class PortNamespace(collections.MutableMapping, Port):
     A container for Ports. Effectively it maintains a dictionary whose members are
     either a Port or yet another PortNamespace. This allows for the nesting of ports
     """
+
     NAMESPACE_SEPARATOR = '.'
 
     def __init__(self,
@@ -365,12 +370,28 @@ def __init__(self,
                  validator=None,
                  valid_type=None,
                  default=UNSPECIFIED,
-                 dynamic=False):
+                 dynamic=False,
+                 populate_defaults=True):
+        """Construct a port namespace.
+
+        :param name: the name of the namespace
+        :param help: the help string
+        :param required: boolean, if `True` the validation will fail if no value is specified for this namespace
+        :param validator: an optional validator for the namespace
+        :param valid_type: optional tuple of valid types in the case of a dynamic namespace
+        :param default: default value for the port
+        :param dynamic: boolean, if `True`, the namespace will accept values even when no explicit port is defined
+        :param populate_defaults: boolean, when set to `False`, the populating of defaults for this namespace is skipped
+            entirely, including all nested namespaces, if no explicit value is passed for this port in the parent
+            namespace. As soon as a value is specified in the parent namespace for this port, even if it is empty, this
+            property is ignored and the population of defaults is always performed.
+        """
         super(PortNamespace, self).__init__(
             name=name, help=help, required=required, validator=validator, valid_type=valid_type)
         self._ports = {}
         self._default = default
         self._dynamic = dynamic
+        self._populate_defaults = populate_defaults
 
     def __str__(self):
         return json.dumps(self.get_description(), sort_keys=True, indent=4)
@@ -434,6 +455,14 @@ def valid_type(self, valid_type):
 
         super(PortNamespace, self.__class__).valid_type.fset(self, valid_type)
 
+    @property
+    def populate_defaults(self):
+        return self._populate_defaults
+
+    @populate_defaults.setter
+    def populate_defaults(self, populate_defaults):
+        self._populate_defaults = populate_defaults
+
     def get_description(self):
         """
         Return a dictionary with a description of the ports this namespace contains
@@ -521,7 +550,7 @@ def create_port_namespace(self, name, **kwargs):
         else:
             return self[port_name]
 
-    def absorb(self, port_namespace, exclude=(), include=None, namespace_options={}):
+    def absorb(self, port_namespace, exclude=None, include=None, namespace_options=None):
         """Absorb another PortNamespace instance into oneself, including all its mutable properties and ports.
 
         Mutable properties of self will be overwritten with those of the port namespace that is to be absorbed.
@@ -539,6 +568,16 @@ def absorb(self, port_namespace, exclude=(), include=None, namespace_options={})
         if not isinstance(port_namespace, PortNamespace):
             raise ValueError('port_namespace has to be an instance of PortNamespace')
 
+        if exclude is not None and include is not None:
+            raise ValueError('exclude and include are mutually exclusive')
+        elif exclude is not None:
+            type_check(exclude, (list, tuple))
+        elif include is not None:
+            type_check(include, (list, tuple))
+
+        if namespace_options is None:
+            namespace_options = {}
+
         # Overload mutable attributes of PortNamespace unless overridden by value in namespace_options
         for attr in dir(port_namespace):
             if is_mutable_property(PortNamespace, attr):
@@ -553,22 +592,34 @@ def absorb(self, port_namespace, exclude=(), include=None, namespace_options={})
 
         absorbed_ports = []
 
-        for port_name, port in self._filter_ports(list(port_namespace.items()), exclude=exclude, include=include):
+        for port_name, port in port_namespace.items():
+
+            # If the current port name occurs in the exclude list, simply skip it entirely, there is no need to consider
+            # any of the nested ports it might have, even if it is a port namespace
+            if exclude and port_name in exclude:
+                continue
 
             if isinstance(port, PortNamespace):
 
-                # Strip the namespace's name from the exclude and include rules
-                stripped_exclude = self.strip_namespace(port_name, self.NAMESPACE_SEPARATOR, exclude)
-                stripped_include = self.strip_namespace(port_name, self.NAMESPACE_SEPARATOR, include)
+                # If the name does not appear at the start of any of the include rules we continue:
+                if include and not any([rule.startswith(port_name) for rule in include]):
+                    continue
+
+                # Determine the sub exclude and include rules for this specific namespace
+                sub_exclude = self.strip_namespace(port_name, self.NAMESPACE_SEPARATOR, exclude)
+                sub_include = self.strip_namespace(port_name, self.NAMESPACE_SEPARATOR, include)
 
-                # Create a new namespace at `port_name` and absorb its ports into it, with the stripped exclude/include.
-                # Note that we copy the port namespace itself such that we keep all its mutable properties, but then
-                # reset its ports, because not all ports need to be included depending on the exclude/include rules.
-                # Instead the copying of the ports is taken care of by the recursive `absorb` call.
+                # Create a new namespace at `port_name` and copy the original port namespace itself such that we keep
+                # all its mutable properties, but reset its ports, since those will be taken care of by the recursive
+                # absorb call that will properly consider the include and exclude rules
                 self[port_name] = copy.copy(port)
                 self[port_name]._ports = {}
-                self[port_name].absorb(port, stripped_exclude, stripped_include)
+                self[port_name].absorb(port, sub_exclude, sub_include)
             else:
+                # If include rules are specified but the port name does not appear, simply skip it
+                if include and port_name not in include:
+                    continue
+
                 self[port_name] = copy.deepcopy(port)
 
             absorbed_ports.append(port_name)
@@ -650,7 +701,13 @@ def pre_process(self, port_values):
 
         for name, port in self.items():
 
+            # If the port was not specified in the inputs values and the port is a namespace with the property
+            # `populate_defaults=False`, we skip the pre-processing and do not populate defaults.
+            if name not in port_values and isinstance(port, PortNamespace) and not port.populate_defaults:
+                continue
+
             if name not in port_values:
+
                 if port.has_default():
                     port_value = port.default
                 elif isinstance(port, PortNamespace):
@@ -712,15 +769,15 @@ def validate_dynamic_ports(self, port_values, breadcrumbs=()):
 
     @staticmethod
     def strip_namespace(namespace, separator, rules=None):
-        """Strip the namespace from the given tuple of exclude/include rules.
+        """Filter given exclude/include rules staring with namespace and strip the first level.
 
         For example if the namespace is `base` and the rules are::
 
-            ('base.a', 'relax.base.c', 'd')
+            ('base.a', 'base.sub.b','relax.base.c', 'd')
 
         the function will return::
 
-            ('a', 'relax.base.c', 'd')
+            ('a', 'sub.c')
 
         If the rules are `None`, that is what is returned as well.
 
@@ -729,7 +786,7 @@ def strip_namespace(namespace, separator, rules=None):
         :param rules: the list or tuple of exclude or include rules to strip
         :return: `None` if `rules=None` or the list of stripped rules
         """
-        if not rules:
+        if rules is None:
             return rules
 
         stripped = []
@@ -739,40 +796,12 @@ def strip_namespace(namespace, separator, rules=None):
         for rule in rules:
             if rule.startswith(prefix):
                 stripped.append(rule[len(prefix):])
-            else:
-                stripped.append(rule)
 
         return stripped
 
-    @staticmethod
-    def _filter_ports(items, exclude, include):
-        """
-        Convenience generator that will filter the items based on its keys and the exclude/include tuples.
-        The exclude and include tuples are mutually exclusive and only one should be defined. A key in items
-        will only be yielded if it appears in include or does not appear in exclude, otherwise it will be skipped
-
-        :param items: a mapping of port names and Ports
-        :param exclude: a tuple of port names that are to be skipped
-        :param include: a tuple of port names that are the only ones to be yielded
-        :returns: tuple of port name and Port
-        """
-        if exclude and include is not None:
-            raise ValueError('exclude and include are mutually exclusive')
-
-        for name, port in items:
-            if include is not None:
-                if name not in include:
-                    continue
-            else:
-                if name in exclude:
-                    continue
-
-            yield name, port
-
 
 def breadcrumbs_to_port(breadcrumbs):
-    """
-    Convert breadcrumbs to a string representing the port
+    """Convert breadcrumbs to a string representing the port
 
     :param breadcrumbs: a tuple of the path to the port
     :type breadcrumbs: typing.Tuple[str]

plumpy/process_spec.py

@@ -171,7 +171,7 @@ def has_output(self, name):
         """
         return name in self.outputs
 
-    def expose_inputs(self, process_class, namespace=None, exclude=(), include=None, namespace_options={}):
+    def expose_inputs(self, process_class, namespace=None, exclude=None, include=None, namespace_options={}):
         """
         This method allows one to automatically add the inputs from another Process to this ProcessSpec.
         The optional namespace argument can be used to group the exposed inputs in a separated PortNamespace.
@@ -195,7 +195,7 @@ def expose_inputs(self, process_class, namespace=None, exclude=(), include=None,
             namespace_options=namespace_options,
         )
 
-    def expose_outputs(self, process_class, namespace=None, exclude=(), include=None, namespace_options={}):
+    def expose_outputs(self, process_class, namespace=None, exclude=None, include=None, namespace_options={}):
         """
         This method allows one to automatically add the ouputs from another Process to this ProcessSpec.
         The optional namespace argument can be used to group the exposed outputs in a separated PortNamespace.

plumpy/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.14.1"
+__version__ = "0.14.2"
 
 __all__ = ['__version__']

plumpy/workchains.py

@@ -3,10 +3,14 @@
 from __future__ import absolute_import
 import abc
 import collections
-import inspect
 import re
 import six
 
+if six.PY2:
+    from inspect import getargspec as get_arg_spec
+else:
+    from inspect import getfullargspec as get_arg_spec
+
 from . import lang
 from . import mixins
 from . import persistence
@@ -235,7 +239,7 @@ class _FunctionCall(_Instruction):
 
     def __init__(self, func):
         try:
-            args = inspect.getargspec(func)[0]
+            args = get_arg_spec(func)[0]
         except TypeError:
             raise TypeError("func is not a function, got {}".format(type(func)))
         if len(args) != 1: