Merge pull request #926 from antonf/004_filters

[CF-257] Implement object selection engine

cloudferrylib/config.py

@@ -11,11 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import collections
 import contextlib
 import logging
 
-from cloudferrylib.os import clients, consts
+import marshmallow
+from marshmallow import fields
+
+from cloudferrylib.os import clients
 from cloudferrylib.utils import bases
+from cloudferrylib.utils import query
 from cloudferrylib.utils import remote
 from cloudferrylib.utils import utils
 
@@ -28,78 +33,147 @@
 ]
 
 
-class SshSettings(bases.Hashable, bases.Representable):
-    def __init__(self, username, sudo_password=None, gateway=None,
-                 connection_attempts=1, cipher=None, key_file=None):
-        self.username = username
-        self.sudo_password = sudo_password
-        self.gateway = gateway
-        self.connection_attempts = connection_attempts
-        self.cipher = cipher
-        self.key_file = key_file
-
-
-class Configuration(bases.Hashable, bases.Representable):
-    def __init__(self, clouds=None):
-        self.clouds = {}
-        for name, cloud in (clouds or {}).items():
-            credential = Credential(**cloud['credential'])
-            scope = Scope(**cloud['scope'])
-            ssh_settings = SshSettings(**cloud['ssh'])
-            self.clouds[name] = OpenstackCloud(name, credential, scope,
-                                               ssh_settings,
-                                               cloud.get('discover'))
-
-    def get_cloud(self, name):
-        return self.clouds[name]
-
-
-class Scope(bases.Hashable, bases.Representable):
-    def __init__(self, project_id=None, project_name=None, domain_id=None):
-        self.project_name = project_name
-        self.project_id = project_id
-        self.domain_id = domain_id
-
-
-class Credential(bases.Hashable, bases.Representable):
-    def __init__(self, auth_url, username, password,
-                 region_name=None, domain_id=None,
-                 https_insecure=False, https_cacert=None,
-                 endpoint_type=consts.EndpointType.ADMIN):
-        self.auth_url = auth_url
-        self.username = username
-        self.password = password
-        self.region_name = region_name
-        self.domain_id = domain_id
-        self.https_insecure = https_insecure
-        self.https_cacert = https_cacert
-        self.endpoint_type = endpoint_type
-
-
-class OpenstackCloud(object):
-    def __init__(self, name, credential, scope, ssh_settings, discover=None):
-        if discover is None:
-            discover = MODEL_LIST
-        self.name = name
-        self.credential = credential
-        self.scope = scope
-        self.ssh_settings = ssh_settings
-        self.discover = discover
+class DictField(fields.Field):
+    def __init__(self, key_field, nested_field, **kwargs):
+        super(DictField, self).__init__(**kwargs)
+        self.key_field = key_field
+        self.nested_field = nested_field
+
+    def _deserialize(self, value, attr, data):
+        if not isinstance(value, dict):
+            self.fail('type')
+
+        ret = {}
+        for key, val in value.items():
+            k = self.key_field.deserialize(key)
+            v = self.nested_field.deserialize(val)
+            ret[k] = v
+        return ret
+
+
+class FirstFit(fields.Field):
+    def __init__(self, *args, **kwargs):
+        many = kwargs.pop('many', False)
+        super(FirstFit, self).__init__(**kwargs)
+        self.many = many
+        self.variants = args
+
+    def _deserialize(self, value, attr, data):
+        if self.many:
+            return [self._do_deserialize(v) for v in value]
+        else:
+            return self._do_deserialize(value)
+
+    def _do_deserialize(self, value):
+        errors = []
+        for field in self.variants:
+            try:
+                return field.deserialize(value)
+            except marshmallow.ValidationError as ex:
+                errors.append(ex)
+        raise marshmallow.ValidationError([e.messages for e in errors])
+
+
+class OneOrMore(fields.Field):
+    def __init__(self, base_type, **kwargs):
+        super(OneOrMore, self).__init__(**kwargs)
+        self.base_type = base_type
+
+    def _deserialize(self, value, attr, data):
+        # pylint: disable=protected-access
+        if isinstance(value, collections.Sequence) and \
+                not isinstance(value, basestring):
+            return [self.base_type._deserialize(v, attr, data)
+                    for v in value]
+        else:
+            return [self.base_type._deserialize(value, attr, data)]
+
+
+class SshSettings(bases.Hashable, bases.Representable,
+                  bases.ConstructableFromDict):
+    class Schema(marshmallow.Schema):
+        username = fields.String()
+        sudo_password = fields.String(missing=None)
+        gateway = fields.String(missing=None)
+        connection_attempts = fields.Integer(missing=1)
+        cipher = fields.String(missing=None)
+        key_file = fields.String(missing=None)
+
+        @marshmallow.post_load
+        def to_scope(self, data):
+            return Scope(data)
+
+
+class Scope(bases.Hashable, bases.Representable, bases.ConstructableFromDict):
+    class Schema(marshmallow.Schema):
+        project_name = fields.String(missing=None)
+        project_id = fields.String(missing=None)
+        domain_id = fields.String(missing=None)
+
+        @marshmallow.post_load
+        def to_scope(self, data):
+            return Scope(data)
+
+        @marshmallow.validates_schema(skip_on_field_errors=True)
+        def check_migration_have_correct_source_and_dict(self, data):
+            if all(data[k] is None for k in self.declared_fields.keys()):
+                raise marshmallow.ValidationError(
+                    'At least one of {keys} shouldn\'t be None'.format(
+                        keys=self.declared_fields.keys()))
+
+
+class Credential(bases.Hashable, bases.Representable,
+                 bases.ConstructableFromDict):
+    class Schema(marshmallow.Schema):
+        auth_url = fields.Url()
+        username = fields.String()
+        password = fields.String()
+        region_name = fields.String(missing=None)
+        domain_id = fields.String(missing=None)
+        https_insecure = fields.Boolean(missing=False)
+        https_cacert = fields.String(missing=None)
+        endpoint_type = fields.String(missing='admin')
+
+        @marshmallow.post_load
+        def to_credential(self, data):
+            return Credential(data)
+
+
+class OpenstackCloud(bases.Hashable, bases.Representable,
+                     bases.ConstructableFromDict):
+    class Schema(marshmallow.Schema):
+        credential = fields.Nested(Credential.Schema)
+        scope = fields.Nested(Scope.Schema)
+        ssh_settings = fields.Nested(SshSettings.Schema, load_from='ssh')
+        discover = OneOrMore(fields.String(), default=MODEL_LIST)
+
+        @marshmallow.post_load
+        def to_cloud(self, data):
+            return OpenstackCloud(data)
+
+    def __init__(self, data):
+        super(OpenstackCloud, self).__init__(data)
+        self.name = None
 
     def image_client(self, scope=None):
+        # pylint: disable=no-member
         return clients.image_client(self.credential, scope or self.scope)
 
     def identity_client(self, scope=None):
+        # pylint: disable=no-member
         return clients.identity_client(self.credential, scope or self.scope)
 
     def volume_client(self, scope=None):
+        # pylint: disable=no-member
         return clients.volume_client(self.credential, scope or self.scope)
 
     def compute_client(self, scope=None):
+        # pylint: disable=no-member
         return clients.compute_client(self.credential, scope or self.scope)
 
     @contextlib.contextmanager
     def remote_executor(self, hostname, key_file=None, ignore_errors=False):
+        # pylint: disable=no-member
         key_files = []
         settings = self.ssh_settings
         if settings.key_file is not None:
@@ -119,3 +193,69 @@ def remote_executor(self, hostname, key_file=None, ignore_errors=False):
                 ignore_errors=ignore_errors)
         finally:
             remote.RemoteExecutor.close_connection(hostname)
+
+
+class Migration(bases.Hashable, bases.Representable):
+    class Schema(marshmallow.Schema):
+        source = fields.String(required=True)
+        destination = fields.String(required=True)
+        objects = DictField(
+            fields.String(),
+            FirstFit(
+                fields.String(),
+                DictField(
+                    fields.String(),
+                    OneOrMore(fields.Raw())),
+                many=True),
+            required=True)
+
+        @marshmallow.post_load
+        def to_migration(self, data):
+            return Migration(**data)
+
+    def __init__(self, source, destination, objects):
+        self.source = source
+        self.destination = destination
+        self.query = query.Query(objects)
+
+
+class Configuration(bases.Hashable, bases.Representable,
+                    bases.ConstructableFromDict):
+    class Schema(marshmallow.Schema):
+        clouds = DictField(
+            fields.String(allow_none=False),
+            fields.Nested(OpenstackCloud.Schema, default=dict))
+        migrations = DictField(
+            fields.String(allow_none=False),
+            fields.Nested(Migration.Schema, default=dict), debug=True)
+
+        @marshmallow.validates_schema(skip_on_field_errors=True)
+        def check_migration_have_correct_source_and_dict(self, data):
+            clouds = data['clouds']
+            migrations = data['migrations']
+            for migration_name, migration in migrations.items():
+                if migration.source not in clouds:
+                    raise marshmallow.ValidationError(
+                        'Migration "{0}" source "{1}" should be defined '
+                        'in clouds'.format(migration_name, migration.source))
+                if migration.destination not in clouds:
+                    raise marshmallow.ValidationError(
+                        'Migration "{0}" destination "{1}" should be defined '
+                        'in clouds'.format(migration_name,
+                                           migration.destination))
+
+        @marshmallow.post_load
+        def to_configuration(self, data):
+            for name, cloud in data['clouds'].items():
+                cloud.name = name
+            return Configuration(data)
+
+
+def load(data):
+    """
+    Loads and validates configuration
+    :param data: dictionary file loaded from discovery YAML
+    :return: Configuration instance
+    """
+    schema = Configuration.Schema(strict=True)
+    return schema.load(data).data

cloudferrylib/os/clients.py

@@ -84,8 +84,8 @@ def _get_authenticated_v2_client(credential, scope):
                                 region_name=credential.region_name,
                                 domain_id=credential.domain_id,
                                 endpoint_type=credential.endpoint_type,
-                                https_insecure=credential.https_insecure,
-                                https_cacert=credential.https_cacert,
+                                insecure=credential.https_insecure,
+                                cacert=credential.https_cacert,
                                 project_domain_id=scope.domain_id,
                                 project_name=scope.project_name,
                                 project_id=scope.project_id,

cloudferrylib/os/consts.py

@@ -34,12 +34,6 @@ def items(cls):
         return [(f, getattr(cls, f)) for f in cls.names()]
 
 
-class EndpointType(EnumType):
-    INTERNAL = "internal"
-    ADMIN = "admin"
-    PUBLIC = "public"
-
-
 class ServiceType(EnumType):
     IDENTITY = 'identity'
     COMPUTE = 'compute'

cloudferrylib/os/discovery/cinder.py

@@ -28,6 +28,7 @@ class Schema(model.Schema):
         device = fields.String(required=True)
 
 
+@model.type_alias('volumes')
 class Volume(model.Model):
     class Schema(model.Schema):
         object_id = model.PrimaryKey('id')

cloudferrylib/os/discovery/glance.py

@@ -31,22 +31,23 @@ class Schema(model.Schema):
 
     @classmethod
     def load_from_cloud(cls, cloud, data, overrides=None):
-        return cls.get(cloud, data.image_id, data.member_id)
+        return cls._get(cloud, data.image_id, data.member_id)
 
     @classmethod
     def load_missing(cls, cloud, object_id):
         image_id, member_id = object_id.id.split(':')
-        return cls.get(cls, image_id, member_id)
+        return cls._get(cls, image_id, member_id)
 
     @classmethod
-    def get(cls, cloud, image_id, member_id):
+    def _get(cls, cloud, image_id, member_id):
         return super(ImageMember, cls).load_from_cloud(cloud, {
             'object_id': '{0}:{1}'.format(image_id, member_id),
             'image': image_id,
             'member': member_id,
         })
 
 
+@model.type_alias('images')
 class Image(model.Model):
     class Schema(model.Schema):
         object_id = model.PrimaryKey('id')

cloudferrylib/os/discovery/keystone.py

@@ -21,6 +21,7 @@
 LOG = logging.getLogger(__name__)
 
 
+@model.type_alias('tenants')
 class Tenant(model.Model):
     class Schema(model.Schema):
         object_id = model.PrimaryKey('id')