From c43537b3756bc48f9606d8479c1b873dde44bc5a Mon Sep 17 00:00:00 2001 From: Anton Frolov Date: Wed, 16 Mar 2016 17:35:35 -0700 Subject: [PATCH 1/3] Add YAML configuration validation. --- cloudferrylib/config.py | 213 +++++++++++++++++++++++++---------- cloudferrylib/utils/bases.py | 13 +++ fabfile.py | 6 +- 3 files changed, 171 insertions(+), 61 deletions(-) diff --git a/cloudferrylib/config.py b/cloudferrylib/config.py index c5ca8262..0249df94 100644 --- a/cloudferrylib/config.py +++ b/cloudferrylib/config.py @@ -11,10 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import collections import contextlib import logging -from cloudferrylib.os import clients, consts +import marshmallow +from marshmallow import fields + +from cloudferrylib.os import clients from cloudferrylib.utils import bases from cloudferrylib.utils import remote from cloudferrylib.utils import utils @@ -28,78 +32,147 @@ ] -class SshSettings(bases.Hashable, bases.Representable): - def __init__(self, username, sudo_password=None, gateway=None, - connection_attempts=1, cipher=None, key_file=None): - self.username = username - self.sudo_password = sudo_password - self.gateway = gateway - self.connection_attempts = connection_attempts - self.cipher = cipher - self.key_file = key_file - - -class Configuration(bases.Hashable, bases.Representable): - def __init__(self, clouds=None): - self.clouds = {} - for name, cloud in (clouds or {}).items(): - credential = Credential(**cloud['credential']) - scope = Scope(**cloud['scope']) - ssh_settings = SshSettings(**cloud['ssh']) - self.clouds[name] = OpenstackCloud(name, credential, scope, - ssh_settings, - cloud.get('discover')) - - def get_cloud(self, name): - return self.clouds[name] - - -class Scope(bases.Hashable, bases.Representable): - def __init__(self, project_id=None, project_name=None, domain_id=None): - self.project_name = project_name - self.project_id = project_id - self.domain_id = domain_id - - -class Credential(bases.Hashable, bases.Representable): - def __init__(self, auth_url, username, password, - region_name=None, domain_id=None, - https_insecure=False, https_cacert=None, - endpoint_type=consts.EndpointType.ADMIN): - self.auth_url = auth_url - self.username = username - self.password = password - self.region_name = region_name - self.domain_id = domain_id - self.https_insecure = https_insecure - self.https_cacert = https_cacert - self.endpoint_type = endpoint_type - - -class OpenstackCloud(object): - def __init__(self, name, credential, scope, ssh_settings, discover=None): - if discover is None: - discover = MODEL_LIST - self.name = name - self.credential = credential - self.scope = scope - self.ssh_settings = ssh_settings - self.discover = discover +class DictField(fields.Field): + def __init__(self, key_field, nested_field, **kwargs): + super(DictField, self).__init__(**kwargs) + self.key_field = key_field + self.nested_field = nested_field + + def _deserialize(self, value, attr, data): + if not isinstance(value, dict): + self.fail('type') + + ret = {} + for key, val in value.items(): + k = self.key_field.deserialize(key) + v = self.nested_field.deserialize(val) + ret[k] = v + return ret + + +class FirstFit(fields.Field): + def __init__(self, *args, **kwargs): + many = kwargs.pop('many', False) + super(FirstFit, self).__init__(**kwargs) + self.many = many + self.variants = args + + def _deserialize(self, value, attr, data): + if self.many: + return [self._do_deserialize(v) for v in value] + else: + return self._do_deserialize(value) + + def _do_deserialize(self, value): + errors = [] + for field in self.variants: + try: + return field.deserialize(value) + except marshmallow.ValidationError as e: + errors.append(e) + raise marshmallow.ValidationError([e.messages for e in errors]) + + +class OneOrMore(fields.Field): + def __init__(self, base_type, **kwargs): + super(OneOrMore, self).__init__(**kwargs) + self.base_type = base_type + + def _deserialize(self, value, attr, data): + # pylint: disable=protected-access + if isinstance(value, collections.Sequence) and \ + not isinstance(value, basestring): + return [self.base_type._deserialize(v, attr, data) + for v in value] + else: + return [self.base_type._deserialize(value, attr, data)] + + +class SshSettings(bases.Hashable, bases.Representable, + bases.ConstructableFromDict): + class Schema(marshmallow.Schema): + username = fields.String() + sudo_password = fields.String(missing=None) + gateway = fields.String(missing=None) + connection_attempts = fields.Integer(missing=1) + cipher = fields.String(missing=None) + key_file = fields.String(missing=None) + + @marshmallow.post_load + def to_scope(self, data): + return Scope(data) + + +class Scope(bases.Hashable, bases.Representable, bases.ConstructableFromDict): + class Schema(marshmallow.Schema): + project_name = fields.String(missing=None) + project_id = fields.String(missing=None) + domain_id = fields.String(missing=None) + + @marshmallow.post_load + def to_scope(self, data): + return Scope(data) + + @marshmallow.validates_schema(skip_on_field_errors=True) + def check_migration_have_correct_source_and_dict(self, data): + if all(data[k] is None for k in self.declared_fields.keys()): + raise marshmallow.ValidationError( + 'At least one of {keys} shouldn\'t be None'.format( + keys=self.declared_fields.keys())) + + +class Credential(bases.Hashable, bases.Representable, + bases.ConstructableFromDict): + class Schema(marshmallow.Schema): + auth_url = fields.Url() + username = fields.String() + password = fields.String() + region_name = fields.String(missing=None) + domain_id = fields.String(missing=None) + https_insecure = fields.Boolean(missing=False) + https_cacert = fields.String(missing=None) + endpoint_type = fields.String(missing='admin') + + @marshmallow.post_load + def to_credential(self, data): + return Credential(data) + + +class OpenstackCloud(bases.Hashable, bases.Representable, + bases.ConstructableFromDict): + class Schema(marshmallow.Schema): + credential = fields.Nested(Credential.Schema) + scope = fields.Nested(Scope.Schema) + ssh_settings = fields.Nested(SshSettings.Schema, load_from='ssh') + discover = OneOrMore(fields.String(), default=MODEL_LIST) + + @marshmallow.post_load + def to_cloud(self, data): + return OpenstackCloud(data) + + def __init__(self, data): + super(OpenstackCloud, self).__init__(data) + self.name = None def image_client(self, scope=None): + # pylint: disable=no-member return clients.image_client(self.credential, scope or self.scope) def identity_client(self, scope=None): + # pylint: disable=no-member return clients.identity_client(self.credential, scope or self.scope) def volume_client(self, scope=None): + # pylint: disable=no-member return clients.volume_client(self.credential, scope or self.scope) def compute_client(self, scope=None): + # pylint: disable=no-member return clients.compute_client(self.credential, scope or self.scope) @contextlib.contextmanager def remote_executor(self, hostname, key_file=None, ignore_errors=False): + # pylint: disable=no-member key_files = [] settings = self.ssh_settings if settings.key_file is not None: @@ -119,3 +192,27 @@ def remote_executor(self, hostname, key_file=None, ignore_errors=False): ignore_errors=ignore_errors) finally: remote.RemoteExecutor.close_connection(hostname) + + +class Configuration(bases.Hashable, bases.Representable, + bases.ConstructableFromDict): + class Schema(marshmallow.Schema): + clouds = DictField( + fields.String(allow_none=False), + fields.Nested(OpenstackCloud.Schema, default=dict)) + + @marshmallow.post_load + def to_configuration(self, data): + for name, cloud in data['clouds'].items(): + cloud.name = name + return Configuration(data) + + +def load(data): + """ + Loads and validates configuration + :param data: dictionary file loaded from discovery YAML + :return: Configuration instance + """ + schema = Configuration.Schema(strict=True) + return schema.load(data).data diff --git a/cloudferrylib/utils/bases.py b/cloudferrylib/utils/bases.py index c359e0ec..e87b5c13 100644 --- a/cloudferrylib/utils/bases.py +++ b/cloudferrylib/utils/bases.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import collections import sys @@ -83,3 +84,15 @@ def __repr__(self): fields=' '.join('{0}:{1}'.format(f, repr(getattr(self, f))) for f in sorted_field_names(self) if getattr(self, f) is not None)) + + +class ConstructableFromDict(object): + """ + Mixin class with __init__ method that just assign values from dictionary + to object attributes with names identical to keys from dictionary. + """ + + def __init__(self, data): + assert isinstance(data, collections.Mapping) + for name, value in data.items(): + setattr(self, name, value) diff --git a/fabfile.py b/fabfile.py index d5d1390e..9299c034 100644 --- a/fabfile.py +++ b/fabfile.py @@ -200,14 +200,14 @@ def discover(config_path, debug=False): """ :config_name - name of config yaml-file, example 'config.yaml' """ - cfg = config.Configuration(**load_yaml_config(config_path, debug)) + cfg = config.load(load_yaml_config(config_path, debug)) stage.execute_stage('cloudferrylib.os.discovery.stages.DiscoverStage', cfg, force=True) @task def estimate_migration(config_path, source, tenant=None, debug=False): - cfg = config.Configuration(**load_yaml_config(config_path, debug)) + cfg = config.load(load_yaml_config(config_path, debug)) stage.execute_stage('cloudferrylib.os.discovery.stages.DiscoverStage', cfg) procedures.estimate_copy(source, tenant) procedures.show_largest_servers(10, source, tenant) @@ -216,7 +216,7 @@ def estimate_migration(config_path, source, tenant=None, debug=False): @task def show_unused_resources(config_path, cloud, count=100, tenant=None, debug=False): - cfg = config.Configuration(**load_yaml_config(config_path, debug)) + cfg = config.load(load_yaml_config(config_path, debug)) stage.execute_stage('cloudferrylib.os.discovery.stages.DiscoverStage', cfg) procedures.show_largest_unused_resources(int(count), cloud, tenant) From c5cfdf94d674fa64b11dda41ff854e844d96c9fb Mon Sep 17 00:00:00 2001 From: Anton Frolov Date: Wed, 16 Mar 2016 17:45:57 -0700 Subject: [PATCH 2/3] [CF-257] Implement object selection engine Implementation is based on JMESPath library. --- cloudferrylib/config.py | 43 ++++++++ cloudferrylib/os/clients.py | 4 +- cloudferrylib/os/consts.py | 6 -- cloudferrylib/os/discovery/cinder.py | 1 + cloudferrylib/os/discovery/glance.py | 7 +- cloudferrylib/os/discovery/keystone.py | 1 + cloudferrylib/os/discovery/model.py | 38 +++++++ cloudferrylib/os/discovery/nova.py | 4 +- cloudferrylib/utils/query.py | 131 +++++++++++++++++++++++ discover.yaml | 31 +++++- requirements.txt | 1 + tests/cloudferrylib/utils/test_query.py | 133 ++++++++++++++++++++++++ 12 files changed, 385 insertions(+), 15 deletions(-) create mode 100644 cloudferrylib/utils/query.py create mode 100644 tests/cloudferrylib/utils/test_query.py diff --git a/cloudferrylib/config.py b/cloudferrylib/config.py index 0249df94..693cd676 100644 --- a/cloudferrylib/config.py +++ b/cloudferrylib/config.py @@ -20,6 +20,7 @@ from cloudferrylib.os import clients from cloudferrylib.utils import bases +from cloudferrylib.utils import query from cloudferrylib.utils import remote from cloudferrylib.utils import utils @@ -194,12 +195,54 @@ def remote_executor(self, hostname, key_file=None, ignore_errors=False): remote.RemoteExecutor.close_connection(hostname) +class Migration(bases.Hashable, bases.Representable): + class Schema(marshmallow.Schema): + source = fields.String(required=True) + destination = fields.String(required=True) + objects = DictField( + fields.String(), + FirstFit( + fields.String(), + DictField( + fields.String(), + OneOrMore(fields.Raw())), + many=True), + required=True) + + @marshmallow.post_load + def to_migration(self, data): + return Migration(**data) + + def __init__(self, source, destination, objects): + self.source = source + self.destination = destination + self.query = query.Query(objects) + + class Configuration(bases.Hashable, bases.Representable, bases.ConstructableFromDict): class Schema(marshmallow.Schema): clouds = DictField( fields.String(allow_none=False), fields.Nested(OpenstackCloud.Schema, default=dict)) + migrations = DictField( + fields.String(allow_none=False), + fields.Nested(Migration.Schema, default=dict), debug=True) + + @marshmallow.validates_schema(skip_on_field_errors=True) + def check_migration_have_correct_source_and_dict(self, data): + clouds = data['clouds'] + migrations = data['migrations'] + for migration_name, migration in migrations.items(): + if migration.source not in clouds: + raise marshmallow.ValidationError( + 'Migration "{0}" source "{1}" should be defined ' + 'in clouds'.format(migration_name, migration.source)) + if migration.destination not in clouds: + raise marshmallow.ValidationError( + 'Migration "{0}" destination "{1}" should be defined ' + 'in clouds'.format(migration_name, + migration.destination)) @marshmallow.post_load def to_configuration(self, data): diff --git a/cloudferrylib/os/clients.py b/cloudferrylib/os/clients.py index 36acdbeb..59ee2cc0 100644 --- a/cloudferrylib/os/clients.py +++ b/cloudferrylib/os/clients.py @@ -84,8 +84,8 @@ def _get_authenticated_v2_client(credential, scope): region_name=credential.region_name, domain_id=credential.domain_id, endpoint_type=credential.endpoint_type, - https_insecure=credential.https_insecure, - https_cacert=credential.https_cacert, + insecure=credential.https_insecure, + cacert=credential.https_cacert, project_domain_id=scope.domain_id, project_name=scope.project_name, project_id=scope.project_id, diff --git a/cloudferrylib/os/consts.py b/cloudferrylib/os/consts.py index fe575138..d89344f0 100644 --- a/cloudferrylib/os/consts.py +++ b/cloudferrylib/os/consts.py @@ -34,12 +34,6 @@ def items(cls): return [(f, getattr(cls, f)) for f in cls.names()] -class EndpointType(EnumType): - INTERNAL = "internal" - ADMIN = "admin" - PUBLIC = "public" - - class ServiceType(EnumType): IDENTITY = 'identity' COMPUTE = 'compute' diff --git a/cloudferrylib/os/discovery/cinder.py b/cloudferrylib/os/discovery/cinder.py index 95ab28c4..a2262b9b 100644 --- a/cloudferrylib/os/discovery/cinder.py +++ b/cloudferrylib/os/discovery/cinder.py @@ -28,6 +28,7 @@ class Schema(model.Schema): device = fields.String(required=True) +@model.type_alias('volumes') class Volume(model.Model): class Schema(model.Schema): object_id = model.PrimaryKey('id') diff --git a/cloudferrylib/os/discovery/glance.py b/cloudferrylib/os/discovery/glance.py index 86a35c21..29ba941a 100644 --- a/cloudferrylib/os/discovery/glance.py +++ b/cloudferrylib/os/discovery/glance.py @@ -31,15 +31,15 @@ class Schema(model.Schema): @classmethod def load_from_cloud(cls, cloud, data, overrides=None): - return cls.get(cloud, data.image_id, data.member_id) + return cls._get(cloud, data.image_id, data.member_id) @classmethod def load_missing(cls, cloud, object_id): image_id, member_id = object_id.id.split(':') - return cls.get(cls, image_id, member_id) + return cls._get(cls, image_id, member_id) @classmethod - def get(cls, cloud, image_id, member_id): + def _get(cls, cloud, image_id, member_id): return super(ImageMember, cls).load_from_cloud(cloud, { 'object_id': '{0}:{1}'.format(image_id, member_id), 'image': image_id, @@ -47,6 +47,7 @@ def get(cls, cloud, image_id, member_id): }) +@model.type_alias('images') class Image(model.Model): class Schema(model.Schema): object_id = model.PrimaryKey('id') diff --git a/cloudferrylib/os/discovery/keystone.py b/cloudferrylib/os/discovery/keystone.py index 999d99c8..d2d4e5c5 100644 --- a/cloudferrylib/os/discovery/keystone.py +++ b/cloudferrylib/os/discovery/keystone.py @@ -21,6 +21,7 @@ LOG = logging.getLogger(__name__) +@model.type_alias('tenants') class Tenant(model.Model): class Schema(model.Schema): object_id = model.PrimaryKey('id') diff --git a/cloudferrylib/os/discovery/model.py b/cloudferrylib/os/discovery/model.py index 3a043961..8be5adb6 100644 --- a/cloudferrylib/os/discovery/model.py +++ b/cloudferrylib/os/discovery/model.py @@ -132,6 +132,7 @@ class Schema(model.Schema): from cloudferrylib.utils import local_db LOG = logging.getLogger(__name__) +type_aliases = {} local_db.execute_once(""" CREATE TABLE IF NOT EXISTS objects ( uuid TEXT, @@ -527,6 +528,12 @@ def get_class(cls): """ return cls + def get(self, name, default=None): + """ + Returns object attribute by name. + """ + return getattr(self, name, default) + def __repr__(self): schema = self.get_schema() obj_fields = sorted(schema.declared_fields.keys()) @@ -665,6 +672,12 @@ def _retrieve_obj(self): with Session.current() as session: self._object = session.retrieve(self._model, self._object_id) + def get(self, name): + """ + Returns object attribute by name. + """ + return getattr(self, name, None) + def get_class(self): """ Return model class. @@ -896,5 +909,30 @@ def _delete_rows(self, cls, cloud, object_id): self.tx.execute(statement, **kwargs) +def type_alias(name): + """ + Decorator function that add alias for some model class + :param name: alias name + """ + + def wrapper(cls): + assert issubclass(cls, Model) + type_aliases[name] = cls + return cls + return wrapper + + +def get_model(type_name): + """ + Return model class instance using either alias or fully qualified name. + :param type_name: alias or fully qualified class name + :return: subclass of Model + """ + if type_name in type_aliases: + return type_aliases[type_name] + else: + return importutils.import_class(type_name) + + def _type_name(cls): return cls.__module__ + '.' + cls.__name__ diff --git a/cloudferrylib/os/discovery/nova.py b/cloudferrylib/os/discovery/nova.py index fb205c03..d082ff77 100644 --- a/cloudferrylib/os/discovery/nova.py +++ b/cloudferrylib/os/discovery/nova.py @@ -50,6 +50,7 @@ class Schema(model.Schema): size = fields.Integer(required=True) +@model.type_alias('vms') class Server(model.Model): class Schema(model.Schema): object_id = model.PrimaryKey('id') @@ -158,8 +159,7 @@ def _list_ephemeral(remote, server): if len(split) != 2: continue target, path = split - if target in volume_targets or not path.startswith('/') or \ - path.endswith('disk.config'): + if target in volume_targets or not path.startswith('/'): continue size_str = remote.sudo('stat -c %s {path}', path=path) if not size_str.succeeded: diff --git a/cloudferrylib/utils/query.py b/cloudferrylib/utils/query.py new file mode 100644 index 00000000..ebc97e3b --- /dev/null +++ b/cloudferrylib/utils/query.py @@ -0,0 +1,131 @@ +# Copyright 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import jmespath +import jmespath.exceptions + +from cloudferrylib.os.discovery import model + + +class DictSubQuery(object): + """ + Simplified query that use JMESPath queries as dictionary keys to get value + from tested object and then compare it to list of objects specified in + value. + + Example: + + objects: + images: + - tenant.name: demo + container_format: bare + + This query will select all images with owner named 'demo' and container + format 'bare' + + Adding !in front of query will negate match result. For example: + + objects: + vms: + - !tenant.name: rally_test + + This query will select all VMs in cloud except for rally_test tenant + """ + + def __init__(self, pattern): + assert isinstance(pattern, dict) + self.pattern = [self._compile_query(k, v) for k, v in pattern.items()] + + def search(self, values): + """ + Return subset of values that match query parameters. + :param values: list of objects that have get method + :return: list of objects that matched query + """ + return [v for v in values if self._matches(v)] + + def _matches(self, value): + return all(match(value) for match in self.pattern) + + @staticmethod + def _compile_query(key, expected): + assert isinstance(key, basestring) + + negative = False + if key.startswith('!'): + negative = True + key = key[1:] + try: + query = jmespath.compile(key) + + def match(value): + return negative ^ (query.search(value) in expected) + return match + except jmespath.exceptions.ParseError as ex: + raise AssertionError( + 'Failed to compile "{0}": {1}'.format(key, str(ex))) + + +class Query(object): + """ + Parsed and compiled query using which it is possible to filter instances of + model.Model class stored in database. + """ + + def __init__(self, query): + """ + Accept dict as specified in configuration, compile all the JMESPath + queries, and store it as internal immutable state. + :param query: query dictionary + """ + + assert isinstance(query, dict) + + self.queries = {} + for type_name, subqueries in query.items(): + cls = model.get_model(type_name) + for subquery in subqueries: + if isinstance(subquery, basestring): + subquery = jmespath.compile(subquery) + else: + subquery = DictSubQuery(subquery) + cls_queries = self.queries.setdefault(cls, []) + cls_queries.append(subquery) + + def search(self, session, cloud=None, cls=None): + """ + Search through list of objects from database of class that specified + in cls argument (if cls is none, then all classes are considered) that + are collected from cloud specified in cloud argument (if cloud is none, + then all clouds are considered) for objects matching this query. + + :param session: active model.Session instance + :param cloud: cloud name + :param cls: class object + :return: list of objects that match query + """ + result = set() + if cls is None: + for cls, queries in self.queries.items(): + objects = session.list(cls, cloud) + for query in queries: + result.update(query.search(objects)) + return result + else: + queries = self.queries.get(cls) + if queries is None: + return [] + objects = session.list(cls, cloud) + for query in queries: + result.update(query.search(objects)) + return result diff --git a/discover.yaml b/discover.yaml index 985332a4..76031306 100644 --- a/discover.yaml +++ b/discover.yaml @@ -1,9 +1,23 @@ clouds: - grizzly_ewr2: + grizzly: credential: auth_url: https://keystone.example.com/v2.0/ + username: admin + password: admin + region_name: grizzly + scope: + project_id: 00000000000000000000000000000000 + ssh: username: foobar - password: foobar + sudo_password: foobar + connection_attempts: 3 + + liberty: + credential: + auth_url: https://keystone.example.com/v2.0/ + username: admin + password: admin + region_name: liberty scope: project_id: 00000000000000000000000000000000 ssh: @@ -24,3 +38,16 @@ clouds: # legacy: configuration.grizzly-juno.ini:src # juno_dst: # legacy: configuration.ini:dst + +migrations: + grizzly_to_liberty: + source: grizzly + destination: liberty + objects: + vms: + - tenant.name: demo # Include VMs owned by tenant named "demo" into migration + images: + - tenant.name: demo # Include images owned by tenant named "demo" into migration + - is_public: True # Also include any public images (no matter owned by which tenant) into migration + volumes: + - tenant.name: demo # Include volumes owned by tenant named "demo" into migration diff --git a/requirements.txt b/requirements.txt index 8763c472..bbff50a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ pywbem redis sqlalchemy marshmallow==2.4.2 +jmespath==0.9.0 diff --git a/tests/cloudferrylib/utils/test_query.py b/tests/cloudferrylib/utils/test_query.py new file mode 100644 index 00000000..349aae44 --- /dev/null +++ b/tests/cloudferrylib/utils/test_query.py @@ -0,0 +1,133 @@ +# Copyright 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from cloudferrylib.os.discovery import model +from cloudferrylib.utils import query +from marshmallow import fields +from tests.cloudferrylib.utils import test_local_db +import mock + + +class TestMode(model.Model): + class Schema(model.Schema): + object_id = model.PrimaryKey('id') + field1 = fields.String() + field2 = fields.String() + +CLASS_FQN = TestMode.__module__ + '.' + TestMode.__name__ + + +class StageTestCase(test_local_db.DatabaseMockingTestCase): + def setUp(self): + super(StageTestCase, self).setUp() + + self.cloud = mock.MagicMock() + self.cloud.name = 'test_cloud' + + self.obj1 = TestMode.load_from_cloud(self.cloud, { + 'id': 'id1', + 'field1': 'a', + 'field2': 'a', + }) + self.obj2 = TestMode.load_from_cloud(self.cloud, { + 'id': 'id2', + 'field1': 'a', + 'field2': 'b', + }) + self.obj3 = TestMode.load_from_cloud(self.cloud, { + 'id': 'id3', + 'field1': 'b', + 'field2': 'a', + }) + self.obj4 = TestMode.load_from_cloud(self.cloud, { + 'id': 'id4', + 'field1': 'b', + 'field2': 'b', + }) + + with model.Session() as s: + s.store(self.obj1) + s.store(self.obj2) + s.store(self.obj3) + s.store(self.obj4) + + def test_simple_query1(self): + q = query.Query({ + CLASS_FQN: [ + { + 'field1': ['a'], + } + ] + }) + with model.Session() as session: + objs = sorted(q.search(session), key=lambda x: x.object_id.id) + self.assertEqual(2, len(objs)) + self.assertEqual(objs[0].object_id.id, 'id1') + self.assertEqual(objs[1].object_id.id, 'id2') + + def test_simple_query2(self): + q = query.Query({ + CLASS_FQN: [ + { + 'field1': ['b'], + 'field2': ['b'], + } + ] + }) + with model.Session() as session: + objs = sorted(q.search(session), key=lambda x: x.object_id.id) + self.assertEqual(1, len(objs)) + self.assertEqual(objs[0].object_id.id, 'id4') + + def test_simple_query3(self): + q = query.Query({ + CLASS_FQN: [ + { + 'field1': ['a'], + }, + { + 'field2': ['b'], + }, + ] + }) + with model.Session() as session: + objs = sorted(q.search(session), key=lambda x: x.object_id.id) + self.assertEqual(3, len(objs)) + self.assertEqual(objs[0].object_id.id, 'id1') + self.assertEqual(objs[1].object_id.id, 'id2') + self.assertEqual(objs[2].object_id.id, 'id4') + + def test_simple_query_negative(self): + q = query.Query({ + CLASS_FQN: [ + { + '!field1': ['b'], + 'field2': ['b'], + } + ] + }) + with model.Session() as session: + objs = sorted(q.search(session), key=lambda x: x.object_id.id) + self.assertEqual(1, len(objs)) + self.assertEqual(objs[0].object_id.id, 'id2') + + def test_jmespath_query(self): + q = query.Query({ + CLASS_FQN: [ + '[? field1 == `b` && field2 == `a` ]' + ] + }) + with model.Session() as session: + objs = sorted(q.search(session), key=lambda x: x.object_id.id) + self.assertEqual(1, len(objs)) + self.assertEqual(objs[0].object_id.id, 'id3') From 61326e8d55b342fe642708f9bf49d06fa6712e85 Mon Sep 17 00:00:00 2001 From: Anton Frolov Date: Thu, 17 Mar 2016 14:07:28 -0700 Subject: [PATCH 3/3] [CF-257] Use migration name for estimate_migration New cloud object queries are defined in migrations section of configuration file. This commit modify estimate_migration to use name of migration in this section to provide estimates for that migration. --- cloudferrylib/config.py | 4 ++-- cloudferrylib/os/estimation/procedures.py | 26 ++++++++++++++++------- fabfile.py | 13 +++++++++--- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/cloudferrylib/config.py b/cloudferrylib/config.py index 693cd676..8229b340 100644 --- a/cloudferrylib/config.py +++ b/cloudferrylib/config.py @@ -69,8 +69,8 @@ def _do_deserialize(self, value): for field in self.variants: try: return field.deserialize(value) - except marshmallow.ValidationError as e: - errors.append(e) + except marshmallow.ValidationError as ex: + errors.append(ex) raise marshmallow.ValidationError([e.messages for e in errors]) diff --git a/cloudferrylib/os/estimation/procedures.py b/cloudferrylib/os/estimation/procedures.py index 22b6afc6..a435cd00 100644 --- a/cloudferrylib/os/estimation/procedures.py +++ b/cloudferrylib/os/estimation/procedures.py @@ -25,7 +25,11 @@ def list_filtered(session, cls, cloud_name, tenant): if tenant is None or tenant == x.tenant.object_id.id) -def estimate_copy(cloud_name, tenant): +def estimate_copy(cfg, migration_name): + migration = cfg.migrations[migration_name] + query = migration.query + src_cloud = migration.source + with model.Session() as session: total_ephemeral_size = 0 total_volume_size = 0 @@ -33,7 +37,7 @@ def estimate_copy(cloud_name, tenant): accounted_volumes = set() accounted_images = set() - for server in list_filtered(session, nova.Server, cloud_name, tenant): + for server in query.search(session, src_cloud, nova.Server): for ephemeral_disk in server.ephemeral_disks: total_ephemeral_size += ephemeral_disk.size if server.image is not None \ @@ -44,14 +48,16 @@ def estimate_copy(cloud_name, tenant): if volume.object_id not in accounted_volumes: total_volume_size += volume.size accounted_volumes.add(volume.object_id) - for volume in list_filtered(session, cinder.Volume, cloud_name, - tenant): + + for volume in query.search(session, src_cloud, cinder.Volume): if volume.object_id not in accounted_volumes: total_volume_size += volume.size - for image in list_filtered(session, glance.Image, cloud_name, tenant): + + for image in query.search(session, src_cloud, glance.Image): if image.object_id not in accounted_images: total_image_size += image.size + print 'Migration', migration_name, 'estimates:' print 'Images:' print ' Size:', sizeof_format.sizeof_fmt(total_image_size) print 'Ephemeral disks:' @@ -60,7 +66,7 @@ def estimate_copy(cloud_name, tenant): print ' Size:', sizeof_format.sizeof_fmt(total_volume_size, 'G') -def show_largest_servers(count, cloud_name, tenant): +def show_largest_servers(cfg, count, migration_name): def server_size(server): size = 0 if server.image is not None: @@ -72,15 +78,19 @@ def server_size(server): return size output = [] + migration = cfg.migrations[migration_name] with model.Session() as session: for index, server in enumerate( heapq.nlargest( count, - list_filtered(session, nova.Server, cloud_name, tenant), + migration.query.search(session, migration.source, + nova.Server), key=server_size), start=1): output.append( - ' {0}. {1.object_id.id} {1.name}'.format(index, server)) + ' {0}. {1.object_id.id} {1.name} - {2}'.format( + index, server, + sizeof_format.sizeof_fmt(server_size(server)))) if output: print '\n{0} largest servers:'.format(len(output)) for line in output: diff --git a/fabfile.py b/fabfile.py index 9299c034..e03c571f 100644 --- a/fabfile.py +++ b/fabfile.py @@ -206,11 +206,18 @@ def discover(config_path, debug=False): @task -def estimate_migration(config_path, source, tenant=None, debug=False): +def estimate_migration(config_path, migration, debug=False): cfg = config.load(load_yaml_config(config_path, debug)) + if migration not in cfg.migrations: + print 'No such migration:', migration + print '\nPlease choose one of this:' + for name in sorted(cfg.migrations.keys()): + print ' -', name + return -1 + stage.execute_stage('cloudferrylib.os.discovery.stages.DiscoverStage', cfg) - procedures.estimate_copy(source, tenant) - procedures.show_largest_servers(10, source, tenant) + procedures.estimate_copy(cfg, migration) + procedures.show_largest_servers(cfg, 10, migration) @task