From 548d5c91fb1c4a5c19b420e4ed375e5f523621ae Mon Sep 17 00:00:00 2001 From: nikolay_tymtsiv Date: Wed, 16 Sep 2015 18:54:05 +0300 Subject: [PATCH 1/9] SAIC-398 Added check router ips are not overlapping Check gw ips of external router on src and dst are different. --- devlab/tests/config.py | 3 +++ devlab/tests/functional_test.py | 10 ++++++++++ devlab/tests/generate_load.py | 14 ++++++-------- devlab/tests/test_resource_migration.py | 22 ++++++++++++++++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/devlab/tests/config.py b/devlab/tests/config.py index 9c9fe9da..612014af 100644 --- a/devlab/tests/config.py +++ b/devlab/tests/config.py @@ -1,5 +1,8 @@ img_url = 'http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img' +# Path to CloudFerry config relative to the root folder +cloud_ferry_conf = 'configuration.ini' + # Users to create/delete users = [ {'name': 'user1', 'password': 'passwd1', 'email': 'mail@example.com', diff --git a/devlab/tests/functional_test.py b/devlab/tests/functional_test.py index be64402f..475fc693 100644 --- a/devlab/tests/functional_test.py +++ b/devlab/tests/functional_test.py @@ -14,11 +14,21 @@ import config import logging +import sys +import os import unittest from generate_load import Prerequisites from filtering_utils import FilteringUtils +def get_cf_root_folder(): + return os.path.dirname(os.path.dirname(os.path.split(__file__)[0])) + +sys.path.append(get_cf_root_folder()) +import cfglib +cfglib.init_config(os.path.join(get_cf_root_folder(), config.cloud_ferry_conf)) + + def suppress_dependency_logging(): suppressed_logs = ['iso8601.iso8601', 'keystoneclient.session', diff --git a/devlab/tests/generate_load.py b/devlab/tests/generate_load.py index 6755b7da..1dd712cb 100644 --- a/devlab/tests/generate_load.py +++ b/devlab/tests/generate_load.py @@ -130,6 +130,12 @@ def get_user_tenant_roles(self, user): tenant=self.get_tenant_id(tenant.name))) return user_tenant_roles + def get_ext_routers(self): + routers = self.neutronclient.list_routers()['routers'] + ext_routers = [router for router in routers + if router['external_gateway_info']] + return ext_routers + def check_vm_state(self, srv): srv = self.novaclient.servers.get(srv) return srv.status == 'ACTIVE' @@ -824,14 +830,6 @@ def wait_until_vms_all_deleted(): except Exception as e: print "Tenant %s failed to delete: %s" % (tenant['name'], repr(e)) - sgs = self.neutronclient.list_security_groups()['security_groups'] - for sg in sgs: - try: - self.neutronclient.delete_security_group(sg['id']) - except Exception as e: - print "Security group %s failed to delete: %s" % (sg['name'], - repr(e)) - volumes = self.config.cinder_volumes volumes += itertools.chain(*[tenant['cinder_volumes'] for tenant in self.config.tenants if 'cinder_volumes' diff --git a/devlab/tests/test_resource_migration.py b/devlab/tests/test_resource_migration.py index 578cc5a6..643d06a2 100644 --- a/devlab/tests/test_resource_migration.py +++ b/devlab/tests/test_resource_migration.py @@ -387,3 +387,25 @@ def get_fips(client): self.fail("{num} floating IPs did not migrate to destination: " "{fips}".format(num=len(missing_fips), fips=pprint.pformat(missing_fips))) + + @unittest.skipUnless(functional_test.cfglib.CONF.migrate.change_router_ips, + 'Change router ips disabled in CloudFerry config') + def test_ext_router_ip_changed(self): + dst_routers = self.dst_cloud.get_ext_routers() + src_routers = self.src_cloud.get_ext_routers() + for dst_router in dst_routers: + for src_router in src_routers: + if dst_router['name'] != src_router['name']: + continue + src_gateway = self.src_cloud.neutronclient.list_ports( + device_id=src_router['id'], + device_owner='network:router_gateway')['ports'][0] + dst_gateway = self.dst_cloud.neutronclient.list_ports( + device_id=dst_router['id'], + device_owner='network:router_gateway')['ports'][0] + self.assertNotEqual( + src_gateway['fixed_ips'][0]['ip_address'], + dst_gateway['fixed_ips'][0]['ip_address'], + 'GW ip addresses of router "{0}" are same on src and dst:' + ' {1}'.format(dst_router['name'], + dst_gateway['fixed_ips'][0]['ip_address'])) From 6f600782c4eb4223c697559b952439ac75889965 Mon Sep 17 00:00:00 2001 From: nikolay_tymtsiv Date: Thu, 24 Sep 2015 13:19:10 +0300 Subject: [PATCH 2/9] SAIC-475 Added test skipping if nothing to check --- devlab/tests/test_resource_migration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/devlab/tests/test_resource_migration.py b/devlab/tests/test_resource_migration.py index 578cc5a6..d38e1a21 100644 --- a/devlab/tests/test_resource_migration.py +++ b/devlab/tests/test_resource_migration.py @@ -12,6 +12,9 @@ class ResourceMigrationTests(functional_test.FunctionalTest): def validate_resource_parameter_in_dst(self, src_list, dst_list, resource_name, parameter): + if not src_list: + self.skipTest( + 'Nothing to migrate - source resources list is empty') name_attr = 'name' if resource_name == 'volume': name_attr = 'display_name' @@ -35,6 +38,9 @@ def validate_resource_parameter_in_dst(self, src_list, dst_list, def validate_neutron_resource_parameter_in_dst(self, src_list, dst_list, resource_name='networks', parameter='name'): + if not src_list[resource_name]: + self.skipTest( + 'Nothing to migrate - source resources list is empty') for i in src_list[resource_name]: for j in dst_list[resource_name]: if i['name'] != j['name']: From 0eab5a985039547f7aff32c745ee0029de9b028f Mon Sep 17 00:00:00 2001 From: Stas Egorov Date: Thu, 17 Sep 2015 19:32:08 +0300 Subject: [PATCH 3/9] Fixed SAIC-452: re-create image without checksum --- cloudferrylib/base/image.py | 16 +++-- .../os/actions/convert_file_to_image.py | 7 +-- .../os/actions/recreate_boot_image.py | 2 +- cloudferrylib/os/image/glance_image.py | 60 +++++++++++++++---- .../os/image/test_glance_image.py | 12 +++- 5 files changed, 73 insertions(+), 24 deletions(-) diff --git a/cloudferrylib/base/image.py b/cloudferrylib/base/image.py index 80bffcaf..2ac764c0 100644 --- a/cloudferrylib/base/image.py +++ b/cloudferrylib/base/image.py @@ -39,20 +39,26 @@ def glance_image_download_cmd(config, image_id, destination_file): file=destination_file) -def glance_image_create_cmd(config, image_name, disk_format, filename, +def glance_image_create_cmd(config, image_name, disk_format, file_path, container_format="bare"): """Generates glance command which creates image based on arguments provided. Command output is filtered for 'id' :returns: Openstack CLI command""" - args = ("--name {image_name} " + if file_path.startswith("http"): + file_prefix = "location" + else: + file_prefix = "file" + args = ("image-create " + "--name {image_name} " "--disk-format={disk_format} " "--container-format={container_format} " - "--file {file}").format( + "--{file_prefix} {file_path}").format( image_name=image_name, disk_format=disk_format, container_format=container_format, - file=filename + file_prefix=file_prefix, + file_path=file_path ) return "{image_create} | grep '\'".format( - image_create=clients.os_cli_cmd(config, 'image-create', *args)) + image_create=clients.os_cli_cmd(config, 'glance', args)) diff --git a/cloudferrylib/os/actions/convert_file_to_image.py b/cloudferrylib/os/actions/convert_file_to_image.py index 398936ef..c368d190 100644 --- a/cloudferrylib/os/actions/convert_file_to_image.py +++ b/cloudferrylib/os/actions/convert_file_to_image.py @@ -1,13 +1,10 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils -from cloudferrylib.utils import remote_runner class ConvertFileToImage(action.Action): - def run(self, file_path=None, image_format=None, image_name=None, **kwargs): + def run(self, file_path=None, img_fmt=None, img_name=None, **kwargs): image_resource = self.cloud.resources[utils.IMAGE_RESOURCE] - cfg = self.cloud.cloud_config.cloud - runner = remote_runner.RemoteRunner(cfg.host, cfg.ssh_user) - return image_resource.glance_img_create(runner, image_name, image_format, file_path) + return image_resource.glance_img_create(img_name, img_fmt, file_path) diff --git a/cloudferrylib/os/actions/recreate_boot_image.py b/cloudferrylib/os/actions/recreate_boot_image.py index 602afab2..22735bfc 100644 --- a/cloudferrylib/os/actions/recreate_boot_image.py +++ b/cloudferrylib/os/actions/recreate_boot_image.py @@ -104,7 +104,7 @@ def process_image(self, img_id=None, diff=None): verified_file_copy(self.src_runner, self.dst_runner, self.dst_user, diff_file, dst_base_file, self.dst_host, 1) image_resource = self.dst_cloud.resources[utils.IMAGE_RESOURCE] - id = image_resource.glance_img_create(self.dst_runner, img_id, 'qcow2', + id = image_resource.glance_img_create(img_id, 'qcow2', dst_base_file) checksum = remote_md5_sum(self.dst_runner, dst_base_file) return {'id': id, 'checksum': checksum} diff --git a/cloudferrylib/os/image/glance_image.py b/cloudferrylib/os/image/glance_image.py index db2dae78..462e910e 100644 --- a/cloudferrylib/os/image/glance_image.py +++ b/cloudferrylib/os/image/glance_image.py @@ -29,6 +29,7 @@ from cloudferrylib.base import image from cloudferrylib.utils import file_like_proxy from cloudferrylib.utils import utils as utl +from cloudferrylib.utils import remote_runner LOG = utl.get_log(__name__) @@ -50,6 +51,8 @@ def __init__(self, config, cloud): self.filter_image = [] # get mysql settings self.mysql_connector = cloud.mysql_connector('glance') + self.runner = remote_runner.RemoteRunner(self.host, + self.config.cloud.ssh_user) super(GlanceImage, self).__init__(config) @property @@ -90,7 +93,8 @@ def get_image_list(self): filters = {'is_public': None} img_list = get_img_list(filters=filters) # getting images if tenant is member - for img in self.glance_client.image_members.list(member=self.filter_tenant_id): + for img in self.glance_client.image_members.list( + member=self.filter_tenant_id): for i in img_list: if i.id == img.image_id: LOG.debug("append image(by member) ID {}".format(i.id)) @@ -279,11 +283,26 @@ def make_image_info(self, glance_image, info): if glance_image.status == "active": gl_image = self.convert(glance_image, self.cloud) - info['images'][glance_image.id] = {'image': gl_image, - 'meta': {}, - } - LOG.debug("find image with ID {}({})".format(glance_image.id, - glance_image.name)) + command = ("SELECT value FROM image_locations " + "WHERE image_id=\"{}\" AND deleted=\"0\";" + .format(glance_image.id)) + res = self.mysql_connector.execute(command) + img_loc = None + for row in res: + if img_loc is not None: + LOG.warning("ignoring multi locations for image {}" + .format(glance_image.name)) + break + img_loc = row[0] + + info['images'][glance_image.id] = { + 'image': gl_image, + 'meta': { + 'img_loc': img_loc + }, + } + LOG.debug("find image with ID {}({})" + .format(glance_image.id, glance_image.name)) else: LOG.warning("image {img} was not migrated according to " "status = {status}, (expected status " @@ -346,6 +365,26 @@ def deploy(self, info): self.identity_client.keystone_client.users.find( username=metadata["user_name"]).id del metadata["user_name"] + if gl_image["image"]["checksum"] is None: + LOG.warning("re-creating image {} " + "from original source URL" + .format(gl_image["image"]["id"])) + if meta['img_loc'] is not None: + self.glance_img_create( + gl_image['image']['name'], + gl_image['image']['disk_format'] or "qcow2", + meta['img_loc'] + ) + recreated_image = utl.ext_dict( + name=gl_image["image"]["name"] + ) + migrate_images_list.append( + (recreated_image, gl_image['meta']) + ) + else: + raise exception.AbortMigrationError( + "image information has no original source URL") + continue LOG.debug("Creating image '{image}' ({image_id})".format( image=gl_image["image"]["name"], @@ -363,7 +402,8 @@ def deploy(self, info): name=gl_image['image']['name'], container_format=(gl_image['image']['container_format'] or "bare"), - disk_format=gl_image['image']['disk_format'] or "qcow2", + disk_format=(gl_image['image']['disk_format'] + or "qcow2"), is_public=gl_image['image']['is_public'], protected=gl_image['image']['protected'], owner=gl_image['image']['owner'], @@ -447,10 +487,10 @@ def patch_image(self, backend_storage, image_id): run("rbd -p images info %s --format json" % image_id)) image_from_glance.update(size=out["size"]) - def glance_img_create(self, runner, image_name, image_format, file_path): + def glance_img_create(self, img_name, img_format, file_path): cfg = self.cloud.cloud_config.cloud - cmd = image.glance_image_create_cmd(cfg, image_name, image_format, + cmd = image.glance_image_create_cmd(cfg, img_name, img_format, file_path) - out = runner.run(cmd) + out = self.runner.run(cmd) image_id = out.split("|")[2].replace(' ', '') return image_id diff --git a/tests/cloudferrylib/os/image/test_glance_image.py b/tests/cloudferrylib/os/image/test_glance_image.py index 1d3686d6..f896d064 100644 --- a/tests/cloudferrylib/os/image/test_glance_image.py +++ b/tests/cloudferrylib/os/image/test_glance_image.py @@ -28,6 +28,7 @@ 'tenant': 'fake_tenant', 'region': None, 'host': '1.1.1.1', + 'ssh_user': 'fake_user' }), migrate=utils.ext_dict({'speed_limit': '10MB', 'retry': '7', @@ -69,6 +70,8 @@ def setUp(self): self.fake_cloud.resources = dict(identity=self.identity_mock, image=self.image_mock) + self.fake_cloud.mysql_connector = mock.MagicMock() + self.glance_image = GlanceImage(FAKE_CONFIG, self.fake_cloud) self.fake_image_1 = mock.Mock() @@ -108,8 +111,10 @@ def setUp(self): 'protected': False, 'size': 1024, 'resource': self.image_mock, - 'properties': {'user_name': 'fake_user_name'}}, - 'meta': {}}}, + 'properties': { + 'user_name': 'fake_user_name' + }}, + 'meta': {'img_loc': None}}}, 'tags': {}, 'members': {} } @@ -133,7 +138,8 @@ def test_get_image_list(self): self.glance_mock_client().images.list.return_value = fake_images images_list = self.glance_image.get_image_list() - self.glance_mock_client().images.list.assert_called_once_with(filters={'is_public': None}) + self.glance_mock_client().images.list.assert_called_once_with( + filters={'is_public': None}) self.assertEquals(fake_images, images_list) def test_create_image(self): From b6af529e828212e9731855f6f97561b7015ff435 Mon Sep 17 00:00:00 2001 From: nikolay_tymtsiv Date: Wed, 23 Sep 2015 16:22:02 +0300 Subject: [PATCH 4/9] SAIC-474 Fixed vms filtering in functional tests Fixed bug in vm filtering, when not all vm are filtered correctly. Refactored tests for vm migration to use properties of the object (that all tests were written in the same style). --- devlab/tests/functional_test.py | 3 +- devlab/tests/test_vm_migration.py | 54 +++++++++++++++---------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/devlab/tests/functional_test.py b/devlab/tests/functional_test.py index be64402f..2fe6938d 100644 --- a/devlab/tests/functional_test.py +++ b/devlab/tests/functional_test.py @@ -93,9 +93,10 @@ def filter_roles(self): def filter_vms(self): vms = config.vms [vms.extend(i['vms']) for i in config.tenants if 'vms' in i] + vms_names = [vm['name'] for vm in vms] opts = {'search_opts': {'all_tenants': 1}} return [i for i in self.src_cloud.novaclient.servers.list(**opts) - if i.name in vms] + if i.name in vms_names] def filter_flavors(self): flavors = [i['name'] for i in config.flavors] diff --git a/devlab/tests/test_vm_migration.py b/devlab/tests/test_vm_migration.py index 3131792b..26567c5a 100644 --- a/devlab/tests/test_vm_migration.py +++ b/devlab/tests/test_vm_migration.py @@ -7,16 +7,15 @@ class VmMigration(functional_test.FunctionalTest): def setUp(self): - src_vms = [x.__dict__ for x in self.filter_vms()] - self.dst_vms = [x.__dict__ for x in - self.dst_cloud.novaclient.servers.list( - search_opts={'all_tenants': 1})] - src_vms = [vm for vm in src_vms if vm['status'] != 'ERROR'] + src_vms = self.filter_vms() + self.dst_vms = self.dst_cloud.novaclient.servers.list( + search_opts={'all_tenants': 1}) + src_vms = [vm for vm in src_vms if vm.status != 'ERROR'] self.dst_vm_indexes = [] for vm in src_vms: - if vm['name'] not in config.vms_not_in_filter: + if vm.name not in config.vms_not_in_filter: self.dst_vm_indexes.append( - [x['name'] for x in self.dst_vms].index(vm['name'])) + [x.name for x in self.dst_vms].index(vm.name)) with open('pre_migration_vm_states.json') as data_file: self.before_migr_states = json.load(data_file) self.filter_vms = self.filtering_utils.filter_vms(src_vms) @@ -26,16 +25,15 @@ def test_vms_not_in_filter_stay_active_on_src(self): vms_filtered_out = filter_results[1] original_states = self.before_migr_states for vm in vms_filtered_out: - self.assertTrue(vm['status'] == original_states[vm['name']]) + self.assertTrue(vm.status == original_states[vm.name]) def test_vm_not_in_filter_did_not_migrate(self): filter_results = self.filter_vms vms_filtered_out = filter_results[1] - dst_vms = [x.__dict__['name'] for x in - self.dst_cloud.novaclient.servers.list( - search_opts={'all_tenants': 1})] + dst_vms = [x.name for x in self.dst_cloud.novaclient.servers.list( + search_opts={'all_tenants': 1})] for vm in vms_filtered_out: - self.assertTrue(vm['name'] not in dst_vms, + self.assertTrue(vm.name not in dst_vms, 'VM migrated despite that it was not included in ' 'filter, VM info: \n{}'.format(vm)) @@ -46,28 +44,28 @@ def test_cold_migrate_vm_state(self): original_states.pop(vm_name) src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): - if src_vm['name'] in original_states.keys(): - if original_states[src_vm['name']] == 'ACTIVE' or \ - original_states[src_vm['name']] == 'VERIFY_RESIZE': + if src_vm.name in original_states.keys(): + if original_states[src_vm.name] == 'ACTIVE' or \ + original_states[src_vm.name] == 'VERIFY_RESIZE': self.assertTrue( - src_vm['status'] == 'SHUTOFF' and - self.dst_vms[vm_index]['status'] == 'ACTIVE') + src_vm.status == 'SHUTOFF' and + self.dst_vms[vm_index].status == 'ACTIVE') else: self.assertTrue( - src_vm['status'] == 'SHUTOFF' and - self.dst_vms[vm_index]['status'] == 'SHUTOFF') + src_vm.status == 'SHUTOFF' and + self.dst_vms[vm_index].status == 'SHUTOFF') else: - self.assertTrue(src_vm['status'] == 'SHUTOFF' and - self.dst_vms[vm_index]['status'] == 'ACTIVE') + self.assertTrue(src_vm.status == 'SHUTOFF' and + self.dst_vms[vm_index].status == 'ACTIVE') def test_cold_migrate_vm_ip(self): src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): - for src_net in src_vm['addresses']: - for src_net_addr, dst_net_addr in zip(src_vm['addresses'] + for src_net in src_vm.addresses: + for src_net_addr, dst_net_addr in zip(src_vm.addresses [src_net], self.dst_vms[vm_index] - ['addresses'][src_net]): + .addresses[src_net]): self.assertTrue(src_net_addr['addr'] == dst_net_addr['addr']) @@ -75,13 +73,13 @@ def test_cold_migrate_vm_security_groups(self): src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): dst_sec_group_names = [x['name'] for x in - self.dst_vms[vm_index]['security_groups']] - for security_group in src_vm['security_groups']: + self.dst_vms[vm_index].security_groups] + for security_group in src_vm.security_groups: self.assertTrue(security_group['name'] in dst_sec_group_names) @unittest.skip("Temporarily disabled: image's id changes after migrating") def test_cold_migrate_vm_image_id(self): src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): - self.assertTrue(src_vm['image']['id'] == - self.dst_vms[vm_index]['image']['id']) + self.assertTrue(src_vm.image.id == + self.dst_vms[vm_index].image.id) From c9cedad32a938639d2fdff3f6dc75c75227578e7 Mon Sep 17 00:00:00 2001 From: Maksym Iarmak Date: Tue, 22 Sep 2015 18:52:01 +0300 Subject: [PATCH 5/9] [SAIC-416] Add `connection_attempts` option to config Add `connection_attempts` option for all Fabric SSH connections in the project. New option is located in the `migrate` section of config and has default value 3 (times). --- cfglib.py | 7 +++++-- cloudferrylib/os/actions/check_ssh.py | 4 +++- cloudferrylib/os/actions/convert_file.py | 3 ++- .../os/actions/convert_image_to_file.py | 4 +++- .../os/actions/load_compute_image_to_file.py | 6 +++++- .../os/actions/merge_base_and_diff.py | 14 +++++++++----- .../os/actions/pre_transport_instance.py | 18 ++++++++++++------ .../os/actions/transport_ephemeral.py | 6 ++++-- .../os/actions/upload_file_to_image.py | 5 ++++- cloudferrylib/os/actions/utils.py | 6 ++++-- cloudferrylib/os/compute/instances.py | 6 ++++-- cloudferrylib/os/image/glance_image.py | 5 ++++- cloudferrylib/os/network/nova_network.py | 4 +++- .../utils/drivers/ssh_ceph_to_ceph.py | 5 +++-- .../utils/drivers/ssh_file_to_ceph.py | 5 +++-- .../utils/drivers/ssh_file_to_file.py | 6 ++++-- cloudferrylib/utils/remote_runner.py | 5 ++++- cloudferrylib/utils/utils.py | 15 ++++++++++----- configs/config.ini | 4 ++++ devlab/config.template | 1 + fabfile.py | 1 + 21 files changed, 92 insertions(+), 38 deletions(-) diff --git a/cfglib.py b/cfglib.py index 8eafc596..8a405dc4 100644 --- a/cfglib.py +++ b/cfglib.py @@ -159,9 +159,12 @@ help='IP or hostname used for creating MySQL dump for rollback.' 'If not set uses `[dst] db_host` config option.'), cfg.BoolOpt('optimize_user_role_fetch', default=True, - help=("Uses low-level DB requests if set to True, " + help="Uses low-level DB requests if set to True, " "may be incompatible with more recent versions of " - "Keystone. Tested on grizzly, icehouse and juno.")) + "Keystone. Tested on grizzly, icehouse and juno."), + cfg.IntOpt('ssh_connection_attempts', default=3, + help='Number of times CloudFerry will attempt to connect when ' + 'connecting to a new server via SSH.'), ] mail = cfg.OptGroup(name='mail', diff --git a/cloudferrylib/os/actions/check_ssh.py b/cloudferrylib/os/actions/check_ssh.py index 72038df5..4d19ecb9 100644 --- a/cloudferrylib/os/actions/check_ssh.py +++ b/cloudferrylib/os/actions/check_ssh.py @@ -51,8 +51,10 @@ def check_access(self, node): runner = remote_runner.RemoteRunner(node, cfg.ssh_user, password=cfg.ssh_sudo_password) gateway = self.cloud.getIpSsh() + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + try: - with settings(gateway=gateway): + with settings(gateway=gateway, connection_attempts=ssh_attempts): runner.run('echo') except Exception as error: LOG.error("SSH connection from '%s' to '%s' failed with error: " diff --git a/cloudferrylib/os/actions/convert_file.py b/cloudferrylib/os/actions/convert_file.py index 1f484983..f4adbce9 100644 --- a/cloudferrylib/os/actions/convert_file.py +++ b/cloudferrylib/os/actions/convert_file.py @@ -24,7 +24,8 @@ def run(self, info=None, **kwargs): @staticmethod def convert_file_to_raw(host, disk_format, filepath): - with settings(host_string=host): + with settings(host_string=host, + connection_attempts=env.connection_attempts): with forward_agent(env.key_filename): run("qemu-img convert -f %s -O raw %s %s.tmp" % (disk_format, filepath, filepath)) diff --git a/cloudferrylib/os/actions/convert_image_to_file.py b/cloudferrylib/os/actions/convert_image_to_file.py index ffd4d9b0..06c370d2 100644 --- a/cloudferrylib/os/actions/convert_image_to_file.py +++ b/cloudferrylib/os/actions/convert_image_to_file.py @@ -8,7 +8,9 @@ class ConvertImageToFile(action.Action): def run(self, image_id=None, base_filename=None, **kwargs): cfg = self.cloud.cloud_config.cloud - with settings(host_string=cfg.host): + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + + with settings(host_string=cfg.host, connection_attempts=ssh_attempts): with forward_agent(env.key_filename): cmd = image.glance_image_download_cmd(cfg, image_id, base_filename) diff --git a/cloudferrylib/os/actions/load_compute_image_to_file.py b/cloudferrylib/os/actions/load_compute_image_to_file.py index 961b6839..59bb2b34 100644 --- a/cloudferrylib/os/actions/load_compute_image_to_file.py +++ b/cloudferrylib/os/actions/load_compute_image_to_file.py @@ -20,13 +20,17 @@ class LoadComputeImageToFile(action.Action): def run(self, info=None, **kwargs): cfg = self.cloud.cloud_config.cloud + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): inst = info[utl.INSTANCES_TYPE][instance_id][utl.INSTANCE_BODY] image_id = inst['image_id'] base_file = "/tmp/%s" % ("temp%s_base" % instance_id) diff_file = "/tmp/%s" % ("temp%s" % instance_id) - with settings(host_string=cfg.host): + + with settings(host_string=cfg.host, + connection_attempts=ssh_attempts): with forward_agent(env.key_filename): cmd = image.glance_image_download_cmd(cfg, image_id, base_file) diff --git a/cloudferrylib/os/actions/merge_base_and_diff.py b/cloudferrylib/os/actions/merge_base_and_diff.py index da602bd9..5c5e31b3 100644 --- a/cloudferrylib/os/actions/merge_base_and_diff.py +++ b/cloudferrylib/os/actions/merge_base_and_diff.py @@ -16,11 +16,15 @@ def run(self, info=None, **kwargs): self.rebase_diff_file(host, base_file, diff_file) self.commit_diff_file(host, diff_file) - def rebase_diff_file(self, host, base_file, diff_file): + @staticmethod + def rebase_diff_file(host, base_file, diff_file): cmd = "qemu-img rebase -u -b %s %s" % (base_file, diff_file) - with settings(host_string=host): + with settings(host_string=host, + connection_attempts=env.connection_attempts): run(cmd) - def commit_diff_file(self, host, diff_file): - with settings(host_string=host): - run("qemu-img commit %s" % diff_file) \ No newline at end of file + @staticmethod + def commit_diff_file(host, diff_file): + with settings(host_string=host, + connection_attempts=env.connection_attempts): + run("qemu-img commit %s" % diff_file) diff --git a/cloudferrylib/os/actions/pre_transport_instance.py b/cloudferrylib/os/actions/pre_transport_instance.py index e7254700..a10701f2 100644 --- a/cloudferrylib/os/actions/pre_transport_instance.py +++ b/cloudferrylib/os/actions/pre_transport_instance.py @@ -170,18 +170,24 @@ def transport_diff_and_merge(self, dst_cloud, info, instance_id): image_name="%s-image" % instance_id) info[INSTANCES][instance_id][INSTANCE_BODY]['image_id'] = dst_image_id - def convert_file_to_raw(self, host, disk_format, filepath): - with settings(host_string=host): + @staticmethod + def convert_file_to_raw(host, disk_format, filepath): + with settings(host_string=host, + connection_attempts=env.connection_attempts): with forward_agent(env.key_filename): run("qemu-img convert -f %s -O raw %s %s.tmp" % (disk_format, filepath, filepath)) run("mv -f %s.tmp %s" % (filepath, filepath)) - def rebase_diff_file(self, host, base_file, diff_file): + @staticmethod + def rebase_diff_file(host, base_file, diff_file): cmd = "qemu-img rebase -u -b %s %s" % (base_file, diff_file) - with settings(host_string=host): + with settings(host_string=host, + connection_attempts=env.connection_attempts): run(cmd) - def commit_diff_file(self, host, diff_file): - with settings(host_string=host): + @staticmethod + def commit_diff_file(host, diff_file): + with settings(host_string=host, + connection_attempts=env.connection_attempts): run("qemu-img commit %s" % diff_file) diff --git a/cloudferrylib/os/actions/transport_ephemeral.py b/cloudferrylib/os/actions/transport_ephemeral.py index 14430c0d..16b01edd 100644 --- a/cloudferrylib/os/actions/transport_ephemeral.py +++ b/cloudferrylib/os/actions/transport_ephemeral.py @@ -85,9 +85,11 @@ def run(self, info=None, **kwargs): 'info': new_info } - def delete_remote_file_on_compute(self, path_file, host_cloud, + @staticmethod + def delete_remote_file_on_compute(path_file, host_cloud, host_instance): - with settings(host_string=host_cloud): + with settings(host_string=host_cloud, + connection_attempts=env.connection_attempts): with forward_agent(env.key_filename): run("ssh -oStrictHostKeyChecking=no %s 'rm -rf %s'" % (host_instance, path_file)) diff --git a/cloudferrylib/os/actions/upload_file_to_image.py b/cloudferrylib/os/actions/upload_file_to_image.py index e3be7942..9058295a 100644 --- a/cloudferrylib/os/actions/upload_file_to_image.py +++ b/cloudferrylib/os/actions/upload_file_to_image.py @@ -31,7 +31,9 @@ class UploadFileToImage(action.Action): def run(self, info=None, **kwargs): cfg = self.cloud.cloud_config.cloud + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts img_res = self.cloud.resources[utl.IMAGE_RESOURCE] + for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): # init inst_body = info[INSTANCES][instance_id][utl.INSTANCE_BODY] @@ -43,7 +45,8 @@ def run(self, info=None, **kwargs): if img_res.config.image.convert_to_raw: image_format = utl.RAW # action - with settings(host_string=cfg.host): + with settings(host_string=cfg.host, + connection_attempts=ssh_attempts): cmd = image.glance_image_create_cmd(cfg, image_name, image_format, base_file) out = run(cmd) diff --git a/cloudferrylib/os/actions/utils.py b/cloudferrylib/os/actions/utils.py index b899b3c2..38d2313b 100644 --- a/cloudferrylib/os/actions/utils.py +++ b/cloudferrylib/os/actions/utils.py @@ -25,7 +25,8 @@ def transfer_file_to_file(src_cloud, dst_cloud, host_src, host_dst, path_src, pa LOG.debug("| | copy file") ssh_ip_src = src_cloud.getIpSsh() ssh_ip_dst = dst_cloud.getIpSsh() - with settings(host_string=ssh_ip_src): + with settings(host_string=ssh_ip_src, + connection_attempts=env.connection_attempts): with utils.forward_agent(cfg_migrate.key_filename): with utils.up_ssh_tunnel(host_dst, ssh_ip_dst, ssh_ip_src) as port: if cfg_migrate.file_compression == "dd": @@ -40,7 +41,8 @@ def transfer_file_to_file(src_cloud, dst_cloud, host_src, host_dst, path_src, pa def delete_file_from_rbd(ssh_ip, file_path): - with settings(host_string=ssh_ip): + with settings(host_string=ssh_ip, + connection_attempts=env.connection_attempts): with utils.forward_agent(env.key_filename): run("rbd rm %s" % file_path) diff --git a/cloudferrylib/os/compute/instances.py b/cloudferrylib/os/compute/instances.py index 77ea2316..9a8d361e 100644 --- a/cloudferrylib/os/compute/instances.py +++ b/cloudferrylib/os/compute/instances.py @@ -69,8 +69,10 @@ def cobalt_live_migrate_vm(config, vm_id, dest_host): host_string = "{user}@{host}".format( user=config.cloud.ssh_user, host=config.cloud.ssh_host) - with settings(warn_only=True, host_string=host_string, - key_filename=config.migrate.key_filename): + with settings(warn_only=True, + host_string=host_string, + key_filename=config.migrate.key_filename, + connection_attempts=config.migrate.ssh_connection_attempts): migrate_cmd = clients.os_cli_cmd(config.cloud, "nova", "cobalt-migrate", vm_id, "--dest", dest_host) diff --git a/cloudferrylib/os/image/glance_image.py b/cloudferrylib/os/image/glance_image.py index c7dc8e13..adf2039d 100644 --- a/cloudferrylib/os/image/glance_image.py +++ b/cloudferrylib/os/image/glance_image.py @@ -439,9 +439,12 @@ def get_status(self, res_id): return self.glance_client.images.get(res_id).status def patch_image(self, backend_storage, image_id): + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + if backend_storage == 'ceph': image_from_glance = self.get_image_by_id(image_id) - with settings(host_string=self.cloud.getIpSsh()): + with settings(host_string=self.cloud.getIpSsh(), + connection_attempts=ssh_attempts): out = json.loads( run("rbd -p images info %s --format json" % image_id)) image_from_glance.update(size=out["size"]) diff --git a/cloudferrylib/os/network/nova_network.py b/cloudferrylib/os/network/nova_network.py index 2ba3b0f8..342c06fd 100644 --- a/cloudferrylib/os/network/nova_network.py +++ b/cloudferrylib/os/network/nova_network.py @@ -92,8 +92,10 @@ def get_func_mac_address(self, instance): def get_mac_addresses(self, instance): compute_node = getattr(instance, nova_compute.INSTANCE_HOST_ATTRIBUTE) libvirt_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name') + ssh_attempts = self.config.migrate.ssh_connection_attempts - with settings(host_string=self.config['host']): + with settings(host_string=self.config['host'], + connection_attempts=ssh_attempts): with forward_agent(env.key_filename): cmd = "virsh dumpxml %s | grep 'mac address' | " \ "cut -d\\' -f2" % libvirt_name diff --git a/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py b/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py index 8443afe7..7ef26982 100644 --- a/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py +++ b/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py @@ -33,8 +33,9 @@ def transfer(self, data, snapshot=None, snapshot_type=1): else self.src_cloud.getIpSsh()) host_dst = (data.get('host_dst') if data.get('host_dst') else self.dst_cloud.getIpSsh()) - with settings(host_string=host_src), utils.forward_agent( - env.key_filename): + with (settings(host_string=host_src, + connection_attempts=env.connection_attempts), + utils.forward_agent(env.key_filename)): rbd_import_diff = rbd_util.RbdUtil.rbd_import_diff_cmd ssh_cmd = cmd_cfg.ssh_cmd diff --git a/cloudferrylib/utils/drivers/ssh_file_to_ceph.py b/cloudferrylib/utils/drivers/ssh_file_to_ceph.py index 78dfef2b..7c5cf930 100644 --- a/cloudferrylib/utils/drivers/ssh_file_to_ceph.py +++ b/cloudferrylib/utils/drivers/ssh_file_to_ceph.py @@ -32,8 +32,9 @@ def transfer(self, data): ssh_ip_src = self.src_cloud.getIpSsh() ssh_ip_dst = self.dst_cloud.getIpSsh() action_utils.delete_file_from_rbd(ssh_ip_dst, data['path_dst']) - with settings(host_string=ssh_ip_src), utils.forward_agent( - env.key_filename): + with (settings(host_string=ssh_ip_src, + connection_attempts=env.connection_attempts), + utils.forward_agent(env.key_filename)): rbd_import = rbd_util.RbdUtil.rbd_import_cmd ssh_cmd_dst = cmd_cfg.ssh_cmd ssh_dst = ssh_cmd_dst(ssh_ip_dst, rbd_import) diff --git a/cloudferrylib/utils/drivers/ssh_file_to_file.py b/cloudferrylib/utils/drivers/ssh_file_to_file.py index c7a2ca21..e5ce7947 100644 --- a/cloudferrylib/utils/drivers/ssh_file_to_file.py +++ b/cloudferrylib/utils/drivers/ssh_file_to_file.py @@ -69,14 +69,16 @@ def transfer(self, data): self.src_cloud.ssh_util.execute(process) def transfer_direct(self, data): + ssh_attempts = self.cfg.migrate.ssh_connection_attempts LOG.debug("| | copy file") if self.cfg.src.ssh_user != 'root' or self.cfg.dst.ssh_user != 'root': LOG.critical("This operation needs 'sudo' access rights, that is " "currently not implemented in this driver. Please use" " 'CopyFilesBetweenComputeHosts' driver from " "cloudferrylib/utils/drivers/.") - with settings(host_string=data['host_src']), utils.forward_agent( - self.cfg.migrate.key_filename): + with (settings(host_string=data['host_src'], + connection_attempts=ssh_attempts), + utils.forward_agent(self.cfg.migrate.key_filename)): if self.cfg.migrate.file_compression == "dd": dd_dst = cmd_cfg.dd_cmd_of ssh_cmd_dst = cmd_cfg.ssh_cmd diff --git a/cloudferrylib/utils/remote_runner.py b/cloudferrylib/utils/remote_runner.py index f5a75c60..989f0ec9 100644 --- a/cloudferrylib/utils/remote_runner.py +++ b/cloudferrylib/utils/remote_runner.py @@ -43,13 +43,16 @@ def run(self, cmd): if not self.ignore_errors: abort_exception = RemoteExecutionError + ssh_attempts = cfglib.CONF.migrate.ssh_connection_attempts + with settings(warn_only=self.ignore_errors, host_string=self.host, user=self.user, password=self.password, abort_exception=abort_exception, reject_unkown_hosts=False, - combine_stderr=False): + combine_stderr=False, + connection_attempts=ssh_attempts): with forward_agent(self.key): LOG.debug("running '%s' on '%s' host as user '%s'", cmd, self.host, self.user) diff --git a/cloudferrylib/utils/utils.py b/cloudferrylib/utils/utils.py index 49cab0db..be85df78 100644 --- a/cloudferrylib/utils/utils.py +++ b/cloudferrylib/utils/utils.py @@ -306,7 +306,8 @@ def __init__(self, key_file): def _agent_already_running(self): with settings(hide('warnings', 'running', 'stdout', 'stderr'), - warn_only=True): + warn_only=True, + connection_attempts=env.connection_attempts): res = local("ssh-add -l", capture=True) if res.succeeded: @@ -383,13 +384,15 @@ def __init__(self, address_dest_compute, address_dest_controller, host, callback def __enter__(self): self.port = self.get_free_port() - with settings(host_string=self.host): + with settings(host_string=self.host, + connection_attempts=env.connection_attempts): run(self.cmd % (self.port, self.address_dest_compute, self.port, self.port, self.address_dest_controller) + " && sleep 2") return self.port def __exit__(self, type, value, traceback): - with settings(host_string=self.host): + with settings(host_string=self.host, + connection_attempts=env.connection_attempts): run(("pkill -f '"+self.cmd+"'") % (self.port, self.address_dest_compute, self.port, self.port, self.address_dest_controller)) time.sleep(2) @@ -422,7 +425,8 @@ def get_libvirt_block_info(libvirt_name, init_host, compute_host, ssh_user, with settings(host_string=compute_host, user=ssh_user, password=ssh_sudo_password, - gateway=init_host): + gateway=init_host, + connection_attempts=env.connection_attempts): out = sudo("virsh domblklist %s" % libvirt_name) libvirt_output = out.split() return libvirt_output @@ -458,7 +462,8 @@ def get_disk_path(instance, blk_list, is_ceph_ephemeral=False, disk=DISK): def get_ips(init_host, compute_host, ssh_user): with settings(host_string=compute_host, user=ssh_user, - gateway=init_host): + gateway=init_host, + connection_attempts=env.connection_attempts): cmd = ("ifconfig | awk -F \"[: ]+\" \'/inet addr:/ " "{ if ($4 != \"127.0.0.1\") print $4 }\'") out = run(cmd) diff --git a/configs/config.ini b/configs/config.ini index 6fe4726a..b79b41c2 100644 --- a/configs/config.ini +++ b/configs/config.ini @@ -163,6 +163,10 @@ mysqldump_host = # default optimize_user_role_fetch = [True|False] +# Number of times CloudFerry will attempt to connect when connecting to a new +# server via SSH. +ssh_connection_attempts = 3 + #============================================================================== # Mailing configuration diff --git a/devlab/config.template b/devlab/config.template index 3880fb2c..d966bdf4 100755 --- a/devlab/config.template +++ b/devlab/config.template @@ -29,6 +29,7 @@ cinder_migration_strategy = cloudferrylib.os.storage.cinder_database.CinderStora skip_down_hosts = True migrate_user_quotas = False incloud_live_migration = nova +ssh_connection_attempts = 3 [src] type = os diff --git a/fabfile.py b/fabfile.py index 37461ad4..fae438d0 100644 --- a/fabfile.py +++ b/fabfile.py @@ -53,6 +53,7 @@ def migrate(name_config=None, name_instance=None, debug=False): cfglib.init_config(name_config) utils.init_singletones(cfglib.CONF) env.key_filename = cfglib.CONF.migrate.key_filename + env.connection_attempts = cfglib.CONF.migrate.ssh_connection_attempts cloud = cloud_ferry.CloudFerry(cfglib.CONF) cloud.migrate(Scenario(path_scenario=cfglib.CONF.migrate.scenario, path_tasks=cfglib.CONF.migrate.tasks_mapping)) From af16f7cf0768ae05daf75aea4bd670d63f0dcb46 Mon Sep 17 00:00:00 2001 From: nikolay_tymtsiv Date: Mon, 21 Sep 2015 19:22:39 +0300 Subject: [PATCH 6/9] SAIC-467 Added test for image without checksum --- devlab/tests/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/devlab/tests/config.py b/devlab/tests/config.py index 9c9fe9da..665d41d1 100644 --- a/devlab/tests/config.py +++ b/devlab/tests/config.py @@ -95,7 +95,10 @@ {'name': 'image4', 'copy_from': img_url, 'container_format': 'bare', 'disk_format': 'qcow2', 'is_public': False}, {'name': 'image5', 'copy_from': img_url, 'container_format': 'bare', - 'disk_format': 'qcow2', 'is_public': False} + 'disk_format': 'qcow2', 'is_public': False}, + # When location field is specified, glance creates images without checksum + {'name': 'without_checksum', 'location': img_url, 'disk_format': 'qcow2', + 'container_format': 'bare'} ] # Images not to be migrated: From 9d16f946593b2a7d54e4165b441422aed651d9cb Mon Sep 17 00:00:00 2001 From: Stas Egorov Date: Fri, 18 Sep 2015 15:26:25 +0300 Subject: [PATCH 7/9] Fixed SAIC-51 partially: fixed pep8 warnings --- addons/__init__.py | 1 - addons/aws_plugin/__init__.py | 2 +- cloud/__init__.py | 2 +- cloud/cloud.py | 38 ++- cloud/os2os.py | 301 ++++++++++++------ cloudferrylib/__init__.py | 3 - cloudferrylib/base/__init__.py | 1 - cloudferrylib/base/action/copy_var.py | 1 - cloudferrylib/base/action/create_reference.py | 2 - cloudferrylib/base/action/get_info_iter.py | 8 +- cloudferrylib/base/action/is_end_iter.py | 5 - cloudferrylib/base/action/is_option.py | 9 +- cloudferrylib/base/action/merge.py | 1 - cloudferrylib/base/action/rename_info.py | 2 - cloudferrylib/base/network.py | 1 - cloudferrylib/base/objstorage.py | 1 - cloudferrylib/base/resource.py | 5 +- cloudferrylib/os/__init__.py | 1 - .../os/actions/attach_used_volumes.py | 2 +- .../attach_used_volumes_via_compute.py | 14 +- cloudferrylib/os/actions/check_bandwidth.py | 7 +- cloudferrylib/os/actions/check_cloud.py | 22 +- cloudferrylib/os/actions/check_filter.py | 37 ++- .../os/actions/check_instance_networks.py | 3 +- .../actions/check_needed_compute_resources.py | 6 +- cloudferrylib/os/actions/check_networks.py | 4 +- .../os/actions/check_openstack_apis.py | 16 +- .../actions/cinder_database_manipulation.py | 4 +- cloudferrylib/os/actions/cleanup_images.py | 3 +- .../os/actions/convert_compute_to_image.py | 3 +- .../os/actions/convert_compute_to_volume.py | 4 +- cloudferrylib/os/actions/convert_file.py | 5 +- .../os/actions/convert_volume_to_compute.py | 5 +- .../os/actions/convert_volume_to_image.py | 5 +- cloudferrylib/os/actions/copy_g2g.py | 4 +- cloudferrylib/os/actions/deploy_snapshots.py | 65 ++-- cloudferrylib/os/actions/deploy_volumes.py | 4 - cloudferrylib/os/actions/failure_act.py | 3 +- cloudferrylib/os/actions/image_snapshot.py | 1 + .../os/actions/is_not_copy_diff_file.py | 13 +- cloudferrylib/os/actions/is_not_merge_diff.py | 10 +- .../os/actions/is_not_transport_image.py | 8 +- cloudferrylib/os/actions/live_migration.py | 30 +- cloudferrylib/os/actions/map_compute_info.py | 13 +- .../os/actions/merge_base_and_diff.py | 1 - .../os/actions/post_transport_instance.py | 38 +-- .../os/actions/pre_transport_instance.py | 57 ++-- .../os/actions/prepare_volumes_data_map.py | 3 +- .../os/actions/recreate_boot_image.py | 44 ++- cloudferrylib/os/actions/remote_execution.py | 4 +- .../os/actions/select_boot_volume.py | 1 - cloudferrylib/os/actions/set_volume_id.py | 1 - .../os/actions/transport_ephemeral.py | 2 +- .../os/actions/transport_instance.py | 14 +- cloudferrylib/os/actions/utils.py | 19 +- cloudferrylib/os/compute/nova_compute.py | 5 +- cloudferrylib/os/identity/keystone.py | 29 +- cloudferrylib/os/network/__init__.py | 1 - cloudferrylib/os/network/neutron.py | 5 +- cloudferrylib/os/object_storage/__init__.py | 1 - .../os/object_storage/swift_storage.py | 62 ++-- cloudferrylib/os/storage/cinder_database.py | 14 +- cloudferrylib/os/storage/cinder_storage.py | 1 - cloudferrylib/scheduler/base/begin_task.py | 2 +- cloudferrylib/scheduler/base/end_task.py | 2 +- cloudferrylib/scheduler/cursor.py | 3 +- cloudferrylib/scheduler/namespace.py | 5 +- cloudferrylib/scheduler/scenario.py | 13 +- cloudferrylib/scheduler/task.py | 5 +- cloudferrylib/scheduler/utils.py | 2 +- cloudferrylib/scheduler/utils/equ_instance.py | 2 +- cloudferrylib/utils/__init__.py | 2 +- cloudferrylib/utils/console_cmd.py | 2 +- cloudferrylib/utils/drivers/__init__.py | 1 - .../utils/drivers/ssh_ceph_to_ceph.py | 11 +- cloudferrylib/utils/drivers/ssh_chunks.py | 4 +- cloudferrylib/utils/file_like_proxy.py | 3 +- cloudferrylib/utils/proxy_client.py | 4 +- cloudferrylib/utils/qemu_img.py | 10 +- cloudferrylib/utils/rbd_util.py | 18 +- cloudferrylib/utils/remote_runner.py | 3 +- cloudferrylib/utils/ssh_util.py | 14 +- cloudferrylib/utils/utils.py | 75 +++-- condensation/process.py | 3 +- data_storage.py | 2 + .../os/storage/test_cinder_netapp.py | 6 +- .../os/storage/test_cinder_storage.py | 6 +- 87 files changed, 707 insertions(+), 468 deletions(-) diff --git a/addons/__init__.py b/addons/__init__.py index 71f21bcd..e69de29b 100644 --- a/addons/__init__.py +++ b/addons/__init__.py @@ -1 +0,0 @@ -import aws_plugin \ No newline at end of file diff --git a/addons/aws_plugin/__init__.py b/addons/aws_plugin/__init__.py index 029374b7..4b32c42c 100644 --- a/addons/aws_plugin/__init__.py +++ b/addons/aws_plugin/__init__.py @@ -17,4 +17,4 @@ cfg_for_reg = [ (migrate, migrate_opts) -] \ No newline at end of file +] diff --git a/cloud/__init__.py b/cloud/__init__.py index 220f7afd..9bf269fa 100644 --- a/cloud/__init__.py +++ b/cloud/__init__.py @@ -1,3 +1,3 @@ import cloud import cloud_ferry -import os2os \ No newline at end of file +import os2os diff --git a/cloud/cloud.py b/cloud/cloud.py index e8560ec5..0eeb56ab 100644 --- a/cloud/cloud.py +++ b/cloud/cloud.py @@ -37,7 +37,8 @@ def __init__(self, resources, position, config): def getIpSsh(self): return self.cloud_config.cloud.ssh_host \ - if self.cloud_config.cloud.ssh_host else self.cloud_config.cloud.host + if self.cloud_config.cloud.ssh_host \ + else self.cloud_config.cloud.host @staticmethod def make_cloud_config(config, position): @@ -75,8 +76,10 @@ def make_resource_config(config, position, cloud_config, resource_name): def get_db_method_create_connection(resource, config): def get_db_connection(db_name): conf_res = getattr(config, resource) - conf = {k: getattr(conf_res, k, None) if getattr(conf_res, k, None) else config.mysql[k] - for k in config.mysql.keys()} + conf = { + k: getattr(conf_res, k, None) if getattr(conf_res, k, None) + else config.mysql[k] + for k in config.mysql.keys()} db_name_use = getattr(conf_res, 'database_name')\ if getattr(conf_res, 'database_name', None) else db_name return mysql_connector.MysqlConnector(conf, db_name_use) @@ -85,16 +88,24 @@ def get_db_connection(db_name): def init_resources(self, cloud_config): resources = self.resources self.resources = dict() - self.rbd_util = rbd_util.RbdUtil(getattr(self.config, "%s" % self.position), self.config.migrate) - self.qemu_img = qemu_img.QemuImg(getattr(self.config, "%s" % self.position), self.config.migrate) - self.ssh_util = ssh_util.SshUtil(getattr(self.config, "%s" % self.position), self.config.migrate) - - identity_conf = self.make_resource_config(self.config, self.position, - cloud_config, 'identity') + self.rbd_util = rbd_util.RbdUtil(getattr(self.config, + "%s" % self.position), + self.config.migrate) + self.qemu_img = qemu_img.QemuImg(getattr(self.config, + "%s" % self.position), + self.config.migrate) + self.ssh_util = ssh_util.SshUtil(getattr(self.config, + "%s" % self.position), + self.config.migrate) + + ident_conf = self.make_resource_config(self.config, + self.position, + cloud_config, + 'identity') self.mysql_connector = self.get_db_method_create_connection('identity', - identity_conf) + ident_conf) identity = resources['identity']( - identity_conf, + ident_conf, self) self.resources['identity'] = identity @@ -107,7 +118,8 @@ def init_resources(self, cloud_config): self.position, cloud_config, resource) - self.mysql_connector = self.get_db_method_create_connection(resource, - resource_config) + self.mysql_connector = \ + self.get_db_method_create_connection(resource, + resource_config) self.resources[resource] = resources[resource]( resource_config, self) diff --git a/cloud/os2os.py b/cloud/os2os.py index 0a93cfa4..ac3fc03e 100644 --- a/cloud/os2os.py +++ b/cloud/os2os.py @@ -15,13 +15,13 @@ import cloud import cloud_ferry -from cloudferrylib.base.action import copy_var, rename_info, merge, is_end_iter, get_info_iter +from cloudferrylib.base.action import copy_var, rename_info, \ + merge, is_end_iter, get_info_iter from cloudferrylib.os.actions import identity_transporter from cloudferrylib.scheduler import scheduler from cloudferrylib.scheduler import namespace from cloudferrylib.scheduler import cursor from cloudferrylib.os.image import glance_image -from cloudferrylib.os.storage import cinder_storage from cloudferrylib.os.network import neutron from cloudferrylib.os.identity import keystone from cloudferrylib.os.compute import nova_compute @@ -120,20 +120,23 @@ def migrate(self, scenario=None): # "rollback" - is cursor that points to tasks must be processed # in case of "migration" failure if not scenario: - process_migration = {"migration": cursor.Cursor(self.process_migrate())} + process_migration = { + "migration": cursor.Cursor(self.process_migrate())} else: scenario.init_tasks(self.init) scenario.load_scenario() - process_migration = {k: cursor.Cursor(v) for k, v in scenario.get_net().items()} - scheduler_migr = scheduler.Scheduler(namespace=namespace_scheduler, **process_migration) + process_migration = {k: cursor.Cursor(v) + for k, v in scenario.get_net().items()} + scheduler_migr = scheduler.Scheduler(namespace=namespace_scheduler, + **process_migration) scheduler_migr.start() def process_migrate(self): check_environment = self.check_environment() task_resources_transporting = self.transport_resources() transport_instances_and_dependency_resources = self.migrate_instances() - return (check_environment >> - task_resources_transporting >> + return (check_environment >> + task_resources_transporting >> transport_instances_and_dependency_resources) def check_environment(self): @@ -165,16 +168,27 @@ def migrate_instances(self): name_result = 'info_result' name_backup = 'info_backup' name_iter = 'info_iter' - save_result = self.save_result(name_data, name_result, name_result, 'instances') + save_result = self.save_result(name_data, + name_result, + name_result, + 'instances') trans_one_inst = self.migrate_process_instance() - init_iteration_instance = self.init_iteration_instance(name_data, name_backup, name_iter) - act_check_needed_compute_resources = check_needed_compute_resources.CheckNeededComputeResources(self.init) + init_iteration_instance = self.init_iteration_instance(name_data, + name_backup, + name_iter) + act_check_needed_compute_resources = \ + check_needed_compute_resources.CheckNeededComputeResources( + self.init) act_get_filter = get_filter.GetFilter(self.init) - act_get_info_inst = get_info_instances.GetInfoInstances(self.init, cloud='src_cloud') + act_get_info_inst = \ + get_info_instances.GetInfoInstances(self.init, + cloud='src_cloud') act_cleanup_images = cleanup_images.CleanupImages(self.init) get_next_instance = get_info_iter.GetInfoIter(self.init) check_for_instance = check_instances.CheckInstances(self.init) - rename_info_iter = rename_info.RenameInfo(self.init, name_result, name_data) + rename_info_iter = rename_info.RenameInfo(self.init, + name_result, + name_data) is_instances = is_end_iter.IsEndIter(self.init) transport_instances_and_dependency_resources = \ @@ -192,12 +206,14 @@ def migrate_instances(self): return transport_instances_and_dependency_resources def init_iteration_instance(self, data, name_backup, name_iter): - init_iteration_instance = copy_var.CopyVar(self.init, data, name_backup, True) >>\ - create_reference.CreateReference(self.init, data, name_iter) + init_iteration_instance = \ + copy_var.CopyVar(self.init, data, name_backup, True) >>\ + create_reference.CreateReference(self.init, data, name_iter) return init_iteration_instance def migration_images(self): - act_get_info_images = get_info_images.GetInfoImages(self.init, cloud='src_cloud') + act_get_info_images = get_info_images.GetInfoImages(self.init, + cloud='src_cloud') act_deploy_images = copy_g2g.CopyFromGlanceToGlance(self.init) return act_get_info_images >> act_deploy_images @@ -206,40 +222,64 @@ def save_result(self, data1, data2, result, resources_name): def transport_volumes_by_instance(self): act_copy_g2g_vols = copy_g2g.CopyFromGlanceToGlance(self.init) - act_convert_c_to_v = convert_compute_to_volume.ConvertComputeToVolume(self.init, cloud='src_cloud') - act_convert_v_to_i = convert_volume_to_image.ConvertVolumeToImage(self.init, cloud='src_cloud') - act_convert_i_to_v = convert_image_to_volume.ConvertImageToVolume(self.init, cloud='dst_cloud') - act_convert_v_to_c = convert_volume_to_compute.ConvertVolumeToCompute(self.init, cloud='dst_cloud') + act_convert_c_to_v = \ + convert_compute_to_volume.ConvertComputeToVolume(self.init, + cloud='src_cloud') + act_convert_v_to_i = \ + convert_volume_to_image.ConvertVolumeToImage(self.init, + cloud='src_cloud') + act_convert_i_to_v = \ + convert_image_to_volume.ConvertImageToVolume(self.init, + cloud='dst_cloud') + act_convert_v_to_c = \ + convert_volume_to_compute.ConvertVolumeToCompute(self.init, + cloud='dst_cloud') task_convert_c_to_v_to_i = act_convert_c_to_v >> act_convert_v_to_i task_convert_i_to_v_to_c = act_convert_i_to_v >> act_convert_v_to_c - return task_convert_c_to_v_to_i >> act_copy_g2g_vols >> task_convert_i_to_v_to_c + return task_convert_c_to_v_to_i >> act_copy_g2g_vols >>\ + task_convert_i_to_v_to_c def transport_volumes_by_instance_via_ssh(self): - act_convert_c_to_v = convert_compute_to_volume.ConvertComputeToVolume(self.init, cloud='src_cloud') - act_rename_inst_vol_src = create_reference.CreateReference(self.init, 'storage_info', - 'src_storage_info') - act_convert_v_to_c = convert_volume_to_compute.ConvertVolumeToCompute(self.init, cloud='dst_cloud') - act_rename_inst_vol_dst = create_reference.CreateReference(self.init, 'storage_info', - 'dst_storage_info') - act_inst_vol_data_map = prepare_volumes_data_map.PrepareVolumesDataMap(self.init, - 'src_storage_info', - 'dst_storage_info') - act_deploy_inst_volumes = deploy_volumes.DeployVolumes(self.init, cloud='dst_cloud') - - act_inst_vol_transport_data = task_transfer.TaskTransfer(self.init, - 'SSHCephToCeph', - input_info='storage_info') - - act_deploy_snapshots = deploy_snapshots.DeployVolSnapshots(self.init, cloud='dst_cloud') - act_convert_v_to_c + act_convert_c_to_v = \ + convert_compute_to_volume.ConvertComputeToVolume(self.init, + cloud='src_cloud') + act_rename_inst_vol_src = \ + create_reference.CreateReference(self.init, + 'storage_info', + 'src_storage_info') + act_convert_v_to_c = \ + convert_volume_to_compute.ConvertVolumeToCompute(self.init, + cloud='dst_cloud') + act_rename_inst_vol_dst = \ + create_reference.CreateReference(self.init, 'storage_info', + 'dst_storage_info') + act_inst_vol_data_map = \ + prepare_volumes_data_map.PrepareVolumesDataMap(self.init, + 'src_storage_info', + 'dst_storage_info') + act_deploy_inst_volumes = \ + deploy_volumes.DeployVolumes(self.init, cloud='dst_cloud') + + act_inst_vol_transport_data = \ + task_transfer.TaskTransfer(self.init, + 'SSHCephToCeph', + input_info='storage_info') + + act_deploy_snapshots = \ + deploy_snapshots.DeployVolSnapshots( + self.init, + cloud='dst_cloud') - act_convert_v_to_c is_snapshots = is_option.IsOption(self.init, 'keep_volume_snapshots') task_get_inst_vol_info = act_convert_c_to_v >> act_rename_inst_vol_src - task_deploy_inst_vol = act_deploy_inst_volumes >> act_rename_inst_vol_dst + task_deploy_inst_vol = act_deploy_inst_volumes >> \ + act_rename_inst_vol_dst return task_get_inst_vol_info >> \ - task_deploy_inst_vol >> act_inst_vol_data_map >> \ - (is_snapshots | act_deploy_snapshots | act_inst_vol_transport_data) >> \ - act_convert_v_to_c + task_deploy_inst_vol >> act_inst_vol_data_map >> \ + (is_snapshots | act_deploy_snapshots | + act_inst_vol_transport_data) >> \ + act_convert_v_to_c def transport_available_volumes_via_ssh(self): is_volume_snapshots = is_option.IsOption(self.init, @@ -247,68 +287,87 @@ def transport_available_volumes_via_ssh(self): final_action = fake_action.FakeAction(self.init) - act_get_info_available_volumes = get_info_volumes.GetInfoVolumes(self.init, - cloud='src_cloud', - search_opts={'status': 'available'}) - act_rename_vol_src = create_reference.CreateReference(self.init, - 'storage_info', - 'src_storage_info') - task_get_available_vol_info = act_get_info_available_volumes >> act_rename_vol_src + act_get_info_available_volumes = \ + get_info_volumes.GetInfoVolumes(self.init, + cloud='src_cloud', + search_opts={ + 'status': 'available'}) + act_rename_vol_src = \ + create_reference.CreateReference(self.init, + 'storage_info', + 'src_storage_info') + task_get_available_vol_info = \ + act_get_info_available_volumes >> act_rename_vol_src act_deploy_vol = deploy_volumes.DeployVolumes(self.init, cloud='dst_cloud') - act_rename_vol_dst = create_reference.CreateReference(self.init, - 'storage_info', - 'dst_storage_info') + act_rename_vol_dst = \ + create_reference.CreateReference(self.init, + 'storage_info', + 'dst_storage_info') task_deploy_available_volumes = act_deploy_vol >> act_rename_vol_dst - act_vol_data_map = prepare_volumes_data_map.PrepareVolumesDataMap(self.init, - 'src_storage_info', - 'dst_storage_info') + act_vol_data_map = \ + prepare_volumes_data_map.PrepareVolumesDataMap(self.init, + 'src_storage_info', + 'dst_storage_info') act_vol_transport_data = \ - task_transfer.TaskTransfer(self.init, - 'SSHCephToCeph', - input_info='storage_info') - final_action + task_transfer.TaskTransfer( + self.init, + 'SSHCephToCeph', + input_info='storage_info') - final_action act_deploy_vol_snapshots = \ - deploy_snapshots.DeployVolSnapshots(self.init,cloud='dst_cloud') - final_action + deploy_snapshots.DeployVolSnapshots( + self.init, cloud='dst_cloud') - final_action return task_get_available_vol_info >> \ - task_deploy_available_volumes >> \ - act_vol_data_map >> \ - (is_volume_snapshots | act_deploy_vol_snapshots | act_vol_transport_data) \ - >> final_action + task_deploy_available_volumes >> \ + act_vol_data_map >> \ + (is_volume_snapshots | act_deploy_vol_snapshots | + act_vol_transport_data) \ + >> final_action def transport_object_storage(self): - act_get_objects_info = get_info_objects.GetInfoObjects(self.init, - cloud='src_cloud') - act_transfer_objects = copy_object2object.CopyFromObjectToObject(self.init, - src_cloud='src_cloud', - dst_cloud='dst_cloud') + act_get_objects_info = \ + get_info_objects.GetInfoObjects(self.init, + cloud='src_cloud') + act_transfer_objects = \ + copy_object2object.CopyFromObjectToObject(self.init, + src_cloud='src_cloud', + dst_cloud='dst_cloud') task_transfer_objects = act_get_objects_info >> act_transfer_objects return task_transfer_objects def transport_cold_data(self): - act_identity_trans = identity_transporter.IdentityTransporter(self.init) - task_transport_available_volumes = self.transport_available_volumes_via_ssh() + act_identity_trans = \ + identity_transporter.IdentityTransporter(self.init) + task_transport_available_volumes = \ + self.transport_available_volumes_via_ssh() task_transport_objects = self.transport_object_storage() task_transport_images = self.migration_images() return act_identity_trans >> \ - task_transport_available_volumes \ - >> task_transport_objects \ - >> task_transport_images + task_transport_available_volumes \ + >> task_transport_objects \ + >> task_transport_images def transport_resources(self): - act_identity_trans = identity_transporter.IdentityTransporter(self.init) + act_identity_trans = \ + identity_transporter.IdentityTransporter(self.init) task_images_trans = self.migration_images() - act_comp_res_trans = transport_compute_resources.TransportComputeResources(self.init) + act_comp_res_trans = \ + transport_compute_resources.TransportComputeResources(self.init) act_network_trans = networks_transporter.NetworkTransporter(self.init) - return act_identity_trans >> task_images_trans >> act_network_trans >> act_comp_res_trans + return act_identity_trans >> task_images_trans >> act_network_trans >>\ + act_comp_res_trans def migrate_images_by_instances(self): - act_conv_comp_img = convert_compute_to_image.ConvertComputeToImage(self.init, cloud='src_cloud') - act_conv_image_comp = convert_image_to_compute.ConvertImageToCompute(self.init) + act_conv_comp_img = \ + convert_compute_to_image.ConvertComputeToImage(self.init, + cloud='src_cloud') + act_conv_image_comp = \ + convert_image_to_compute.ConvertImageToCompute(self.init) act_copy_inst_images = copy_g2g.CopyFromGlanceToGlance(self.init) return act_conv_comp_img >> act_copy_inst_images >> act_conv_image_comp @@ -324,20 +383,37 @@ def migrate_resources_by_instance_via_ssh(self): def migrate_instance(self): act_map_com_info = map_compute_info.MapComputeInfo(self.init) - act_net_prep = prepare_networks.PrepareNetworks(self.init, cloud='dst_cloud') + act_net_prep = prepare_networks.PrepareNetworks(self.init, + cloud='dst_cloud') act_deploy_instances = transport_instance.TransportInstance(self.init) - act_i_to_f = load_compute_image_to_file.LoadComputeImageToFile(self.init, cloud='dst_cloud') - act_merge = merge_base_and_diff.MergeBaseDiff(self.init, cloud='dst_cloud') - act_convert_image = convert_file.ConvertFile(self.init, cloud='dst_cloud') - act_f_to_i = upload_file_to_image.UploadFileToImage(self.init, cloud='dst_cloud') - act_transfer_file = task_transfer.TaskTransfer(self.init, 'SSHCephToFile', - resource_name=utl.INSTANCES_TYPE, - resource_root_name=utl.DIFF_BODY) - act_f_to_i_after_transfer = upload_file_to_image.UploadFileToImage(self.init, cloud='dst_cloud') - act_is_not_trans_image = is_not_transport_image.IsNotTransportImage(self.init, cloud='src_cloud') - act_is_not_merge_diff = is_not_merge_diff.IsNotMergeDiff(self.init, cloud='src_cloud') - act_post_transport_instance = post_transport_instance.PostTransportInstance(self.init, cloud='dst_cloud') - act_transport_ephemeral = transport_ephemeral.TransportEphemeral(self.init, cloud='dst_cloud') + act_i_to_f = \ + load_compute_image_to_file.LoadComputeImageToFile( + self.init, + cloud='dst_cloud') + act_merge = \ + merge_base_and_diff.MergeBaseDiff(self.init, cloud='dst_cloud') + act_convert_image = convert_file.ConvertFile(self.init, + cloud='dst_cloud') + act_f_to_i = upload_file_to_image.UploadFileToImage(self.init, + cloud='dst_cloud') + act_transfer_file = \ + task_transfer.TaskTransfer(self.init, + 'SSHCephToFile', + resource_name=utl.INSTANCES_TYPE, + resource_root_name=utl.DIFF_BODY) + act_f_to_i_after_transfer = \ + upload_file_to_image.UploadFileToImage(self.init, + cloud='dst_cloud') + act_is_not_trans_image = \ + is_not_transport_image.IsNotTransportImage(self.init, + cloud='src_cloud') + act_is_not_merge_diff = \ + is_not_merge_diff.IsNotMergeDiff(self.init, cloud='src_cloud') + post_transport_instance.PostTransportInstance(self.init, + cloud='dst_cloud') + act_transport_ephemeral = \ + transport_ephemeral.TransportEphemeral(self.init, + cloud='dst_cloud') trans_file_to_file = task_transfer.TaskTransfer( self.init, 'SSHFileToFile', @@ -348,31 +424,44 @@ def migrate_instance(self): 'SSHFileToFile', resource_name=utl.INSTANCES_TYPE, resource_root_name=utl.DIFF_BODY) - act_is_not_copy_diff_file = is_not_copy_diff_file.IsNotCopyDiffFile(self.init) - process_merge_diff_and_base = act_i_to_f >> trans_file_to_file >> act_merge >> act_convert_image >> act_f_to_i + act_is_not_copy_diff_file = \ + is_not_copy_diff_file.IsNotCopyDiffFile(self.init) + process_merge_diff_and_base = act_i_to_f >> trans_file_to_file >> \ + act_merge >> act_convert_image >> act_f_to_i process_merge_diff_and_base = act_i_to_f - process_transport_image = act_transfer_file >> act_f_to_i_after_transfer + process_transport_image = act_transfer_file >> \ + act_f_to_i_after_transfer process_transport_image = act_transfer_file - act_pre_transport_instance = (act_is_not_trans_image | act_is_not_merge_diff) >> \ - process_transport_image >> \ - (act_is_not_merge_diff | act_deploy_instances) >> \ - process_merge_diff_and_base + act_pre_transport_instance = (act_is_not_trans_image | + act_is_not_merge_diff) >> \ + process_transport_image >> \ + (act_is_not_merge_diff | + act_deploy_instances) >> \ + process_merge_diff_and_base act_post_transport_instance = (act_is_not_copy_diff_file | - act_transport_ephemeral) >> act_trans_diff_file + act_transport_ephemeral) >> \ + act_trans_diff_file return act_net_prep >> \ - act_map_com_info >> \ - act_pre_transport_instance >> \ - act_deploy_instances >> \ - act_post_transport_instance >> \ - act_transport_ephemeral + act_map_com_info >> \ + act_pre_transport_instance >> \ + act_deploy_instances >> \ + act_post_transport_instance >> \ + act_transport_ephemeral def migrate_process_instance(self): - act_attaching = attach_used_volumes_via_compute.AttachVolumesCompute(self.init, cloud='dst_cloud') + act_attaching = \ + attach_used_volumes_via_compute.AttachVolumesCompute( + self.init, + cloud='dst_cloud') act_stop_vms = stop_vm.StopVms(self.init, cloud='src_cloud') act_start_vms = start_vm.StartVms(self.init, cloud='dst_cloud') - #transport_resource_inst = self.migrate_resources_by_instance_via_ssh() + # transport_resource_inst = \ + # self.migrate_resources_by_instance_via_ssh() transport_resource_inst = self.migrate_resources_by_instance() transport_inst = self.migrate_instance() - act_dissociate_floatingip = instance_floatingip_actions.DisassociateAllFloatingips(self.init, cloud='src_cloud') + act_dissociate_floatingip = \ + instance_floatingip_actions.DisassociateAllFloatingips( + self.init, + cloud='src_cloud') return act_stop_vms >> transport_resource_inst >> transport_inst >> \ - act_attaching >> act_dissociate_floatingip >> act_start_vms + act_attaching >> act_dissociate_floatingip >> act_start_vms diff --git a/cloudferrylib/__init__.py b/cloudferrylib/__init__.py index 2f3634b0..e69de29b 100644 --- a/cloudferrylib/__init__.py +++ b/cloudferrylib/__init__.py @@ -1,3 +0,0 @@ -import base -import scheduler -import os diff --git a/cloudferrylib/base/__init__.py b/cloudferrylib/base/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/base/__init__.py +++ b/cloudferrylib/base/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/base/action/copy_var.py b/cloudferrylib/base/action/copy_var.py index d8262b73..0254ccd2 100644 --- a/cloudferrylib/base/action/copy_var.py +++ b/cloudferrylib/base/action/copy_var.py @@ -33,4 +33,3 @@ def run(self, **kwargs): return { self.info_name: new_obj } - diff --git a/cloudferrylib/base/action/create_reference.py b/cloudferrylib/base/action/create_reference.py index b42ab70c..0de9535e 100644 --- a/cloudferrylib/base/action/create_reference.py +++ b/cloudferrylib/base/action/create_reference.py @@ -14,7 +14,6 @@ from cloudferrylib.base.action import action -import copy class CreateReference(action.Action): @@ -28,4 +27,3 @@ def run(self, **kwargs): return { self.info_name: kwargs[self.original_info_name] } - diff --git a/cloudferrylib/base/action/get_info_iter.py b/cloudferrylib/base/action/get_info_iter.py index d7689c20..daf86e0b 100644 --- a/cloudferrylib/base/action/get_info_iter.py +++ b/cloudferrylib/base/action/get_info_iter.py @@ -15,12 +15,12 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils as utl -import copy class GetInfoIter(action.Action): - def __init__(self, init, iter_info_name='info_iter', info_name='info', resource_name=utl.INSTANCES_TYPE): + def __init__(self, init, iter_info_name='info_iter', info_name='info', + resource_name=utl.INSTANCES_TYPE): self.iter_info_name = iter_info_name self.info_name = info_name self.resource_name = resource_name @@ -39,7 +39,3 @@ def run(self, **kwargs): self.iter_info_name: info, self.info_name: new_info } - - - - diff --git a/cloudferrylib/base/action/is_end_iter.py b/cloudferrylib/base/action/is_end_iter.py index 696f166b..6fa01694 100644 --- a/cloudferrylib/base/action/is_end_iter.py +++ b/cloudferrylib/base/action/is_end_iter.py @@ -33,8 +33,3 @@ def run(self, **kwargs): else: self.num_element = 0 return {} - - - - - diff --git a/cloudferrylib/base/action/is_option.py b/cloudferrylib/base/action/is_option.py index b29a7714..38a90cdf 100644 --- a/cloudferrylib/base/action/is_option.py +++ b/cloudferrylib/base/action/is_option.py @@ -19,6 +19,7 @@ PATH_ONE = 1 PATH_TWO = 2 + class IsOption(action.Action): def __init__(self, init, option_name): @@ -26,16 +27,10 @@ def __init__(self, init, option_name): super(IsOption, self).__init__(init) def run(self, **kwargs): - self.set_next_path(DEFAULT) # DEFAULT PATH + self.set_next_path(DEFAULT) # DEFAULT PATH option_value = self.cfg.migrate[self.option_name] if option_value: self.set_next_path(PATH_ONE) else: self.set_next_path(PATH_TWO) return {} - - - - - - diff --git a/cloudferrylib/base/action/merge.py b/cloudferrylib/base/action/merge.py index 060aaee5..4bf9338b 100644 --- a/cloudferrylib/base/action/merge.py +++ b/cloudferrylib/base/action/merge.py @@ -35,4 +35,3 @@ def run(self, **kwargs): return { self.result: data2 } - diff --git a/cloudferrylib/base/action/rename_info.py b/cloudferrylib/base/action/rename_info.py index f6a3bde7..4bd568b3 100644 --- a/cloudferrylib/base/action/rename_info.py +++ b/cloudferrylib/base/action/rename_info.py @@ -14,7 +14,6 @@ from cloudferrylib.base.action import action -import copy class RenameInfo(action.Action): @@ -29,4 +28,3 @@ def run(self, **kwargs): self.original_info_name: None, self.info_name: kwargs[self.original_info_name] } - diff --git a/cloudferrylib/base/network.py b/cloudferrylib/base/network.py index 9a82ad2b..0f4a91a7 100644 --- a/cloudferrylib/base/network.py +++ b/cloudferrylib/base/network.py @@ -38,4 +38,3 @@ def check_existing_port(self, network_id, mac): def get_security_groups(self): raise NotImplemented("it's base class") - diff --git a/cloudferrylib/base/objstorage.py b/cloudferrylib/base/objstorage.py index 3d7e474b..05c8ac8e 100644 --- a/cloudferrylib/base/objstorage.py +++ b/cloudferrylib/base/objstorage.py @@ -19,4 +19,3 @@ class ObjStorage(resource.Resource): def __init__(self): super(ObjStorage, self).__init__() - diff --git a/cloudferrylib/base/resource.py b/cloudferrylib/base/resource.py index 2e3718af..be408a1c 100644 --- a/cloudferrylib/base/resource.py +++ b/cloudferrylib/base/resource.py @@ -49,8 +49,9 @@ def wait_for_status(self, res_id, get_status, wait_status, timeout=60): time.sleep(delay) delay *= 2 else: - raise timeout_exception.TimeoutException(get_status(res_id).lower(), - wait_status, "Timeout exp") + raise timeout_exception.TimeoutException( + get_status(res_id).lower(), + wait_status, "Timeout exp") def try_wait_for_status(self, res_id, get_status, wait_status, timeout=60): try: diff --git a/cloudferrylib/os/__init__.py b/cloudferrylib/os/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/os/__init__.py +++ b/cloudferrylib/os/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/os/actions/attach_used_volumes.py b/cloudferrylib/os/actions/attach_used_volumes.py index 2ebde35f..36eec0f1 100644 --- a/cloudferrylib/os/actions/attach_used_volumes.py +++ b/cloudferrylib/os/actions/attach_used_volumes.py @@ -23,4 +23,4 @@ def run(self, storage_info={}, **kwargs): resource_storage = self.cloud.resources[utl.STORAGE_RESOURCE] for vol in storage_info[utl.VOLUMES_TYPE].itervalues(): resource_storage.attach_volume_to_instance(vol) - return {} \ No newline at end of file + return {} diff --git a/cloudferrylib/os/actions/attach_used_volumes_via_compute.py b/cloudferrylib/os/actions/attach_used_volumes_via_compute.py index 96e2dd68..f5f3416a 100644 --- a/cloudferrylib/os/actions/attach_used_volumes_via_compute.py +++ b/cloudferrylib/os/actions/attach_used_volumes_via_compute.py @@ -22,16 +22,16 @@ class AttachVolumesCompute(action.Action): def run(self, info, **kwargs): info = copy.deepcopy(info) - compute_resource = self.cloud.resources[utl.COMPUTE_RESOURCE] - storage_resource = self.cloud.resources[utl.STORAGE_RESOURCE] + compute_res = self.cloud.resources[utl.COMPUTE_RESOURCE] + storage_res = self.cloud.resources[utl.STORAGE_RESOURCE] for instance in info[utl.INSTANCES_TYPE].itervalues(): if not instance[utl.META_INFO].get(utl.VOLUME_BODY): continue for vol in instance[utl.META_INFO][utl.VOLUME_BODY]: - if storage_resource.get_status( + if storage_res.get_status( vol['volume']['id']) != 'in-use': - compute_resource.attach_volume_to_instance(instance, vol) - storage_resource.wait_for_status(vol['volume']['id'], - storage_resource.get_status, - 'in-use') + compute_res.attach_volume_to_instance(instance, vol) + storage_res.wait_for_status(vol['volume']['id'], + storage_res.get_status, + 'in-use') return {} diff --git a/cloudferrylib/os/actions/check_bandwidth.py b/cloudferrylib/os/actions/check_bandwidth.py index eb7e5c3d..126cc0fa 100644 --- a/cloudferrylib/os/actions/check_bandwidth.py +++ b/cloudferrylib/os/actions/check_bandwidth.py @@ -64,8 +64,11 @@ def run(self, **kwargs): with files.RemoteDir(runner, temp_dir_name): try: with utils.forward_agent(env.key_filename): - dd_command = cmd_cfg.dd_full('/dev/zero', remote_file_path, 1, - 0, test_file_size) + dd_command = cmd_cfg.dd_full('/dev/zero', + remote_file_path, + 1, + 0, + test_file_size) self.cloud.ssh_util.execute(dd_command) LOG.info("Checking upload speed... Wait please.") diff --git a/cloudferrylib/os/actions/check_cloud.py b/cloudferrylib/os/actions/check_cloud.py index 9773f65f..caa4fdc3 100644 --- a/cloudferrylib/os/actions/check_cloud.py +++ b/cloudferrylib/os/actions/check_cloud.py @@ -14,7 +14,7 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils as utl -from keystoneclient.openstack.common.apiclient import exceptions as keystone_exc +from keystoneclient.openstack.common.apiclient import exceptions as ks_exc from novaclient import exceptions as nova_exc from cloudferrylib.base import exception @@ -32,8 +32,8 @@ def run(self, **kwargs): compute_resource = self.cloud.resources[utl.COMPUTE_RESOURCE] volume_resource = self.cloud.resources[utl.STORAGE_RESOURCE] net_resource = self.cloud.resources[utl.NETWORK_RESOURCE] - admin_tenant_name = self.cloud.cloud_config.cloud.tenant - admin_tenant_id = ident_resource.get_tenant_id_by_name(admin_tenant_name) + adm_tenant_name = self.cloud.cloud_config.cloud.tenant + adm_tenant_id = ident_resource.get_tenant_id_by_name(adm_tenant_name) tenant_name = 'test_name' flavor_id = 'c0c0c0c0' err_message = 'Failed to create object in the cloud' @@ -65,8 +65,7 @@ def run(self, **kwargs): ident_resource._deploy_tenants(tenant) tenant_id = ident_resource.get_tenant_id_by_name(tenant_name) compute_resource._deploy_flavors(flavor, None) - except (keystone_exc.ClientException, - nova_exc.ClientException) as e: + except (ks_exc.ClientException, nova_exc.ClientException): LOG.error(err_message) raise exception.AbortMigrationError(err_message) migrate_image = image_res.create_image( @@ -75,7 +74,7 @@ def run(self, **kwargs): disk_format='qcow2', is_public=True, protected=False, - owner=admin_tenant_id, + owner=adm_tenant_id, size=4, properties={'user_name': 'test_user_name'}, data='test' @@ -106,7 +105,7 @@ def run(self, **kwargs): } } - subnet = net_resource.neutron_client.create_subnet(subnet_info) + net_resource.neutron_client.create_subnet(subnet_info) nics = [{'net-id': private_network_id}] @@ -123,7 +122,7 @@ def run(self, **kwargs): 'user_id': '1', 'boot_mode': utl.BOOT_FROM_IMAGE, 'availability_zone': 'nova', - 'tenant_name': admin_tenant_name + 'tenant_name': adm_tenant_name } } }, @@ -146,7 +145,7 @@ def run(self, **kwargs): volume_resource.cinder_client.volumes.delete(vol_new_ids.keys()[0]) network_info = { 'network': { - 'tenant_id': admin_tenant_id, + 'tenant_id': adm_tenant_id, 'admin_state_up': True, 'shared': True, 'name': 'test_net', @@ -157,7 +156,10 @@ def run(self, **kwargs): network_info)['network']['id'] net_resource.neutron_client.delete_network(new_net_id) vm_new_ids = compute_resource._deploy_instances(info) - if not vm_new_ids or not vol_new_ids or not migrate_image or not new_net_id: + if (not vm_new_ids or + not vol_new_ids or + not migrate_image or + not new_net_id): LOG.error(err_message) raise exception.AbortMigrationError(err_message) compute_resource.nova_client.servers.delete(vm_new_ids.keys()[0]) diff --git a/cloudferrylib/os/actions/check_filter.py b/cloudferrylib/os/actions/check_filter.py index 20d8afa7..9e9007ab 100644 --- a/cloudferrylib/os/actions/check_filter.py +++ b/cloudferrylib/os/actions/check_filter.py @@ -24,8 +24,8 @@ class CheckFilter(action.Action): def run(self, **kwargs): - """Check filter file and make sure all entries are present in source cloud. - + """Check filter file and make sure all entries are present in source + cloud. """ search_opts = kwargs.get('search_opts', {}) search_opts_img = kwargs.get('search_opts_img', {}) @@ -38,10 +38,13 @@ def run(self, **kwargs): try: img = image_resource.glance_client.images.get(img_id) if img: - LOG.debug('Filter config check: Image ID {} is OK'.format(img_id)) + LOG.debug('Filter config check: Image ID {} is OK' + .format(img_id)) except glance_exc.HTTPNotFound as e: - LOG.error('Filter config check: Image ID {} is not present in source cloud, ' - 'please update your filter config. Aborting.'.format(img_id)) + LOG.error('Filter config check: Image ID {} ' + 'is not present in source cloud, ' + 'please update your filter config. Aborting.' + .format(img_id)) raise e ident_resource = self.cloud.resources[utl.IDENTITY_RESOURCE] if search_opts_tenant and search_opts_tenant.get('tenant_id'): @@ -49,12 +52,16 @@ def run(self, **kwargs): for tenant_id in tenants: LOG.debug('Filtered tenant id: {}'.format(tenant_id)) try: - tenant = ident_resource.keystone_client.tenants.find(id=tenant_id) + tenant = ident_resource.keystone_client.tenants.find( + id=tenant_id) if tenant: - LOG.debug('Filter config check: Tenant ID {} is OK'.format(tenant_id)) + LOG.debug('Filter config check: Tenant ID {} is OK' + .format(tenant_id)) except keystone_exc.NotFound as e: - LOG.error('Filter config check: Tenant ID {} is not present in source cloud, ' - 'please update your filter config. Aborting.'.format(tenant_id)) + LOG.error('Filter config check: Tenant ID {} ' + 'is not present in source cloud, ' + 'please update your filter config. Aborting.' + .format(tenant_id)) raise e compute_resource = self.cloud.resources[utl.COMPUTE_RESOURCE] if search_opts and search_opts.get('id'): @@ -62,10 +69,14 @@ def run(self, **kwargs): for instance_id in instances: LOG.debug('Filtered instance id: {}'.format(instance_id)) try: - instance = compute_resource.nova_client.servers.get(instance_id) + instance = \ + compute_resource.nova_client.servers.get(instance_id) if instance: - LOG.debug('Filter config check: Instance ID {} is OK'.format(instance_id)) + LOG.debug('Filter config check: Instance ID {} is OK' + .format(instance_id)) except nova_exc.NotFound as e: - LOG.error('Filter config check: Instance ID {} is not present in source cloud, ' - 'please update your filter config. Aborting.'.format(instance_id)) + LOG.error('Filter config check: Instance ID {} ' + 'is not present in source cloud, ' + 'please update your filter config. Aborting.' + .format(instance_id)) raise e diff --git a/cloudferrylib/os/actions/check_instance_networks.py b/cloudferrylib/os/actions/check_instance_networks.py index b9beaaab..cf991ccf 100644 --- a/cloudferrylib/os/actions/check_instance_networks.py +++ b/cloudferrylib/os/actions/check_instance_networks.py @@ -49,6 +49,7 @@ def run(self, **kwargs): instance_names.append(compute_info['instance']['name']) if len(instance_names) > 0: - raise exceptions.EnvironmentError("Instances %s spawned " + raise exceptions.EnvironmentError( + "Instances %s spawned " "in external network directly. CloudFerry can't " "migrate it." % ", ".join(instance_names)) diff --git a/cloudferrylib/os/actions/check_needed_compute_resources.py b/cloudferrylib/os/actions/check_needed_compute_resources.py index cdd3d83d..09cc22d8 100644 --- a/cloudferrylib/os/actions/check_needed_compute_resources.py +++ b/cloudferrylib/os/actions/check_needed_compute_resources.py @@ -61,7 +61,8 @@ def check_in_use_flavor(self, objs, info): # is in this flavor, it will keep the old flavor id, can not be matched # to existing flavors src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] - src_flavor_ids = [flavor.id for flavor in src_compute.get_flavor_list()] + src_flavor_ids = \ + [flavor.id for flavor in src_compute.get_flavor_list()] dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] dst_flavors = dst_compute.get_flavor_list() dst_flavor_ids = [flavor.id for flavor in dst_flavors] @@ -82,5 +83,6 @@ def check_in_use_flavor(self, objs, info): ram=flav_details['memory_mb'], vcpus=flav_details['vcpus'], disk=flav_details['root_gb'], - ephemeral=flav_details['ephemeral_gb']) + ephemeral=flav_details[ + 'ephemeral_gb']) src_flavor_ids.append(_instance['flavor_id']) diff --git a/cloudferrylib/os/actions/check_networks.py b/cloudferrylib/os/actions/check_networks.py index 56e4272f..066c202c 100644 --- a/cloudferrylib/os/actions/check_networks.py +++ b/cloudferrylib/os/actions/check_networks.py @@ -82,8 +82,8 @@ def check_network_overlapping(self, network): if overlapping_subnet: message = ("Subnet %s in network %s on src overlap " "subnet %s in network %s on dst" % ( - overlapping_subnet, self.id, - subnet['id'], network.id)) + overlapping_subnet, self.id, + subnet['id'], network.id)) LOG.error(message) raise exceptions.EnvironmentError(message) diff --git a/cloudferrylib/os/actions/check_openstack_apis.py b/cloudferrylib/os/actions/check_openstack_apis.py index 852d5a1c..ffa35b76 100644 --- a/cloudferrylib/os/actions/check_openstack_apis.py +++ b/cloudferrylib/os/actions/check_openstack_apis.py @@ -17,14 +17,18 @@ from cloudferrylib.utils import utils as utl from neutronclient.common import exceptions as neutron_exc from glanceclient import exc as glance_exc -from keystoneclient.openstack.common.apiclient import exceptions as keystone_exc +from keystoneclient.openstack.common.apiclient import exceptions as ks_exc from cinderclient import exceptions as cinder_exc from novaclient import exceptions as nova_exc LOG = utl.get_log(__name__) -def check(os_api_call, os_api_type, position, *os_api_call_args, **os_api_call_kwargs): +def check(os_api_call, + os_api_type, + position, + *os_api_call_args, + **os_api_call_kwargs): try: LOG.info("Checking %s APIs availability on %s.", os_api_type, position.upper()) @@ -32,7 +36,7 @@ def check(os_api_call, os_api_type, position, *os_api_call_args, **os_api_call_k except (neutron_exc.NeutronException, glance_exc.BaseException, glance_exc.ClientException, - keystone_exc.ClientException, + ks_exc.ClientException, cinder_exc.ClientException, nova_exc.ClientException) as e: message = ('{os_api_type} APIs on {position} check failed with: ' @@ -76,13 +80,15 @@ def run(self, **kwargs): storage_resource = self.cloud.resources[utl.STORAGE_RESOURCE] check(storage_resource.cinder_client.volumes.list, 'Cinder volumes', self.cloud.position) - check(storage_resource.cinder_client.volume_snapshots.list, 'Cinder snapshots', + check(storage_resource.cinder_client.volume_snapshots.list, + 'Cinder snapshots', self.cloud.position) class CheckNetworkingAPIs(action.Action): def run(self, **kwargs): - """Check networking backend by getting network/subnets/routers lists.""" + """Check networking backend + by getting network/subnets/routers lists.""" neutron = self.cloud.resources[utl.NETWORK_RESOURCE] check(neutron.neutron_client.list_networks, 'Neutron networks', self.cloud.position) diff --git a/cloudferrylib/os/actions/cinder_database_manipulation.py b/cloudferrylib/os/actions/cinder_database_manipulation.py index 1ce47587..a238c945 100644 --- a/cloudferrylib/os/actions/cinder_database_manipulation.py +++ b/cloudferrylib/os/actions/cinder_database_manipulation.py @@ -41,8 +41,8 @@ class GetVolumesDb(CinderDatabaseInteraction): def run(self, *args, **kwargs): search_opts = kwargs.get('search_opts_tenant', {}) - return {NAMESPACE_CINDER_CONST: \ - self.get_resource().read_db_info(**search_opts)} + return {NAMESPACE_CINDER_CONST: + self.get_resource().read_db_info(**search_opts)} class WriteVolumesDb(CinderDatabaseInteraction): diff --git a/cloudferrylib/os/actions/cleanup_images.py b/cloudferrylib/os/actions/cleanup_images.py index ed2a624f..7c28c211 100644 --- a/cloudferrylib/os/actions/cleanup_images.py +++ b/cloudferrylib/os/actions/cleanup_images.py @@ -37,7 +37,8 @@ def run(self, info, **kwargs): for vol in volumes: if utl.IMAGE_BODY in vol[utl.META_INFO]: - image_checksum = vol[utl.META_INFO][utl.IMAGE_BODY]['checksum'] + image_checksum = \ + vol[utl.META_INFO][utl.IMAGE_BODY]['checksum'] if image_checksum not in checksum_list: checksum_list.append(image_checksum) diff --git a/cloudferrylib/os/actions/convert_compute_to_image.py b/cloudferrylib/os/actions/convert_compute_to_image.py index 6894dc1c..e5d748e0 100644 --- a/cloudferrylib/os/actions/convert_compute_to_image.py +++ b/cloudferrylib/os/actions/convert_compute_to_image.py @@ -71,7 +71,8 @@ def run(self, info=None, **kwargs): images_body[image_id][utl.META_INFO][ utl.INSTANCE_BODY] = [instance] else: - LOG.warning("No boot image for instance, need to re-create it") + LOG.warning("No boot image for instance, " + "need to re-create it") missing_images[instance_id] = image_id else: compute_ignored_images[instance_id] = instance diff --git a/cloudferrylib/os/actions/convert_compute_to_volume.py b/cloudferrylib/os/actions/convert_compute_to_volume.py index 9d0b21d8..5bd5cc79 100644 --- a/cloudferrylib/os/actions/convert_compute_to_volume.py +++ b/cloudferrylib/os/actions/convert_compute_to_volume.py @@ -22,11 +22,11 @@ class ConvertComputeToVolume(action.Action): def run(self, info=None, **kwargs): - compute_info = copy.deepcopy(info) + info = copy.deepcopy(info) storage_info = {utl.VOLUMES_TYPE: {}} ignored = {} resource_storage = self.cloud.resources[utl.STORAGE_RESOURCE] - for instance_id, instance in compute_info[utl.INSTANCES_TYPE].iteritems(): + for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): volumes_exists = True if not instance[utl.INSTANCE_BODY]['volumes']: if 'volume' in instance['meta']: diff --git a/cloudferrylib/os/actions/convert_file.py b/cloudferrylib/os/actions/convert_file.py index f4adbce9..0eea833c 100644 --- a/cloudferrylib/os/actions/convert_file.py +++ b/cloudferrylib/os/actions/convert_file.py @@ -13,7 +13,8 @@ def run(self, info=None, **kwargs): if image_res.config.image.convert_to_raw: return {} for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): - image_id = info[INSTANCES][instance_id][utl.INSTANCE_BODY]['image_id'] + image_id = \ + info[INSTANCES][instance_id][utl.INSTANCE_BODY]['image_id'] images = image_res.read_info(image_id=image_id) image = images[utl.IMAGES_TYPE][image_id] disk_format = image[utl.IMAGE_BODY]['disk_format'] @@ -29,4 +30,4 @@ def convert_file_to_raw(host, disk_format, filepath): with forward_agent(env.key_filename): run("qemu-img convert -f %s -O raw %s %s.tmp" % (disk_format, filepath, filepath)) - run("mv -f %s.tmp %s" % (filepath, filepath)) \ No newline at end of file + run("mv -f %s.tmp %s" % (filepath, filepath)) diff --git a/cloudferrylib/os/actions/convert_volume_to_compute.py b/cloudferrylib/os/actions/convert_volume_to_compute.py index 5eaabe67..3aea6970 100644 --- a/cloudferrylib/os/actions/convert_volume_to_compute.py +++ b/cloudferrylib/os/actions/convert_volume_to_compute.py @@ -31,7 +31,8 @@ def run(self, storage_info, compute_ignored={}, **kwargs): instances[instance_id] = volume['meta']['instance'] instances[instance_id]['meta']['volume'] = [] volume['meta'].pop('instance') - instances[instance_id] = self.map_volume(instances[instance_id], volume) + instances[instance_id] = self.map_volume(instances[instance_id], + volume) for inst in instances.itervalues(): for vol in inst['instance']['volumes']: volumes_old[vol['id']]['volume']['device'] = vol['device'] @@ -43,4 +44,4 @@ def map_volume(instance, volume): for vol_old in instance['instance']['volumes']: if volume['old_id'] == vol_old['id']: vol_old['id'] = volume['volume']['id'] - return instance \ No newline at end of file + return instance diff --git a/cloudferrylib/os/actions/convert_volume_to_image.py b/cloudferrylib/os/actions/convert_volume_to_image.py index 68a43023..2fd7fa5a 100644 --- a/cloudferrylib/os/actions/convert_volume_to_image.py +++ b/cloudferrylib/os/actions/convert_volume_to_image.py @@ -18,6 +18,7 @@ from cloudferrylib.utils import utils import copy + LOG = utils.get_log(__name__) CEPH = 'ceph' ACTIVE = 'active' @@ -53,7 +54,9 @@ def run(self, storage_info={}, **kwargs): vol['id'], force=True, image_name=vol['id'], container_format=self.container_format, disk_format=self.disk_format) - resource_image.wait_for_status(image_id, resource_image.get_status, ACTIVE) + resource_image.wait_for_status(image_id, + resource_image.get_status, + ACTIVE) resource_image.patch_image(resource_image.get_backend(), image_id) image_vol = resource_image.read_info(image_id=image_id) img_new = { diff --git a/cloudferrylib/os/actions/copy_g2g.py b/cloudferrylib/os/actions/copy_g2g.py index 3a844fa0..542557b3 100644 --- a/cloudferrylib/os/actions/copy_g2g.py +++ b/cloudferrylib/os/actions/copy_g2g.py @@ -28,9 +28,9 @@ def run(self, images_info=None, **kwargs): dst_image = self.dst_cloud.resources[utl.IMAGE_RESOURCE] if not images_info: - action_get_im = get_info_images.GetInfoImages(self.init, cloud='src_cloud') + action_get_im = get_info_images.GetInfoImages(self.init, + cloud='src_cloud') images_info = action_get_im.run() new_info = dst_image.deploy(images_info) return {'images_info': new_info} - diff --git a/cloudferrylib/os/actions/deploy_snapshots.py b/cloudferrylib/os/actions/deploy_snapshots.py index 92293856..7a32dd31 100644 --- a/cloudferrylib/os/actions/deploy_snapshots.py +++ b/cloudferrylib/os/actions/deploy_snapshots.py @@ -45,38 +45,51 @@ def run(self, storage_info=None, identity_info=None, **kwargs): for snap in snapshots_list: if snapshots_list.index(snap) == 0: - act_snap_transfer = snap_transfer.SnapTransfer(self.init, - ssh_ceph_to_ceph.SSHCephToCeph, 1) + act_snap_transfer = \ + snap_transfer.SnapTransfer( + self.init, + ssh_ceph_to_ceph.SSHCephToCeph, + 1) else: snap_num = snapshots_list.index(snap) - snap['prev_snapname'] = snapshots_list[snap_num - 1]['name'] - act_snap_transfer = snap_transfer.SnapTransfer(self.init, - ssh_ceph_to_ceph.SSHCephToCeph, 2) - - + snap['prev_snapname'] = \ + snapshots_list[snap_num - 1]['name'] + act_snap_transfer = \ + snap_transfer.SnapTransfer( + self.init, + ssh_ceph_to_ceph.SSHCephToCeph, + 2) act_snap_transfer.run(volume=vol_info, snapshot_info=snap) - new_snapshot = volume_resource.create_snapshot(volume_id=vol_id, - display_name=snap['display_name'], - display_description=snap['display_description']) + volume_resource.create_snapshot( + volume_id=vol_id, + display_name=snap['display_name'], + display_description=snap['display_description']) - act_snap_transfer = snap_transfer.SnapTransfer(self.init, - ssh_ceph_to_ceph.SSHCephToCeph, 3) - act_snap_transfer.run(volume=vol_info, snapshot_info=snapshots_list[-1]) + act_snap_transfer = snap_transfer.SnapTransfer( + self.init, + ssh_ceph_to_ceph.SSHCephToCeph, + 3) + act_snap_transfer.run(volume=vol_info, + snapshot_info=snapshots_list[-1]) for snap in snapshots_list: if volume_resource.config.storage.host: - act_delete_redundant_snap = rbd_util.RbdUtil(cloud=self.cloud, - config_migrate=self.cfg.migrate, - host=vol_info[utl.HOST_DST]) - act_delete_redundant_snap.snap_rm(vol_info[utl.PATH_DST], - snap['name']) + act_delete_redundant_snap = \ + rbd_util.RbdUtil(cloud=self.cloud, + config_migrate=self.cfg.migrate, + host=vol_info[utl.HOST_DST]) + act_delete_redundant_snap.snap_rm( + vol_info[utl.PATH_DST], + snap['name']) else: - act_delete_redundant_snap = rbd_util.RbdUtil(cloud=self.cloud, - config_migrate=self.cfg.migrate) - act_delete_redundant_snap.snap_rm(vol_info[utl.PATH_DST], - snap['name'], vol_info[utl.HOST_DST]) + act_delete_redundant_snap = \ + rbd_util.RbdUtil(cloud=self.cloud, + config_migrate=self.cfg.migrate) + act_delete_redundant_snap.snap_rm( + vol_info[utl.PATH_DST], + snap['name'], vol_info[utl.HOST_DST]) else: one_volume_info = { @@ -87,10 +100,10 @@ def run(self, storage_info=None, identity_info=None, **kwargs): } } - - act_transport_vol_data = task_transfer.TaskTransfer(self.init, - 'SSHCephToCeph', - input_info='one_volume_info') + act_transport_vol_data = \ + task_transfer.TaskTransfer(self.init, + 'SSHCephToCeph', + input_info='one_volume_info') act_transport_vol_data.run(**one_volume_info) diff --git a/cloudferrylib/os/actions/deploy_volumes.py b/cloudferrylib/os/actions/deploy_volumes.py index 8e4f543f..df6e9c1c 100644 --- a/cloudferrylib/os/actions/deploy_volumes.py +++ b/cloudferrylib/os/actions/deploy_volumes.py @@ -48,7 +48,3 @@ def run(self, storage_info={}, identity_info={}, **kwargs): return { 'storage_info': storage_info_new } - - - - diff --git a/cloudferrylib/os/actions/failure_act.py b/cloudferrylib/os/actions/failure_act.py index 7eee6864..fe1e60bf 100644 --- a/cloudferrylib/os/actions/failure_act.py +++ b/cloudferrylib/os/actions/failure_act.py @@ -22,6 +22,7 @@ class DivisionByZero(action.Action): def run(self, *args, **kwargs): - LOG.debug("Dividing by zero (special case to fail migration execution)") + LOG.debug("Dividing by zero " + "(special case to fail migration execution)") a = 1 / 0 return a diff --git a/cloudferrylib/os/actions/image_snapshot.py b/cloudferrylib/os/actions/image_snapshot.py index 987ddda2..1b355dac 100644 --- a/cloudferrylib/os/actions/image_snapshot.py +++ b/cloudferrylib/os/actions/image_snapshot.py @@ -21,6 +21,7 @@ NAMESPACE = 'IMAGE_SNAPSHOT' + class ImageSnapshotBasic(action.Action): """Base class that contains common to image_snapshot methods""" diff --git a/cloudferrylib/os/actions/is_not_copy_diff_file.py b/cloudferrylib/os/actions/is_not_copy_diff_file.py index 12396960..c74e7edd 100644 --- a/cloudferrylib/os/actions/is_not_copy_diff_file.py +++ b/cloudferrylib/os/actions/is_not_copy_diff_file.py @@ -21,6 +21,8 @@ ISCSI = 'iscsi' PATH_COPY_DIFF = 0 DEFAULT = 1 +INST = utl.INSTANCES_TYPE +BODY = utl.INSTANCE_BODY class IsNotCopyDiffFile(action.Action): @@ -31,9 +33,10 @@ def run(self, info=None, **kwargs): dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend backend_ephem_drv_dst = dst_compute.config.compute.backend - instance_boot = info[utl.INSTANCES_TYPE].values()[0][utl.INSTANCE_BODY]['boot_mode'] - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == ISCSI): + instance_boot = \ + info[INST].values()[0][BODY]['boot_mode'] + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == ISCSI)): self.set_next_path(PATH_COPY_DIFF) - return { - } + return {} diff --git a/cloudferrylib/os/actions/is_not_merge_diff.py b/cloudferrylib/os/actions/is_not_merge_diff.py index 705f6542..98342fce 100644 --- a/cloudferrylib/os/actions/is_not_merge_diff.py +++ b/cloudferrylib/os/actions/is_not_merge_diff.py @@ -21,6 +21,8 @@ ISCSI = 'iscsi' PATH_MERGE_DIFF_IMAGE = 0 DEFAULT = 1 +INST = utl.INSTANCES_TYPE +BODY = utl.INSTANCE_BODY class IsNotMergeDiff(action.Action): @@ -31,9 +33,11 @@ def run(self, info=None, **kwargs): dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend backend_ephem_drv_dst = dst_compute.config.compute.backend - instance_boot = info[utl.INSTANCES_TYPE].values()[0][utl.INSTANCE_BODY]['boot_mode'] - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == CEPH): + instance_boot = \ + info[INST].values()[0][BODY]['boot_mode'] + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == CEPH)): self.set_next_path(PATH_MERGE_DIFF_IMAGE) return { } diff --git a/cloudferrylib/os/actions/is_not_transport_image.py b/cloudferrylib/os/actions/is_not_transport_image.py index 5c188c33..9c48cc17 100644 --- a/cloudferrylib/os/actions/is_not_transport_image.py +++ b/cloudferrylib/os/actions/is_not_transport_image.py @@ -21,6 +21,8 @@ ISCSI = 'iscsi' PATH_TRANSPORT_IMAGE = 0 DEFAULT = 1 +INST = utl.INSTANCES_TYPE +BODY = utl.INSTANCE_BODY class IsNotTransportImage(action.Action): @@ -29,8 +31,10 @@ def run(self, info=None, **kwargs): info = copy.deepcopy(info) src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend - instance_boot = info[utl.INSTANCES_TYPE].values()[0][utl.INSTANCE_BODY]['boot_mode'] - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == CEPH): + instance_boot = \ + info[INST].values()[0][BODY]['boot_mode'] + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == CEPH)): self.set_next_path(PATH_TRANSPORT_IMAGE) return { } diff --git a/cloudferrylib/os/actions/live_migration.py b/cloudferrylib/os/actions/live_migration.py index c93d8ed0..9d66a8d6 100644 --- a/cloudferrylib/os/actions/live_migration.py +++ b/cloudferrylib/os/actions/live_migration.py @@ -96,8 +96,10 @@ def run(self, info=None, **kwargs): src_host = instance_host(src_instance) dst_host = instance_host(dst_instance) - src_runner = remote_runner.RemoteRunner(src_host, cfglib.CONF.src.ssh_user) - dst_runner = remote_runner.RemoteRunner(dst_host, cfglib.CONF.dst.ssh_user) + src_runner = remote_runner.RemoteRunner(src_host, + cfglib.CONF.src.ssh_user) + dst_runner = remote_runner.RemoteRunner(dst_host, + cfglib.CONF.dst.ssh_user) src_libvirt = libvirt.Libvirt(src_runner) dst_libvirt = libvirt.Libvirt(dst_runner) @@ -116,14 +118,24 @@ def run(self, info=None, **kwargs): dst_backing_file = dst_libvirt.get_backing_file(new_id) src_backing_file = src_libvirt.get_backing_file(old_id) migration_backing_file = os.path.join( - libvirt.nova_instances_path, '_base', 'migration_disk_{}'.format(old_id)) + libvirt.nova_instances_path, + '_base', + 'migration_disk_{}'.format(old_id)) dst_compute.wait_for_status(new_id, dst_compute.get_status, 'active') - with files.RemoteTempFile(src_runner, "migrate-{}".format(old_id), src_vm_xml.dump()) as migration_file,\ - files.RemoteSymlink(src_runner, src_backing_file, migration_backing_file),\ - files.RemoteSymlink(dst_runner, dst_backing_file, migration_backing_file),\ + with files.RemoteTempFile(src_runner, + "migrate-{}".format(old_id), + src_vm_xml.dump()) as migration_file,\ + files.RemoteSymlink(src_runner, + src_backing_file, + migration_backing_file),\ + files.RemoteSymlink(dst_runner, + dst_backing_file, + migration_backing_file),\ ubuntu.StopNovaCompute(dst_runner),\ - libvirt.QemuBackingFileMover(src_libvirt.runner, migration_backing_file, old_id): + libvirt.QemuBackingFileMover(src_libvirt.runner, + migration_backing_file, + old_id): destroyer = libvirt.DestNovaInstanceDestroyer(dst_libvirt, dst_compute, @@ -131,7 +143,9 @@ def run(self, info=None, **kwargs): dst_instance.id) try: destroyer.do() - src_libvirt.live_migrate(src_virsh_name, dst_host, migration_file.filename) + src_libvirt.live_migrate(src_virsh_name, + dst_host, + migration_file.filename) except remote_runner.RemoteExecutionError: destroyer.undo() finally: diff --git a/cloudferrylib/os/actions/map_compute_info.py b/cloudferrylib/os/actions/map_compute_info.py index c30138f7..6ffd987a 100644 --- a/cloudferrylib/os/actions/map_compute_info.py +++ b/cloudferrylib/os/actions/map_compute_info.py @@ -16,6 +16,7 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils as utl import copy + INSTANCES = 'instances' DIFF = 'diff' @@ -27,28 +28,28 @@ class MapComputeInfo(action.Action): def run(self, info=None, **kwargs): - new_compute_info = copy.deepcopy(info) + new_info = copy.deepcopy(info) src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] src_flavors_dict = \ - {flavor.id: flavor.name for flavor in src_compute.get_flavor_list()} + {flav.id: flav.name for flav in src_compute.get_flavor_list()} dst_flavors_dict = \ - {flavor.name: flavor.id for flavor in dst_compute.get_flavor_list()} + {flav.name: flav.id for flav in dst_compute.get_flavor_list()} - for instance_id, instance in new_compute_info[utl.INSTANCES_TYPE].iteritems(): + for instance_id, instance in new_info[utl.INSTANCES_TYPE].iteritems(): _instance = instance['instance'] if _instance['flavor_id'] in src_flavors_dict: flavor_name = src_flavors_dict[_instance['flavor_id']] _instance['flavor_id'] = dst_flavors_dict[flavor_name] - #TODO: path_dst is probably non used code, need functional testing + # TODO: path_dst is probably non used code, need functional testing self.dst_cloud.cloud_config.cloud.temp = '-' path_dst = "%s/%s" % (self.dst_cloud.cloud_config.cloud.temp, "temp%s_base" % instance_id) instance[DIFF][PATH_DST] = path_dst instance[DIFF][HOST_DST] = self.dst_cloud.getIpSsh() return { - 'info': new_compute_info + 'info': new_info } diff --git a/cloudferrylib/os/actions/merge_base_and_diff.py b/cloudferrylib/os/actions/merge_base_and_diff.py index 5c5e31b3..99ae1f91 100644 --- a/cloudferrylib/os/actions/merge_base_and_diff.py +++ b/cloudferrylib/os/actions/merge_base_and_diff.py @@ -1,6 +1,5 @@ from fabric.api import run, settings, env from cloudferrylib.base.action import action -from cloudferrylib.utils import forward_agent from cloudferrylib.utils import utils as utl INSTANCES = 'instances' diff --git a/cloudferrylib/os/actions/post_transport_instance.py b/cloudferrylib/os/actions/post_transport_instance.py index 148e374d..f77f21c3 100644 --- a/cloudferrylib/os/actions/post_transport_instance.py +++ b/cloudferrylib/os/actions/post_transport_instance.py @@ -15,22 +15,9 @@ import copy -from fabric.api import env -from fabric.api import run -from fabric.api import settings - from cloudferrylib.base.action import action -from cloudferrylib.os.actions import convert_file_to_image -from cloudferrylib.os.actions import convert_image_to_file -from cloudferrylib.os.actions import convert_volume_to_image -from cloudferrylib.os.actions import copy_g2g from cloudferrylib.os.actions import task_transfer -from cloudferrylib.utils import utils as utl, forward_agent - -from cloudferrylib.utils.drivers import ssh_ceph_to_ceph -from cloudferrylib.utils.drivers import ssh_ceph_to_file -from cloudferrylib.utils.drivers import ssh_file_to_file -from cloudferrylib.utils.drivers import ssh_file_to_ceph +from cloudferrylib.utils import utils as utl CLOUD = 'cloud' @@ -66,7 +53,7 @@ class PostTransportInstance(action.Action): def run(self, info=None, **kwargs): info = copy.deepcopy(info) - #Init before run + # Init before run src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend @@ -76,7 +63,7 @@ def run(self, info=None, **kwargs): } } - #Get next one instance + # Get next one instance for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): instance_boot = instance[utl.INSTANCE_BODY]['boot_mode'] one_instance = { @@ -84,9 +71,12 @@ def run(self, info=None, **kwargs): instance_id: instance } } - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == ISCSI): - self.copy_diff_file(self.src_cloud, self.dst_cloud, one_instance) + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == ISCSI)): + self.copy_diff_file(self.src_cloud, + self.dst_cloud, + one_instance) new_info[utl.INSTANCES_TYPE].update( one_instance[utl.INSTANCES_TYPE]) @@ -95,7 +85,13 @@ def run(self, info=None, **kwargs): 'info': new_info } - def copy_data_via_ssh(self, src_cloud, dst_cloud, info, body, resources, types): + def copy_data_via_ssh(self, + src_cloud, + dst_cloud, + info, + body, + resources, + types): dst_storage = dst_cloud.resources[resources] src_compute = src_cloud.resources[resources] src_backend = src_compute.config.compute.backend @@ -113,4 +109,4 @@ def copy_diff_file(self, src_cloud, dst_cloud, info): info, utl.DIFF_BODY, utl.COMPUTE_RESOURCE, - utl.INSTANCES_TYPE) \ No newline at end of file + utl.INSTANCES_TYPE) diff --git a/cloudferrylib/os/actions/pre_transport_instance.py b/cloudferrylib/os/actions/pre_transport_instance.py index a10701f2..6849f388 100644 --- a/cloudferrylib/os/actions/pre_transport_instance.py +++ b/cloudferrylib/os/actions/pre_transport_instance.py @@ -22,8 +22,6 @@ from cloudferrylib.base.action import action from cloudferrylib.os.actions import convert_file_to_image from cloudferrylib.os.actions import convert_image_to_file -from cloudferrylib.os.actions import convert_volume_to_image -from cloudferrylib.os.actions import copy_g2g from cloudferrylib.os.actions import task_transfer from cloudferrylib.utils import utils as utl, forward_agent @@ -66,7 +64,7 @@ class PreTransportInstance(action.Action): def run(self, info=None, **kwargs): info = copy.deepcopy(info) - #Init before run + # Init before run src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] dst_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend @@ -76,7 +74,7 @@ def run(self, info=None, **kwargs): } } - #Get next one instance + # Get next one instance for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): instance_boot = instance[utl.INSTANCE_BODY]['boot_mode'] one_instance = { @@ -84,12 +82,16 @@ def run(self, info=None, **kwargs): instance_id: instance } } - #Pre processing deploy - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == CEPH): + # Pre processing deploy + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == CEPH)): self.transport_image(self.dst_cloud, one_instance, instance_id) - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == CEPH): - self.transport_diff_and_merge(self.dst_cloud, one_instance, instance_id) + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == CEPH)): + self.transport_diff_and_merge(self.dst_cloud, + one_instance, + instance_id) new_info[utl.INSTANCES_TYPE].update( one_instance[utl.INSTANCES_TYPE]) @@ -97,15 +99,21 @@ def run(self, info=None, **kwargs): 'info': new_info } - def convert_file_to_image(self, dst_cloud, base_file, disk_format, instance_id): - converter = convert_file_to_image.ConvertFileToImage(self.init, cloud=dst_cloud) + def convert_file_to_image(self, + dst_cloud, + base_file, + disk_format, + instance_id): + converter = convert_file_to_image.ConvertFileToImage(self.init, + cloud=dst_cloud) dst_image_id = converter.run(file_path=base_file, image_format=disk_format, image_name="%s-image" % instance_id) return dst_image_id def convert_image_to_file(self, cloud, image_id, filename): - convertor = convert_image_to_file.ConvertImageToFile(self.init, cloud=cloud) + convertor = convert_image_to_file.ConvertImageToFile(self.init, + cloud=cloud) convertor.run(image_id=image_id, base_filename=filename) @@ -115,7 +123,8 @@ def merge_file(self, cloud, base_file, diff_file): self.commit_diff_file(host, diff_file) def transport_image(self, dst_cloud, info, instance_id): - path_dst = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, "temp%s_base" % instance_id) + path_dst = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, + "temp%s_base" % instance_id) info[INSTANCES][instance_id][DIFF][PATH_DST] = path_dst info[INSTANCES][instance_id][DIFF][HOST_DST] = dst_cloud.getIpSsh() @@ -126,8 +135,8 @@ def transport_image(self, dst_cloud, info, instance_id): resource_root_name=utl.DIFF_BODY) transporter.run(info=info) - converter = convert_file_to_image.ConvertFileToImage(self.init, 'dst_cloud') - + converter = convert_file_to_image.ConvertFileToImage(self.init, + 'dst_cloud') dst_image_id = converter.run(file_path=path_dst, image_format='raw', @@ -135,7 +144,8 @@ def transport_image(self, dst_cloud, info, instance_id): info[INSTANCES][instance_id][INSTANCE_BODY]['image_id'] = dst_image_id def transport_diff_and_merge(self, dst_cloud, info, instance_id): - convertor = convert_image_to_file.ConvertImageToFile(self.init, cloud='dst_cloud') + convertor = convert_image_to_file.ConvertImageToFile(self.init, + cloud='dst_cloud') transporter = task_transfer.TaskTransfer( self.init, TRANSPORTER_MAP[ISCSI][ISCSI], @@ -143,8 +153,10 @@ def transport_diff_and_merge(self, dst_cloud, info, instance_id): resource_root_name=utl.DIFF_BODY) image_id = info[INSTANCES][instance_id][utl.INSTANCE_BODY]['image_id'] - base_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, "temp%s_base" % instance_id) - diff_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, "temp%s" % instance_id) + base_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, + "temp%s_base" % instance_id) + diff_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, + "temp%s" % instance_id) info[INSTANCES][instance_id][DIFF][PATH_DST] = diff_file info[INSTANCES][instance_id][DIFF][HOST_DST] = dst_cloud.getIpSsh() @@ -162,9 +174,12 @@ def transport_diff_and_merge(self, dst_cloud, info, instance_id): disk_format = image[utl.IMAGE_BODY]['disk_format'] if image_res.config.image.convert_to_raw: if disk_format.lower() != utl.RAW: - self.convert_file_to_raw(dst_cloud.cloud_config.cloud.host, disk_format, base_file) + self.convert_file_to_raw(dst_cloud.cloud_config.cloud.host, + disk_format, + base_file) disk_format = utl.RAW - converter = convert_file_to_image.ConvertFileToImage(self.init, cloud='dst_cloud') + converter = convert_file_to_image.ConvertFileToImage(self.init, + cloud='dst_cloud') dst_image_id = converter.run(file_path=base_file, image_format=disk_format, image_name="%s-image" % instance_id) @@ -190,4 +205,4 @@ def rebase_diff_file(host, base_file, diff_file): def commit_diff_file(host, diff_file): with settings(host_string=host, connection_attempts=env.connection_attempts): - run("qemu-img commit %s" % diff_file) + run("qemu-img commit %s" % diff_file) diff --git a/cloudferrylib/os/actions/prepare_volumes_data_map.py b/cloudferrylib/os/actions/prepare_volumes_data_map.py index 766fcbd8..d260c9b4 100644 --- a/cloudferrylib/os/actions/prepare_volumes_data_map.py +++ b/cloudferrylib/os/actions/prepare_volumes_data_map.py @@ -49,7 +49,8 @@ def run(self, **kwargs): utl.HOST_DST: dst_host, utl.PATH_DST: dst_path }) - volumes_data_map[dst_id][utl.META_INFO].update(src_volumes[src_id][utl.META_INFO]) + volumes_data_map[dst_id][utl.META_INFO].\ + update(src_volumes[src_id][utl.META_INFO]) volumes = { utl.VOLUMES_TYPE: volumes_data_map } diff --git a/cloudferrylib/os/actions/recreate_boot_image.py b/cloudferrylib/os/actions/recreate_boot_image.py index 22735bfc..a5b5ddbb 100644 --- a/cloudferrylib/os/actions/recreate_boot_image.py +++ b/cloudferrylib/os/actions/recreate_boot_image.py @@ -15,7 +15,8 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import files from cloudferrylib.utils import remote_runner -from cloudferrylib.utils.drivers.ssh_chunks import verified_file_copy, remote_md5_sum +from cloudferrylib.utils.drivers.ssh_chunks import verified_file_copy,\ + remote_md5_sum from cloudferrylib.utils import utils import copy import os @@ -43,12 +44,17 @@ def __init__(self, init, cloud=None): password=dst_password, sudo=True) - def run(self, images_info=None, compute_ignored_images={}, missing_images={}, **kwargs): + def run(self, + images_info=None, + compute_ignored_images={}, + missing_images={}, + **kwargs): """ Create boot image on destination based on root disk of instance. Use diff&base images, commit all changes from diff to base, copy base and add as a glance image. - Image ID from source is used as a name of new image because we can't get name of deleted image. + Image ID from source is used as a name of new image because we can't + get name of deleted image. :param images_info: dict with all images on source :param compute_ignored_images: not used, just resending to down level :param missing_images: dict with images that has been removed on source @@ -61,12 +67,15 @@ def run(self, images_info=None, compute_ignored_images={}, missing_images={}, ** for image_id_src, gl_image in images_info['images'].iteritems(): if image_id_src == img_id and not gl_image['image']: diff = gl_image['meta']['instance'][0]['diff']['path_src'] - img_src_host = gl_image['meta']['instance'][0]['diff']['host_src'] + img_src_host = \ + gl_image['meta']['instance'][0]['diff']['host_src'] if img_src_host != self.src_host: LOG.warning('Different host information. ' + - 'Image is located on host {img_src_host},' + - 'but host in the configuration file {src_host}.'.format(img_src_host, - self.src_host)) + 'Image is located on host ' + + '{img_src_host},' + + 'but host in the configuration file ' + + '{src_host}.'.format(img_src_host, + self.src_host)) continue new_img = self.process_image(img_id, diff) gl_image['image']['id'] = new_img['id'] @@ -79,7 +88,8 @@ def run(self, images_info=None, compute_ignored_images={}, missing_images={}, ** def process_image(self, img_id=None, diff=None): """ - Processing image file: copy from source to destination, create glance image + Processing image file: copy from source to destination, + create glance image :param img_id: image ID from source :param diff: diff file of root disk for instance :return: new image ID if image is created @@ -98,11 +108,21 @@ def process_image(self, img_id=None, diff=None): self.src_runner.run('cp {} {}'.format(base, base_file)) qemu_img_src.diff_rebase(base_file, diff_file, self.src_host) qemu_img_src.diff_commit(src_tmp_dir, diff_name, self.src_host) - verified_file_copy(self.src_runner, self.dst_runner, self.dst_user, - base_file, dst_base_file, self.dst_host, 1) + verified_file_copy(self.src_runner, + self.dst_runner, + self.dst_user, + base_file, + dst_base_file, + self.dst_host, + 1) else: - verified_file_copy(self.src_runner, self.dst_runner, self.dst_user, - diff_file, dst_base_file, self.dst_host, 1) + verified_file_copy(self.src_runner, + self.dst_runner, + self.dst_user, + diff_file, + dst_base_file, + self.dst_host, + 1) image_resource = self.dst_cloud.resources[utils.IMAGE_RESOURCE] id = image_resource.glance_img_create(img_id, 'qcow2', dst_base_file) diff --git a/cloudferrylib/os/actions/remote_execution.py b/cloudferrylib/os/actions/remote_execution.py index 3c7d9b3f..f96296c9 100644 --- a/cloudferrylib/os/actions/remote_execution.py +++ b/cloudferrylib/os/actions/remote_execution.py @@ -24,7 +24,9 @@ def __init__(self, cloud, host=None, int_host=None, config_migrate=None): self.host = host self.int_host = int_host self.config_migrate = config_migrate - self.remote_exec_obj = SshUtil(self.cloud, self.config_migrate, self.host) + self.remote_exec_obj = SshUtil(self.cloud, + self.config_migrate, + self.host) super(RemoteExecution, self).__init__({}) def run(self, command, **kwargs): diff --git a/cloudferrylib/os/actions/select_boot_volume.py b/cloudferrylib/os/actions/select_boot_volume.py index 30d0b85d..178bba35 100644 --- a/cloudferrylib/os/actions/select_boot_volume.py +++ b/cloudferrylib/os/actions/select_boot_volume.py @@ -9,4 +9,3 @@ def run(self, info=None, **kwargs): return { 'info': info_boot } - diff --git a/cloudferrylib/os/actions/set_volume_id.py b/cloudferrylib/os/actions/set_volume_id.py index 9b0b472a..a984981f 100644 --- a/cloudferrylib/os/actions/set_volume_id.py +++ b/cloudferrylib/os/actions/set_volume_id.py @@ -33,4 +33,3 @@ def run(self, info=None, **kwargs): meta_volume_array.append({'volume': volume}) instance['meta'] = {'volume': meta_volume_array} return {'info': info} - diff --git a/cloudferrylib/os/actions/transport_ephemeral.py b/cloudferrylib/os/actions/transport_ephemeral.py index 16b01edd..b7b16814 100644 --- a/cloudferrylib/os/actions/transport_ephemeral.py +++ b/cloudferrylib/os/actions/transport_ephemeral.py @@ -140,7 +140,7 @@ def copy_ephemeral_ceph_to_iscsi(self, src_cloud, dst_cloud, info): qemu_img_dst = dst_cloud.qemu_img qemu_img_src = src_cloud.qemu_img - temp_path_src = temp_src+"/%s"+utl.DISK_EPHEM + temp_path_src = temp_src + "/%s" + utl.DISK_EPHEM for inst_id, inst in instances.iteritems(): path_src_id_temp = temp_path_src % inst_id diff --git a/cloudferrylib/os/actions/transport_instance.py b/cloudferrylib/os/actions/transport_instance.py index cc4505a6..1b19fba1 100644 --- a/cloudferrylib/os/actions/transport_instance.py +++ b/cloudferrylib/os/actions/transport_instance.py @@ -58,7 +58,7 @@ def run(self, info=None, **kwargs): } } - #Get next one instance + # Get next one instance for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): instance = self._replace_user_ids(instance) @@ -105,22 +105,22 @@ def prepare_ephemeral_drv(self, info, new_info, map_new_to_old_ids): instance_new = new_info[INSTANCES][new_id] ephemeral_path_dst = instance_new[EPHEMERAL][PATH_SRC] - instance_new[EPHEMERAL][PATH_DST] = ephemeral_path_dst + instance_new[EPHEMERAL][PATH_DST] = ephemeral_path_dst ephemeral_host_dst = instance_new[EPHEMERAL][HOST_SRC] instance_new[EPHEMERAL][HOST_DST] = ephemeral_host_dst - + diff_path_dst = instance_new[DIFF][PATH_SRC] - instance_new[DIFF][PATH_DST] = diff_path_dst + instance_new[DIFF][PATH_DST] = diff_path_dst diff_host_dst = instance_new[DIFF][HOST_SRC] instance_new[DIFF][HOST_DST] = diff_host_dst ephemeral_path_src = instance_old[EPHEMERAL][PATH_SRC] - instance_new[EPHEMERAL][PATH_SRC] = ephemeral_path_src + instance_new[EPHEMERAL][PATH_SRC] = ephemeral_path_src ephemeral_host_src = instance_old[EPHEMERAL][HOST_SRC] instance_new[EPHEMERAL][HOST_SRC] = ephemeral_host_src - + diff_path_src = instance_old[DIFF][PATH_SRC] - instance_new[DIFF][PATH_SRC] = diff_path_src + instance_new[DIFF][PATH_SRC] = diff_path_src diff_host_src = instance_old[DIFF][HOST_SRC] instance_new[DIFF][HOST_SRC] = diff_host_src diff --git a/cloudferrylib/os/actions/utils.py b/cloudferrylib/os/actions/utils.py index 38d2313b..dd79634c 100644 --- a/cloudferrylib/os/actions/utils.py +++ b/cloudferrylib/os/actions/utils.py @@ -20,7 +20,13 @@ LOG = utils.get_log(__name__) -def transfer_file_to_file(src_cloud, dst_cloud, host_src, host_dst, path_src, path_dst, cfg_migrate): +def transfer_file_to_file(src_cloud, + dst_cloud, + host_src, + host_dst, + path_src, + path_dst, + cfg_migrate): # TODO: Delete after transport_db_via_ssh action rewriting LOG.debug("| | copy file") ssh_ip_src = src_cloud.getIpSsh() @@ -30,12 +36,15 @@ def transfer_file_to_file(src_cloud, dst_cloud, host_src, host_dst, path_src, pa with utils.forward_agent(cfg_migrate.key_filename): with utils.up_ssh_tunnel(host_dst, ssh_ip_dst, ssh_ip_src) as port: if cfg_migrate.file_compression == "dd": - run(("ssh -oStrictHostKeyChecking=no %s 'dd bs=1M if=%s' " + - "| ssh -oStrictHostKeyChecking=no -p %s localhost 'dd bs=1M of=%s'") % + run(("ssh -oStrictHostKeyChecking=no %s 'dd bs=1M " + + "if=%s' | ssh -oStrictHostKeyChecking=no " + + "-p %s localhost 'dd bs=1M of=%s'") % (host_src, path_src, port, path_dst)) elif cfg_migrate.file_compression == "gzip": - run(("ssh -oStrictHostKeyChecking=no %s 'gzip -%s -c %s' " + - "| ssh -oStrictHostKeyChecking=no -p %s localhost 'gunzip | dd bs=1M of=%s'") % + run(("ssh -oStrictHostKeyChecking=no " + + "%s 'gzip -%s -c %s' " + + "| ssh -oStrictHostKeyChecking=no -p %s localhost " + + "'gunzip | dd bs=1M of=%s'") % (host_src, cfg_migrate.level_compression, path_src, port, path_dst)) diff --git a/cloudferrylib/os/compute/nova_compute.py b/cloudferrylib/os/compute/nova_compute.py index ac5e3b51..d3878ba5 100644 --- a/cloudferrylib/os/compute/nova_compute.py +++ b/cloudferrylib/os/compute/nova_compute.py @@ -308,7 +308,7 @@ def convert_instance(instance, cfg, cloud): instance, instance_block_info, is_ceph_ephemeral=is_ceph, - disk=DISK+LOCAL) + disk=DISK + LOCAL) diff = { 'path_src': None, @@ -693,8 +693,7 @@ def change_status(self, status, instance=None, instance_id=None): 'status': lambda status: lambda instance: self.wait_for_status( instance_id, self.get_status, - status, - ) + status) } map_status = { 'paused': { diff --git a/cloudferrylib/os/identity/keystone.py b/cloudferrylib/os/identity/keystone.py index 0d7b2a30..63306e76 100644 --- a/cloudferrylib/os/identity/keystone.py +++ b/cloudferrylib/os/identity/keystone.py @@ -475,7 +475,8 @@ def _passwd_notification(self, email, name, password): def _deploy_roles(self, roles): LOG.info("Role deployment started...") - dst_roles = {role.name.lower(): role.id for role in self.get_roles_list()} + dst_roles = { + role.name.lower(): role.id for role in self.get_roles_list()} for _role in roles: role = _role['role'] if role['name'].lower() not in dst_roles: @@ -503,11 +504,13 @@ def _get_user_tenants_roles(self, tenant_list=None, user_list=None): if user_list is None: user_list = [] if not self.config.migrate.optimize_user_role_fetch: - user_tenants_roles = self._get_user_tenants_roles_by_api(tenant_list, - user_list) + user_tenants_roles = \ + self._get_user_tenants_roles_by_api(tenant_list, + user_list) else: - user_tenants_roles = self._get_user_tenants_roles_by_db(tenant_list, - user_list) + user_tenants_roles = \ + self._get_user_tenants_roles_by_db(tenant_list, + user_list) return user_tenants_roles def _get_roles_sql_request(self): @@ -515,16 +518,17 @@ def _get_roles_sql_request(self): try: is_project_metadata = self.mysql_connector.execute( "SHOW TABLES LIKE 'user_project_metadata'").rowcount - if is_project_metadata: #for grizzly case + if is_project_metadata: # for grizzly case return self.mysql_connector.execute( "SELECT * FROM user_project_metadata") is_assignment = self.mysql_connector.execute( "SHOW TABLES LIKE 'assignment'").rowcount - if is_assignment: #for icehouse case + if is_assignment: # for icehouse case res_raw = self.mysql_connector.execute( "SELECT * FROM assignment") res_tmp = {} - for type_record, actor_id, project_id, role_id, inher_tmp in res_raw: + for (type_record, actor_id, project_id, + role_id, inher_tmp) in res_raw: if (actor_id, project_id) not in res_tmp: res_tmp[(actor_id, project_id)] = {'roles': []} res_tmp[(actor_id, project_id)]['roles'].append(role_id) @@ -609,9 +613,12 @@ def _upload_user_tenant_roles(self, user_tenants_roles, users, tenants): continue for _tenant in tenants: tenant = _tenant['tenant'] - user_roles_objs = _get_user_roles_cached(_user['meta']['new_id'], - _tenant['meta']['new_id']) - exists_roles = [dst_roles[role] if not hasattr(role, 'name') else role.name + user_roles_objs = _get_user_roles_cached( + _user['meta']['new_id'], + _tenant['meta']['new_id']) + exists_roles = [dst_roles[role] if not hasattr(role, + 'name') + else role.name for role in user_roles_objs] for _role in user_tenants_roles[user['name']][tenant['name']]: role = _role['role'] diff --git a/cloudferrylib/os/network/__init__.py b/cloudferrylib/os/network/__init__.py index a3e51dd3..ba605e43 100644 --- a/cloudferrylib/os/network/__init__.py +++ b/cloudferrylib/os/network/__init__.py @@ -11,4 +11,3 @@ # implied. # See the License for the specific language governing permissions and# # limitations under the License. - diff --git a/cloudferrylib/os/network/neutron.py b/cloudferrylib/os/network/neutron.py index d5f7822a..9ac928bc 100644 --- a/cloudferrylib/os/network/neutron.py +++ b/cloudferrylib/os/network/neutron.py @@ -15,11 +15,10 @@ import ipaddr import netaddr -import collections from neutronclient.common import exceptions as neutron_exc from neutronclient.v2_0 import client as neutron_client -from cloudferrylib.base import network, exception +from cloudferrylib.base import network from cloudferrylib.os.identity import keystone as ksresource from cloudferrylib.utils import utils as utl @@ -881,7 +880,7 @@ def upload_lb_pools(self, pools, subnets): 'subnet_id': snet_id, 'protocol': pool['protocol'], 'lb_method': pool['lb_method'] - } + } } LOG.debug("Creating LB pool '%s'", pool['name']) pool['meta']['id'] = \ diff --git a/cloudferrylib/os/object_storage/__init__.py b/cloudferrylib/os/object_storage/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/os/object_storage/__init__.py +++ b/cloudferrylib/os/object_storage/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/os/object_storage/swift_storage.py b/cloudferrylib/os/object_storage/swift_storage.py index ba51df02..5285f521 100644 --- a/cloudferrylib/os/object_storage/swift_storage.py +++ b/cloudferrylib/os/object_storage/swift_storage.py @@ -40,15 +40,20 @@ def get_swift_conn(self, params=None): return conn.get_auth() def read_info(self, **kwargs): - info = {utl.OBJSTORAGE_RESOURCE: - {utl.CONTAINERS: {}}} + info = { + utl.OBJSTORAGE_RESOURCE: { + utl.CONTAINERS: {} + } + } account_info = self.get_account_info() info[utl.OBJSTORAGE_RESOURCE][utl.CONTAINERS] = account_info[1] for container_info in info[utl.OBJSTORAGE_RESOURCE][utl.CONTAINERS]: - container_info['objects'] = self.get_container(container_info['name'])[1] + container_info['objects'] = \ + self.get_container(container_info['name'])[1] for object_info in container_info['objects']: - resp, object_info['data'] = self.get_object(container_info['name'], - object_info['name']) + resp, object_info['data'] = \ + self.get_object(container_info['name'], + object_info['name']) return info def deploy(self, info, **kwargs): @@ -65,30 +70,35 @@ def get_account_info(self): return swift_client.get_account(self.storage_url, self.token) def get_container(self, container, *args): - return swift_client.get_container(self.storage_url, self.token, container, *args) + return swift_client.get_container(self.storage_url, + self.token, + container, *args) def get_object(self, container, obj_name, *args): - return swift_client.get_object(self.storage_url, self.token, container, obj_name, *args) - - def put_object(self, container, obj_name, content=None, content_type=None, *args): - return swift_client.put_object(self.storage_url, self.token, container, obj_name, content, content_type, *args) + return swift_client.get_object(self.storage_url, + self.token, + container, + obj_name, *args) + + def put_object(self, + container, + obj_name, + content=None, + content_type=None, + *args): + return swift_client.put_object(self.storage_url, + self.token, + container, + obj_name, + content, + content_type, + *args) def put_container(self, container, *args): - return swift_client.put_container(self.storage_url, self.token, container, *args) + return swift_client.put_container(self.storage_url, + self.token, + container, *args) def delete_container(self, container, *args): - return swift_client.delete_container(self.storage_url, self.token, container) - - - - - - - - - - - - - - + return swift_client.delete_container(self.storage_url, + self.token, container) diff --git a/cloudferrylib/os/storage/cinder_database.py b/cloudferrylib/os/storage/cinder_database.py index d2c86efa..30801b1a 100644 --- a/cloudferrylib/os/storage/cinder_database.py +++ b/cloudferrylib/os/storage/cinder_database.py @@ -98,9 +98,13 @@ def list_of_dicts_for_table(self, table): if DELETED in column_names: result = filter(lambda a: a.get(DELETED) == 0, result) if PROJECT_ID in column_names: - result = filter(lambda e: self._check_update_tenant_names(e, PROJECT_ID), result) + result = \ + filter(lambda e: self._check_update_tenant_names( + e, PROJECT_ID), result) if TENANT_ID in column_names: - result = filter(lambda e: self._check_update_tenant_names(e, TENANT_ID), result) + result = \ + filter(lambda e: self._check_update_tenant_names( + e, TENANT_ID), result) if STATUS in column_names: result = filter(lambda e: 'error' not in e[STATUS], result) if USER_ID in column_names: @@ -150,7 +154,8 @@ def get_key_and_auto_increment(cursor, table): auto_increment = cursor.fetchone().get("auto_increment") return primary_key, auto_increment - def filter_data(existing_data, data_to_be_added, primary_key, auto_increment): + def filter_data(existing_data, data_to_be_added, primary_key, + auto_increment): """ handle duplicates in database """ existing_hash = {i.get(primary_key): i for i in existing_data} unique_entries, duplicated_pk = [], [] @@ -171,7 +176,8 @@ def filter_data(existing_data, data_to_be_added, primary_key, auto_increment): # add entry to database without primary_key # primary key will be generated automaticaly duplicated_pk.append( - {i: candidate[i] for i in candidate if i != primary_key}) + {i: candidate[i] for i in candidate + if i != primary_key}) else: unique_entries.append(candidate) return unique_entries, duplicated_pk diff --git a/cloudferrylib/os/storage/cinder_storage.py b/cloudferrylib/os/storage/cinder_storage.py index efc4fbe2..1f687368 100644 --- a/cloudferrylib/os/storage/cinder_storage.py +++ b/cloudferrylib/os/storage/cinder_storage.py @@ -16,7 +16,6 @@ from cinderclient.v1 import client as cinder_client from cloudferrylib.base import storage -from cloudferrylib.utils import mysql_connector from cloudferrylib.utils import utils as utl LOG = utl.get_log(__name__) diff --git a/cloudferrylib/scheduler/base/begin_task.py b/cloudferrylib/scheduler/base/begin_task.py index c195bbcf..596bff1a 100644 --- a/cloudferrylib/scheduler/base/begin_task.py +++ b/cloudferrylib/scheduler/base/begin_task.py @@ -20,4 +20,4 @@ class BeginTask(Task): def __init__(self, **kwargs): self.__dict__.update(kwargs) - super(BeginTask, self).__init__() \ No newline at end of file + super(BeginTask, self).__init__() diff --git a/cloudferrylib/scheduler/base/end_task.py b/cloudferrylib/scheduler/base/end_task.py index 8bb911ee..a4e0d1b0 100644 --- a/cloudferrylib/scheduler/base/end_task.py +++ b/cloudferrylib/scheduler/base/end_task.py @@ -20,4 +20,4 @@ class EndTask(Task): def __init__(self, **kwargs): self.__dict__.update(kwargs) - super(EndTask, self).__init__() \ No newline at end of file + super(EndTask, self).__init__() diff --git a/cloudferrylib/scheduler/cursor.py b/cloudferrylib/scheduler/cursor.py index 6e0ad89a..44b55b17 100644 --- a/cloudferrylib/scheduler/cursor.py +++ b/cloudferrylib/scheduler/cursor.py @@ -33,7 +33,8 @@ def next(self): if self.threads: return self.threads.pop() if self.next_iter.next_element[0]: - if self.next_iter.num_element < len(self.next_iter.next_element): + if self.next_iter.num_element < len( + self.next_iter.next_element): self.__change_state_cursor(self.next_iter.num_element) else: self.__change_state_cursor(DEFAULT) diff --git a/cloudferrylib/scheduler/namespace.py b/cloudferrylib/scheduler/namespace.py index f3529570..752ba3c8 100644 --- a/cloudferrylib/scheduler/namespace.py +++ b/cloudferrylib/scheduler/namespace.py @@ -20,9 +20,10 @@ class Namespace: def __init__(self, vars={}): - if not CHILDREN in vars: + if CHILDREN not in vars: vars[CHILDREN] = dict() self.vars = vars def fork(self, is_deep_copy=False): - return Namespace(copy.copy(self.vars)) if not is_deep_copy else Namespace(copy.deepcopy(self.vars)) + return Namespace(copy.copy(self.vars)) if not is_deep_copy \ + else Namespace(copy.deepcopy(self.vars)) diff --git a/cloudferrylib/scheduler/scenario.py b/cloudferrylib/scheduler/scenario.py index bbec3303..59800c2d 100644 --- a/cloudferrylib/scheduler/scenario.py +++ b/cloudferrylib/scheduler/scenario.py @@ -47,7 +47,9 @@ def init_tasks(self, init={}): args = args[:-1] else: args_map = {} - tasks[task] = actions[tasks_file['tasks'][task][0]](init, *args, **args_map) + tasks[task] = actions[tasks_file['tasks'][task][0]](init, + *args, + **args_map) self.tasks = tasks def load_scenario(self, path_scenario=None): @@ -58,7 +60,8 @@ def load_scenario(self, path_scenario=None): self.namespace = migrate.get('namespace', {}) # "migration" yaml chain is responsible for migration self.migration = migrate.get("process") - # "preparation" yaml chain can be added to process pre-migration tasks + # "preparation" yaml chain can be added to process pre-migration + # tasks self.preparation = migrate.get("preparation") # "rollback" yaml chain can be added to rollback to previous state # in case of main chain failure @@ -110,11 +113,13 @@ def get_actions(self, mod): list_name_modules = map(lambda x: x.string.replace(".py", ""), modules_matches) - modules = [imp.load_source(name, path+'/%s.py' % name) for name in list_name_modules] + modules = [imp.load_source(name, path + '/%s.py' % name) + for name in list_name_modules] actions = {} for module in modules: for item in module.__dict__: - if inspect.isclass(module.__dict__[item]) and issubclass(module.__dict__[item], action.Action): + if inspect.isclass(module.__dict__[item]) and \ + issubclass(module.__dict__[item], action.Action): actions[item] = module.__dict__[item] return actions diff --git a/cloudferrylib/scheduler/task.py b/cloudferrylib/scheduler/task.py index a086a1a4..1c7d21af 100644 --- a/cloudferrylib/scheduler/task.py +++ b/cloudferrylib/scheduler/task.py @@ -65,7 +65,8 @@ def add_thread(self, other): def dual_link_with(self, other): other = other.go_start() self.next_element[0] = other - other.prev_element = self if not other.prev_element else other.prev_element + other.prev_element = self if not other.prev_element \ + else other.prev_element return other.go_end() @@ -103,5 +104,3 @@ def __repr__(self): class Task(BaseTask): pass - - diff --git a/cloudferrylib/scheduler/utils.py b/cloudferrylib/scheduler/utils.py index d48a9726..f6aaa6ef 100644 --- a/cloudferrylib/scheduler/utils.py +++ b/cloudferrylib/scheduler/utils.py @@ -22,4 +22,4 @@ def chain(net=None): if net: net.go_end() >> e_t b_t >> net.go_begin() - return b_t, e_t \ No newline at end of file + return b_t, e_t diff --git a/cloudferrylib/scheduler/utils/equ_instance.py b/cloudferrylib/scheduler/utils/equ_instance.py index 7c105b1d..e69b1e90 100644 --- a/cloudferrylib/scheduler/utils/equ_instance.py +++ b/cloudferrylib/scheduler/utils/equ_instance.py @@ -23,4 +23,4 @@ def __hash__(self): return hash(self.__class__.__name__) def __eq__(self, other): - return hash(self) == hash(other) \ No newline at end of file + return hash(self) == hash(other) diff --git a/cloudferrylib/utils/__init__.py b/cloudferrylib/utils/__init__.py index ca5e4a2b..e6a088e0 100644 --- a/cloudferrylib/utils/__init__.py +++ b/cloudferrylib/utils/__init__.py @@ -14,4 +14,4 @@ __author__ = 'mirrorcoder' -from utils import * \ No newline at end of file +from utils import * diff --git a/cloudferrylib/utils/console_cmd.py b/cloudferrylib/utils/console_cmd.py index b25a1793..b5d598e6 100644 --- a/cloudferrylib/utils/console_cmd.py +++ b/cloudferrylib/utils/console_cmd.py @@ -32,4 +32,4 @@ def __call__(self, *args): return BC(str(self) % args) def __str__(self): - return self.command \ No newline at end of file + return self.command diff --git a/cloudferrylib/utils/drivers/__init__.py b/cloudferrylib/utils/drivers/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/utils/drivers/__init__.py +++ b/cloudferrylib/utils/drivers/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py b/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py index 7ef26982..099d0b8b 100644 --- a/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py +++ b/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py @@ -16,8 +16,6 @@ from fabric.api import env from fabric.api import settings -from cloudferrylib.os.actions import utils as action_utils - from cloudferrylib.utils import cmd_cfg from cloudferrylib.utils import driver_transporter from cloudferrylib.utils import rbd_util @@ -42,16 +40,19 @@ def transfer(self, data, snapshot=None, snapshot_type=1): ssh_rbd_import_diff = ssh_cmd(host_dst, rbd_import_diff) if snapshot: - process_params = [snapshot['name'], data['path_src'], '-', '-', data['path_dst']] + process_params = [snapshot['name'], data['path_src'], '-', '-', + data['path_dst']] if snapshot_type == 1: rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_snap_cmd elif snapshot_type == 2: - rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_from_snap_cmd + rbd_export_diff = \ + rbd_util.RbdUtil.rbd_export_diff_from_snap_cmd process_params.insert(0, snapshot['prev_snapname']) elif snapshot_type == 3: rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_from_cmd else: - raise ValueError("Unsupported snapshot type %s", snapshot_type) + raise ValueError("Unsupported snapshot type %s", + snapshot_type) else: rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_cmd process_params = [data['path_src'], '-', '-', data['path_dst']] diff --git a/cloudferrylib/utils/drivers/ssh_chunks.py b/cloudferrylib/utils/drivers/ssh_chunks.py index ad672052..4e88c667 100644 --- a/cloudferrylib/utils/drivers/ssh_chunks.py +++ b/cloudferrylib/utils/drivers/ssh_chunks.py @@ -69,7 +69,7 @@ def verified_file_copy(src_runner, dst_runner, dst_user, src_path, dst_path, """ copy_failed = True attempt = 0 - while copy_failed and attempt < num_retries+1: + while copy_failed and attempt < num_retries + 1: attempt += 1 try: LOG.info("Copying file '%s' to '%s', attempt '%d'", @@ -155,7 +155,7 @@ def transfer(self, data): src_md5 = remote_md5_sum(src_runner, src_path) - num_blocks = int(math.ceil(float(file_size)/block_size)) + num_blocks = int(math.ceil(float(file_size) / block_size)) for i in xrange(num_blocks): part = os.path.basename(src_path) + '.part{i}'.format(i=i) diff --git a/cloudferrylib/utils/file_like_proxy.py b/cloudferrylib/utils/file_like_proxy.py index 584072f4..c5bde36f 100644 --- a/cloudferrylib/utils/file_like_proxy.py +++ b/cloudferrylib/utils/file_like_proxy.py @@ -27,7 +27,8 @@ class FileLikeProxy: def __init__(self, transfer_object, callback, speed_limit='1mb'): - self.__callback = callback if callback else lambda size, length, obj_id, name: True + self.__callback = callback if callback \ + else lambda size, length, obj_id, name: True self.resp = transfer_object['resource'].get_ref_image( transfer_object['id']) self.length = ( diff --git a/cloudferrylib/utils/proxy_client.py b/cloudferrylib/utils/proxy_client.py index b89f6895..fb207a6e 100644 --- a/cloudferrylib/utils/proxy_client.py +++ b/cloudferrylib/utils/proxy_client.py @@ -80,6 +80,8 @@ def __call__(self, *args, **kwargs): def __getattr__(self, name): attr = getattr(self.client, name) - if inspect.ismethod(attr) or (type(attr) is method_wrapper) or is_wrapping(attr): + if inspect.ismethod(attr) or \ + (type(attr) is method_wrapper) or \ + is_wrapping(attr): return Proxy(attr, self.retry, self.wait_time) return attr diff --git a/cloudferrylib/utils/qemu_img.py b/cloudferrylib/utils/qemu_img.py index a8d82594..e330d81f 100644 --- a/cloudferrylib/utils/qemu_img.py +++ b/cloudferrylib/utils/qemu_img.py @@ -27,7 +27,8 @@ class QemuImg(ssh_util.SshUtil): commit_cd_cmd = cmd_cfg.cd_cmd & commit_cmd convert_cmd = cmd_cfg.qemu_img_cmd("convert %s") convert_full_image_cmd = cmd_cfg.cd_cmd & convert_cmd("-f %s -O %s %s %s") - backing_file_cmd = cmd_cfg.qemu_img_cmd("info %s") >> cmd_cfg.grep_cmd("\"backing file\"") + backing_file_cmd = \ + cmd_cfg.qemu_img_cmd("info %s") >> cmd_cfg.grep_cmd("\"backing file\"") rebase_cmd = cmd_cfg.qemu_img_cmd("rebase -u -b %s %s") convert_cmd = convert_cmd("-O %s %s %s") @@ -50,12 +51,13 @@ def convert_image(self, cmd2 = cmd_cfg.move_cmd(path_to_image, baseimage_tmp, baseimage) - return self.execute(cmd1, host_compute), \ - self.execute(cmd2, host_compute) + return \ + self.execute(cmd1, host_compute), self.execute(cmd2, host_compute) def detect_backing_file(self, dest_disk_ephemeral, host_instance): cmd = self.backing_file_cmd(dest_disk_ephemeral) - return self.parsing_output_backing(self.execute(cmd=cmd, host_exec=host_instance, ignore_errors=True)) + return self.parsing_output_backing( + self.execute(cmd=cmd, host_exec=host_instance, ignore_errors=True)) @staticmethod def parsing_output_backing(output): diff --git a/cloudferrylib/utils/rbd_util.py b/cloudferrylib/utils/rbd_util.py index 61cc3a76..afc141e4 100644 --- a/cloudferrylib/utils/rbd_util.py +++ b/cloudferrylib/utils/rbd_util.py @@ -24,12 +24,14 @@ class RbdUtil(ssh_util.SshUtil): rbd_export_cmd = cmd_cfg.rbd_cmd("export %s %s") rbd_export_diff_cmd = cmd_cfg.rbd_cmd("export-diff %s %s") rbd_export_diff_snap_cmd = cmd_cfg.rbd_cmd("export-diff --snap %s %s %s") - rbd_export_diff_from_snap_cmd = cmd_cfg.rbd_cmd("export-diff --from-snap %s --snap %s %s %s") - rbd_export_diff_from_cmd = cmd_cfg.rbd_cmd("export-diff --from-snap %s %s %s") + rbd_export_diff_from_snap_cmd = \ + cmd_cfg.rbd_cmd("export-diff --from-snap %s --snap %s %s %s") + rbd_export_diff_from_cmd = \ + cmd_cfg.rbd_cmd("export-diff --from-snap %s %s %s") rbd_info_cmd = cmd_cfg.rbd_cmd("-p %s info %s --format %s") rbd_snap_rm = cmd_cfg.rbd_cmd("snap rm %s@%s") - #exmaple pool=compute filename = %s_disk.local % instance_id + # exmaple pool=compute filename = %s_disk.local % instance_id def rm(self, pool, filename, int_host=None): cmd = self.rbd_rm_cmd(pool, filename) return self.execute(cmd, int_host) @@ -38,27 +40,27 @@ def snap_rm(self, volume_path, snapshot_name, int_host=None): cmd = self.rbd_snap_rm(volume_path, snapshot_name) return self.execute(cmd, int_host) - #example image-format=2 output="-" filename=%s_disk.local + # example image-format=2 output="-" filename=%s_disk.local def rbd_import(self, image_format, output, filename, int_host=None): cmd = self.rbd_import_cmd(image_format, output, filename) return self.execute(cmd, int_host) - #example output="-" ceph_path=%s_disk.local + # example output="-" ceph_path=%s_disk.local def rbd_import_diff(self, output, ceph_path, int_host=None): cmd = self.rbd_import_cmd(output, ceph_path) return self.execute(cmd, int_host) - #example filename=volume-id1 output=- + # example filename=volume-id1 output=- def rbd_export(self, filename, output, int_host=None): cmd = self.rbd_export_cmd(filename, output) return self.execute(cmd, int_host) - #example ceph_path=volume-id1 output=- + # example ceph_path=volume-id1 output=- def rbd_export_diff(self, ceph_path, output, int_host=None): cmd = self.rbd_export_cmd(ceph_path, output) return self.execute(cmd, int_host) - #pool=images filename=image_id format=json + # pool=images filename=image_id format=json def rbd_get_info(self, pool, filename, format_output='json', int_host=None): cmd = self.rbd_info_cmd(pool, filename, format_output) diff --git a/cloudferrylib/utils/remote_runner.py b/cloudferrylib/utils/remote_runner.py index 989f0ec9..492a767d 100644 --- a/cloudferrylib/utils/remote_runner.py +++ b/cloudferrylib/utils/remote_runner.py @@ -28,7 +28,8 @@ class RemoteExecutionError(RuntimeError): class RemoteRunner(object): - def __init__(self, host, user, password=None, sudo=False, key=None, ignore_errors=False): + def __init__(self, host, user, password=None, sudo=False, key=None, + ignore_errors=False): self.host = host if key is None: key = cfglib.CONF.migrate.key_filename diff --git a/cloudferrylib/utils/ssh_util.py b/cloudferrylib/utils/ssh_util.py index 1e465e4f..d7b452c8 100644 --- a/cloudferrylib/utils/ssh_util.py +++ b/cloudferrylib/utils/ssh_util.py @@ -24,13 +24,15 @@ def __init__(self, cloud, config_migrate, host=None): self.host = host if host else cloud.host self.config_migrate = config_migrate - def execute(self, cmd, internal_host=None, host_exec=None, ignore_errors=False): + def execute(self, cmd, internal_host=None, host_exec=None, + ignore_errors=False): host = host_exec if host_exec else self.host - runner = remote_runner.RemoteRunner(host, - self.cloud.ssh_user, - password=self.cloud.ssh_sudo_password, - sudo=False, - ignore_errors=ignore_errors) + runner = \ + remote_runner.RemoteRunner(host, + self.cloud.ssh_user, + password=self.cloud.ssh_sudo_password, + sudo=False, + ignore_errors=ignore_errors) if internal_host: return self.execute_on_inthost(runner, str(cmd), internal_host) else: diff --git a/cloudferrylib/utils/utils.py b/cloudferrylib/utils/utils.py index be85df78..ec33a015 100644 --- a/cloudferrylib/utils/utils.py +++ b/cloudferrylib/utils/utils.py @@ -96,6 +96,9 @@ up_ssh_tunnel = None +SSH_CMD = \ + "ssh -oStrictHostKeyChecking=no -L %s:%s:22 -R %s:localhost:%s %s -Nf" + class ext_dict(dict): def __getattr__(self, name): @@ -105,8 +108,8 @@ def __getattr__(self, name): def get_snapshots_list_repository(path=PATH_TO_SNAPSHOTS): - path_source = path+'/source' - path_dest = path+'/dest' + path_source = path + '/source' + path_dest = path + '/dest' s = os.listdir(path_source) s.sort() source = [{'path': '%s/%s' % (path_source, f), @@ -137,19 +140,22 @@ def convert_to_dict(obj, ident=0, limit_ident=6): ident += 1 if type(obj) in primitive: return obj - if isinstance(obj, inspect.types.InstanceType) or (type(obj) not in (list, tuple, dict)): + if isinstance(obj, inspect.types.InstanceType) or \ + (type(obj) not in (list, tuple, dict)): if ident <= limit_ident: try: obj = obj.convert_to_dict() - except AttributeError as e: + except AttributeError: try: t = obj.__dict__ t['_type_class'] = str(obj.__class__) obj = t - except AttributeError as e: - return str(obj.__class__ if hasattr(obj, '__class__') else type(obj)) + except AttributeError: + return str(obj.__class__ if hasattr(obj, '__class__') + else type(obj)) else: - return str(obj.__class__ if hasattr(obj, '__class__') else type(obj)) + return str(obj.__class__ if hasattr(obj, '__class__') + else type(obj)) if type(obj) is dict: res = {} for item in obj: @@ -238,13 +244,16 @@ def render(self, name_file, args): config.dictConfig(yaml.load(logging_config)) LOGGER = logging.getLogger("CF") + def configure_logging(level): # redefine default logging level LOGGER.setLevel(level) + def get_log(name): return LOGGER + class StackCallFunctions(object): def __init__(self): self.stack_call_functions = [] @@ -288,7 +297,8 @@ def decorator(func): @wraps(func) def inner(*args, **kwargs): stack_call_functions.append(func.__name__, args, kwargs) - log.info("%s> Step %s" % ("- - "*stack_call_functions.depth(), func.__name__)) + log.info("%s> Step %s" % ("- - " * stack_call_functions.depth(), + func.__name__)) res = func(*args, **kwargs) stack_call_functions.pop(res) return res @@ -298,7 +308,8 @@ def inner(*args, **kwargs): class forward_agent(object): """ - Forwarding ssh-key for access on to source and destination clouds via ssh + Forwarding ssh-key for access on to source and + destination clouds via ssh """ def __init__(self, key_file): @@ -340,7 +351,8 @@ def __exit__(self, type, value, traceback): class wrapper_singletone_ssh_tunnel: def __init__(self, interval_ssh="9000-9999", locker=Lock()): - self.interval_ssh = [int(interval_ssh.split('-')[0]), int(interval_ssh.split('-')[1])] + self.interval_ssh = [int(interval_ssh.split('-')[0]), + int(interval_ssh.split('-')[1])] self.busy_port = [] self.locker = locker @@ -360,40 +372,48 @@ def free_port(self, port): if port in self.busy_port: self.busy_port.remove(port) - def __call__(self, address_dest_compute, address_dest_controller, host, **kwargs): - return up_ssh_tunnel_class(address_dest_compute, - address_dest_controller, - host, - self.get_free_port, - self.free_port) + def __call__(self, address_dest_compute, address_dest_controller, host, + **kwargs): + return UpSshTunnelClass(address_dest_compute, + address_dest_controller, + host, + self.get_free_port, + self.free_port) -class up_ssh_tunnel_class: +class UpSshTunnelClass: """ Up ssh tunnel on dest controller node for transferring data """ - def __init__(self, address_dest_compute, address_dest_controller, host, callback_get, callback_free): + def __init__(self, address_dest_compute, address_dest_controller, host, + callback_get, callback_free): self.address_dest_compute = address_dest_compute self.address_dest_controller = address_dest_controller self.get_free_port = callback_get self.remove_port = callback_free self.host = host - self.cmd = "ssh -oStrictHostKeyChecking=no -L %s:%s:22 -R %s:localhost:%s %s -Nf" + self.cmd = SSH_CMD def __enter__(self): self.port = self.get_free_port() with settings(host_string=self.host, connection_attempts=env.connection_attempts): - run(self.cmd % (self.port, self.address_dest_compute, self.port, self.port, + run(self.cmd % (self.port, + self.address_dest_compute, + self.port, + self.port, self.address_dest_controller) + " && sleep 2") return self.port def __exit__(self, type, value, traceback): with settings(host_string=self.host, connection_attempts=env.connection_attempts): - run(("pkill -f '"+self.cmd+"'") % (self.port, self.address_dest_compute, self.port, self.port, + run(("pkill -f '"+self.cmd+"'") % (self.port, + self.address_dest_compute, + self.port, + self.port, self.address_dest_controller)) time.sleep(2) self.remove_port(self.port) @@ -405,17 +425,19 @@ def __init__(self, checksum_source, checksum_dest): self.checksum_dest = checksum_dest def __str__(self): - return repr("Checksum of image source = %s Checksum of image dest = %s" % + return repr("Checksum of image source = %s" + + "Checksum of image dest = %s" % (self.checksum_source, self.checksum_dest)) -def render_info(info_values, template_path="templates", template_file="info.html"): +def render_info(info_values, template_path="templates", + template_file="info.html"): info_env = Environment(loader=FileSystemLoader(template_path)) template = info_env.get_template(template_file) return template.render(info_values) -def write_info(rendered_info, info_file = "source_info.html"): +def write_info(rendered_info, info_file="source_info.html"): with open(info_file, "wb") as ifile: ifile.write(rendered_info) @@ -439,7 +461,8 @@ def find_element_by_in(list_values, word): def init_singletones(cfg): - globals()['up_ssh_tunnel'] = wrapper_singletone_ssh_tunnel(cfg.migrate.ssh_transfer_port) + globals()['up_ssh_tunnel'] = wrapper_singletone_ssh_tunnel( + cfg.migrate.ssh_transfer_port) def get_disk_path(instance, blk_list, is_ceph_ephemeral=False, disk=DISK): @@ -470,7 +493,7 @@ def get_ips(init_host, compute_host, ssh_user): list_ips = [] for info in out.split(): try: - ip = ipaddr.IPAddress(info) + ipaddr.IPAddress(info) except ValueError: continue list_ips.append(info) diff --git a/condensation/process.py b/condensation/process.py index e0c0dfaf..916a868e 100644 --- a/condensation/process.py +++ b/condensation/process.py @@ -35,6 +35,7 @@ def process(nodes, flavors, vms, groups): if __name__ == "__main__": process(nodes=condense_utils.read_file(cfglib.CONF.condense.nodes_file), - flavors=condense_utils.read_file(cfglib.CONF.condense.flavors_file), + flavors=condense_utils.read_file( + cfglib.CONF.condense.flavors_file), vms=condense_utils.read_file(cfglib.CONF.condense.vms_file), groups=condense_utils.read_file(cfglib.CONF.condense.groups_file)) diff --git a/data_storage.py b/data_storage.py index d54839b8..bcadfd7b 100644 --- a/data_storage.py +++ b/data_storage.py @@ -83,6 +83,7 @@ def get(key, connection): def delete(key, connection): return connection.delete(key) + @redis_socket_to_kwargs def delete_batch(keys, connection): pipe = connection.pipeline() @@ -90,6 +91,7 @@ def delete_batch(keys, connection): pipe.delete(key) pipe.execute() + @redis_socket_to_kwargs def keys(pattern, connection): return connection.keys(pattern) diff --git a/tests/cloudferrylib/os/storage/test_cinder_netapp.py b/tests/cloudferrylib/os/storage/test_cinder_netapp.py index a9774c66..3940b65c 100644 --- a/tests/cloudferrylib/os/storage/test_cinder_netapp.py +++ b/tests/cloudferrylib/os/storage/test_cinder_netapp.py @@ -54,10 +54,8 @@ def setUp(self): self.fake_cloud.position = 'src' self.fake_cloud.resources = dict(identity=self.identity_mock) - with mock.patch( - 'cloudferrylib.os.storage.cinder_storage.mysql_connector'): - self.cinder_client = cinder_netapp.CinderNetApp(FAKE_CONFIG, - self.fake_cloud) + self.cinder_client = cinder_netapp.CinderNetApp(FAKE_CONFIG, + self.fake_cloud) def test_make_hostname(self): entry_0 = copy.deepcopy(FAKE_ENTRY_0) diff --git a/tests/cloudferrylib/os/storage/test_cinder_storage.py b/tests/cloudferrylib/os/storage/test_cinder_storage.py index ef5fe720..cb623824 100644 --- a/tests/cloudferrylib/os/storage/test_cinder_storage.py +++ b/tests/cloudferrylib/os/storage/test_cinder_storage.py @@ -58,10 +58,8 @@ def setUp(self): self.fake_cloud.resources = dict(identity=self.identity_mock, compute=self.compute_mock) - with mock.patch( - 'cloudferrylib.os.storage.cinder_storage.mysql_connector'): - self.cinder_client = cinder_storage.CinderStorage(FAKE_CONFIG, - self.fake_cloud) + self.cinder_client = cinder_storage.CinderStorage(FAKE_CONFIG, + self.fake_cloud) self.fake_volume_0 = mock.Mock() self.fake_volume_1 = mock.Mock() From e0d07e732617ebe9718e774c4697b3db62065934 Mon Sep 17 00:00:00 2001 From: Maksym Iarmak Date: Tue, 29 Sep 2015 12:36:26 +0300 Subject: [PATCH 8/9] [SAIC-458] Add networks segmentation id check Add networks segmentation id checking to the `check_networks` action. Patch provides handling duplicates in segmentation ids. If busy segmentation id is found on the destination, raise exception. --- cloudferrylib/os/actions/check_networks.py | 94 ++++++++++++++-- .../os/actions/test_check_networks.py | 106 +++++++++++++++--- 2 files changed, 171 insertions(+), 29 deletions(-) diff --git a/cloudferrylib/os/actions/check_networks.py b/cloudferrylib/os/actions/check_networks.py index 066c202c..92a4b149 100644 --- a/cloudferrylib/os/actions/check_networks.py +++ b/cloudferrylib/os/actions/check_networks.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and# # limitations under the License. -import ipaddr + import collections -import exceptions + +import ipaddr + +from cloudferrylib.base import exception from cloudferrylib.base.action import action from cloudferrylib.utils import utils @@ -24,9 +27,12 @@ class CheckNetworks(action.Action): """ - Check subnets overlapping and raise EnvironmentError if needed. - It must be done before actual migration in check section. - The action use filter search opts and must run after act_get_filter action + Check networks segmentation ID and subnets overlapping. + + Raise exception (AbortMigrationError) if overlapping has been found. + It must be done before actual migration in the 'preparation' section. + The action uses filtered search opts and must be run after 'act_get_filter' + action. """ def run(self, **kwargs): @@ -36,12 +42,22 @@ def run(self, **kwargs): search_opts = kwargs.get('search_opts_tenant', {}) src_info = NetworkInfo(src_net.read_info(**search_opts)) dst_info = NetworkInfo(dst_net.read_info(**search_opts)) + dst_seg_ids = dst_info.get_segmentation_ids() + for network in src_info.get_networks(): dst_net = dst_info.by_hash.get(network.hash) if dst_net: - LOG.debug("src network %s, dst network %s" % + # Current network matches with network on DST + # Have the same networks on SRC and DST + LOG.debug("SRC network: '%s', DST network: '%s'" % (network.id, dst_net.id)) network.check_network_overlapping(dst_net) + else: + # Current network does not match with any network on DST + # Check Segmentation ID overlapping with DST + LOG.debug("Check segmentation ID for SRC network: '%s'", + network.id) + network.check_segmentation_id_overlapping(dst_seg_ids) class NetworkInfo(object): @@ -59,6 +75,40 @@ def __init__(self, info): def get_networks(self): return self.by_hash.values() + def get_segmentation_ids(self): + """Get busy segmentation IDs. + + We need to handle duplicates in segmentation ids. + Neutron has different validation rules for different network types. + + For 'gre' and 'vxlan' network types there is no strong requirement + for 'physical_network' attribute, if we want to have + 'segmentation_id', because traffic is encapsulated in L3 packets. + + For 'vlan' and 'flat' network types there is a strong requirement for + 'physical_network' attribute, if we want to have 'segmentation_id'. + + :result: Dictionary with busy segmentation IDs. + Hash is used with structure {"gre": [1, 2, ...], + "vlan": [1, 2, ...]} + """ + + used_seg_ids = {} + networks = self.by_id.values() + + for net in networks: + network_has_segmentation_id = ( + net.info["provider:physical_network"] or + (net.network_type in ['gre', 'vxlan'])) + + if network_has_segmentation_id: + if net.network_type not in used_seg_ids: + used_seg_ids[net.network_type] = [] + if net.seg_id: + used_seg_ids[net.network_type].append(net.seg_id) + + return used_seg_ids + class Network(object): def __init__(self, info): @@ -68,24 +118,27 @@ def __init__(self, info): self.id = info['id'] self.hash = info['res_hash'] + self.network_type = self.info['provider:network_type'] + self.seg_id = self.info["provider:segmentation_id"] + def add_subnet(self, info): self.subnets.append(info) self.subnets_hash.add(info['res_hash']) def check_network_overlapping(self, network): for subnet in network.subnets: - LOG.debug("work on src subnet %s" % subnet['id']) + LOG.debug("Work with SRC subnet: '%s'" % subnet['id']) if self.is_subnet_eq(subnet): - LOG.debug("We have the same on dst by hash") + LOG.debug("We have the same subnet on DST by hash") continue overlapping_subnet = self.get_overlapping_subnet(subnet) if overlapping_subnet: - message = ("Subnet %s in network %s on src overlap " - "subnet %s in network %s on dst" % ( + message = ("Subnet '%s' in network '%s' on SRC overlaps with " + "subnet '%s' in network '%s' on DST" % ( overlapping_subnet, self.id, subnet['id'], network.id)) LOG.error(message) - raise exceptions.EnvironmentError(message) + raise exception.AbortMigrationError(message) def is_subnet_eq(self, subnet): return subnet['res_hash'] in self.subnets_hash @@ -94,5 +147,22 @@ def get_overlapping_subnet(self, subnet): cidr = ipaddr.IPNetwork(subnet['cidr']) for self_subnet in self.subnets: self_cidr = ipaddr.IPNetwork(self_subnet['cidr']) - if (cidr.Contains(self_cidr) or self_cidr.Contains(cidr)): + if cidr.Contains(self_cidr) or self_cidr.Contains(cidr): return self_subnet['id'] + + def check_segmentation_id_overlapping(self, dst_seg_ids): + """ + Check if segmentation ID of current network overlaps with destination. + + :param dst_seg_ids: Dictionary with busy segmentation IDs on DST + """ + + if self.network_type not in dst_seg_ids: + return + + if self.seg_id in dst_seg_ids[self.network_type]: + message = ("Segmentation ID '%s' (network type = '%s', " + "network ID = '%s') is already busy on the destination " + "cloud.") % (self.seg_id, self.network_type, self.id) + LOG.error(message) + raise exception.AbortMigrationError(message) diff --git a/tests/cloudferrylib/os/actions/test_check_networks.py b/tests/cloudferrylib/os/actions/test_check_networks.py index 684f11d0..aa3c777d 100644 --- a/tests/cloudferrylib/os/actions/test_check_networks.py +++ b/tests/cloudferrylib/os/actions/test_check_networks.py @@ -13,18 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. + import mock -import exceptions -from tests import test +from cloudferrylib.base import exception from cloudferrylib.os.actions import check_networks +from tests import test class CheckNetworksTestCase(test.TestCase): - - def setUp(self): - super(CheckNetworksTestCase, self).setUp() - @staticmethod def get_action(src_net_info, dst_net_info): fake_src_net = mock.Mock() @@ -52,7 +49,10 @@ def test_all_empty(self): def test_empty_dst(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', @@ -63,13 +63,19 @@ def test_empty_dst(self): def test_equals_networks(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', 'id': 'sub1'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id2', @@ -78,7 +84,10 @@ def test_equals_networks(self): def test_equals_and_new_networks(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', @@ -88,7 +97,10 @@ def test_equals_and_new_networks(self): 'network_id': 'id1', 'id': 'sub2'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id2', @@ -97,32 +109,92 @@ def test_equals_and_new_networks(self): def test_diff(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', 'id': 'sub1'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 3, 'network_id': 'id2', 'id': 'sub2'}]} action = self.get_action(src_net_info, dst_net_info) - self.assertRaises(exceptions.EnvironmentError, action.run) + self.assertRaises(exception.AbortMigrationError, action.run) def test_overlap(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/28', 'res_hash': 2, 'network_id': 'id1', 'id': 'sub1'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 3, 'network_id': 'id2', 'id': 'sub2'}]} action = self.get_action(src_net_info, dst_net_info) - self.assertRaises(exceptions.EnvironmentError, action.run) + self.assertRaises(exception.AbortMigrationError, action.run) + + def test_check_segmentation_id_overlapping_no_dst_networks(self): + src_net_info = {'networks': [{'id': 'id1', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + dst_net_info = {'networks': [], + 'subnets': []} + + self.get_action(src_net_info, dst_net_info).run() + + def test_check_segmentation_id_overlapping_same_network(self): + src_net_info = {'networks': [{'id': 'id1', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + dst_net_info = {'networks': [{'id': 'id2', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + self.get_action(src_net_info, dst_net_info).run() + + def test_check_segmentation_id_overlapping_different_network(self): + src_net_info = {'networks': [{'id': 'id1', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + dst_net_info = {'networks': [{'id': 'id2', + 'res_hash': 2, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + action = self.get_action(src_net_info, dst_net_info) + self.assertRaises(exception.AbortMigrationError, action.run) From 8ea41f63b2e8d83ebc19694353671c41abe930c4 Mon Sep 17 00:00:00 2001 From: Stas Egorov Date: Tue, 29 Sep 2015 16:40:50 +0300 Subject: [PATCH 9/9] Fixed pep8 warnings --- cfglib.py | 4 +- cloudferrylib/utils/utils.py | 11 ++-- condensation/process.py | 2 +- devlab/tests/data_collector.py | 2 +- devlab/tests/generate_load_real_env.py | 2 +- .../actions/test_check_instance_networks.py | 4 +- .../cloudferrylib/os/compute/test_libvirt.py | 64 +++++++++---------- .../os/identity/test_keystone.py | 3 +- .../os/storage/test_cinder_storage.py | 3 +- 9 files changed, 49 insertions(+), 46 deletions(-) diff --git a/cfglib.py b/cfglib.py index 8a405dc4..5a013324 100644 --- a/cfglib.py +++ b/cfglib.py @@ -160,8 +160,8 @@ 'If not set uses `[dst] db_host` config option.'), cfg.BoolOpt('optimize_user_role_fetch', default=True, help="Uses low-level DB requests if set to True, " - "may be incompatible with more recent versions of " - "Keystone. Tested on grizzly, icehouse and juno."), + "may be incompatible with more recent versions of " + "Keystone. Tested on grizzly, icehouse and juno."), cfg.IntOpt('ssh_connection_attempts', default=3, help='Number of times CloudFerry will attempt to connect when ' 'connecting to a new server via SSH.'), diff --git a/cloudferrylib/utils/utils.py b/cloudferrylib/utils/utils.py index ec33a015..97af27cf 100644 --- a/cloudferrylib/utils/utils.py +++ b/cloudferrylib/utils/utils.py @@ -410,11 +410,12 @@ def __enter__(self): def __exit__(self, type, value, traceback): with settings(host_string=self.host, connection_attempts=env.connection_attempts): - run(("pkill -f '"+self.cmd+"'") % (self.port, - self.address_dest_compute, - self.port, - self.port, - self.address_dest_controller)) + run(("pkill -f '" + self.cmd + "'") % + (self.port, + self.address_dest_compute, + self.port, + self.port, + self.address_dest_controller)) time.sleep(2) self.remove_port(self.port) diff --git a/condensation/process.py b/condensation/process.py index 916a868e..f2e19a54 100644 --- a/condensation/process.py +++ b/condensation/process.py @@ -29,7 +29,7 @@ def process(nodes, flavors, vms, groups): # read files with nova data and node data LOG.info("started creating schedule for node condensation") - source = cloud.Cloud.from_dicts('source', nodes, flavors, vms, groups) + source = cloud.Cloud.from_dicts('source', nodes, flavors, vms, groups) source.migrate_to(cloud.Cloud('destination')) diff --git a/devlab/tests/data_collector.py b/devlab/tests/data_collector.py index 109de0ac..be8dc749 100644 --- a/devlab/tests/data_collector.py +++ b/devlab/tests/data_collector.py @@ -133,7 +133,7 @@ def nova_collector(self, destination, *args): 'novaclient', arg, 'list') return collected_items - def neutron_collector(self, destination, *args): + def neutron_collector(self, destination, *args): """ Neutron data collector method. """ diff --git a/devlab/tests/generate_load_real_env.py b/devlab/tests/generate_load_real_env.py index 05a4226d..6d2e70da 100644 --- a/devlab/tests/generate_load_real_env.py +++ b/devlab/tests/generate_load_real_env.py @@ -233,7 +233,7 @@ def clean_objects(self): parser = argparse.ArgumentParser( description='Script to generate load for Openstack and delete ' 'generated objects') - parser.add_argument('--clean', action='store_true', + parser.add_argument('--clean', action='store_true', help='clean objects described in real_env_conf.ini') args = parser.parse_args() if args.clean: diff --git a/tests/cloudferrylib/os/actions/test_check_instance_networks.py b/tests/cloudferrylib/os/actions/test_check_instance_networks.py index c2ddefd8..0c44237a 100644 --- a/tests/cloudferrylib/os/actions/test_check_instance_networks.py +++ b/tests/cloudferrylib/os/actions/test_check_instance_networks.py @@ -52,7 +52,7 @@ def test_net_empty(self): action.run() def test_instance_empty(self): - action = self.get_action(FakeSubnets().add('1.1.1.1/24',True).toMap(), + action = self.get_action(FakeSubnets().add('1.1.1.1/24', True).toMap(), []) action.run() @@ -70,7 +70,7 @@ def test_negative(self): 'name-10.0.0.1', action.run) - def test_negative(self): + def test_negative_ext(self): action = self.get_action(FakeSubnets().add('10.0.0.0/24', True) .add('100.0.0.0/24', True).toMap(), ['10.0.0.1', '10.0.0.2', '10.0.0.4']) diff --git a/tests/cloudferrylib/os/compute/test_libvirt.py b/tests/cloudferrylib/os/compute/test_libvirt.py index 0081a608..1804f328 100644 --- a/tests/cloudferrylib/os/compute/test_libvirt.py +++ b/tests/cloudferrylib/os/compute/test_libvirt.py @@ -84,7 +84,7 @@ def test_backing_file_returns_none_in_case_of_error(self): grizzly_xml = """ instance-00000008 - 7f9cfeab-05c6-4dcc-91d4-3074383ff73d + 7f9cfeab-05c6-4dcc 524288 524288 1 @@ -94,7 +94,7 @@ def test_backing_file_returns_none_in_case_of_error(self): OpenStack Nova 2013.1.5 72827a02-9e88-41b9-82de-ec2ae162a93c - 7f9cfeab-05c6-4dcc-91d4-3074383ff73d + 7f9cfeab-05c6-4dcc @@ -117,12 +117,12 @@ def test_backing_file_returns_none_in_case_of_error(self): /usr/bin/qemu-system-x86_64 - + -
+
-
+
@@ -131,7 +131,7 @@ def test_backing_file_returns_none_in_case_of_error(self): -
+
@@ -139,7 +139,7 @@ def test_backing_file_returns_none_in_case_of_error(self): -
+
@@ -147,31 +147,31 @@ def test_backing_file_returns_none_in_case_of_error(self): -
+
- + - + - +