diff --git a/addons/__init__.py b/addons/__init__.py index 71f21bcd..e69de29b 100644 --- a/addons/__init__.py +++ b/addons/__init__.py @@ -1 +0,0 @@ -import aws_plugin \ No newline at end of file diff --git a/addons/aws_plugin/__init__.py b/addons/aws_plugin/__init__.py index 029374b7..4b32c42c 100644 --- a/addons/aws_plugin/__init__.py +++ b/addons/aws_plugin/__init__.py @@ -17,4 +17,4 @@ cfg_for_reg = [ (migrate, migrate_opts) -] \ No newline at end of file +] diff --git a/cfglib.py b/cfglib.py index 8eafc596..5a013324 100644 --- a/cfglib.py +++ b/cfglib.py @@ -159,9 +159,12 @@ help='IP or hostname used for creating MySQL dump for rollback.' 'If not set uses `[dst] db_host` config option.'), cfg.BoolOpt('optimize_user_role_fetch', default=True, - help=("Uses low-level DB requests if set to True, " - "may be incompatible with more recent versions of " - "Keystone. Tested on grizzly, icehouse and juno.")) + help="Uses low-level DB requests if set to True, " + "may be incompatible with more recent versions of " + "Keystone. Tested on grizzly, icehouse and juno."), + cfg.IntOpt('ssh_connection_attempts', default=3, + help='Number of times CloudFerry will attempt to connect when ' + 'connecting to a new server via SSH.'), ] mail = cfg.OptGroup(name='mail', diff --git a/cloud/__init__.py b/cloud/__init__.py index 220f7afd..9bf269fa 100644 --- a/cloud/__init__.py +++ b/cloud/__init__.py @@ -1,3 +1,3 @@ import cloud import cloud_ferry -import os2os \ No newline at end of file +import os2os diff --git a/cloud/cloud.py b/cloud/cloud.py index e8560ec5..0eeb56ab 100644 --- a/cloud/cloud.py +++ b/cloud/cloud.py @@ -37,7 +37,8 @@ def __init__(self, resources, position, config): def getIpSsh(self): return self.cloud_config.cloud.ssh_host \ - if self.cloud_config.cloud.ssh_host else self.cloud_config.cloud.host + if self.cloud_config.cloud.ssh_host \ + else self.cloud_config.cloud.host @staticmethod def make_cloud_config(config, position): @@ -75,8 +76,10 @@ def make_resource_config(config, position, cloud_config, resource_name): def get_db_method_create_connection(resource, config): def get_db_connection(db_name): conf_res = getattr(config, resource) - conf = {k: getattr(conf_res, k, None) if getattr(conf_res, k, None) else config.mysql[k] - for k in config.mysql.keys()} + conf = { + k: getattr(conf_res, k, None) if getattr(conf_res, k, None) + else config.mysql[k] + for k in config.mysql.keys()} db_name_use = getattr(conf_res, 'database_name')\ if getattr(conf_res, 'database_name', None) else db_name return mysql_connector.MysqlConnector(conf, db_name_use) @@ -85,16 +88,24 @@ def get_db_connection(db_name): def init_resources(self, cloud_config): resources = self.resources self.resources = dict() - self.rbd_util = rbd_util.RbdUtil(getattr(self.config, "%s" % self.position), self.config.migrate) - self.qemu_img = qemu_img.QemuImg(getattr(self.config, "%s" % self.position), self.config.migrate) - self.ssh_util = ssh_util.SshUtil(getattr(self.config, "%s" % self.position), self.config.migrate) - - identity_conf = self.make_resource_config(self.config, self.position, - cloud_config, 'identity') + self.rbd_util = rbd_util.RbdUtil(getattr(self.config, + "%s" % self.position), + self.config.migrate) + self.qemu_img = qemu_img.QemuImg(getattr(self.config, + "%s" % self.position), + self.config.migrate) + self.ssh_util = ssh_util.SshUtil(getattr(self.config, + "%s" % self.position), + self.config.migrate) + + ident_conf = self.make_resource_config(self.config, + self.position, + cloud_config, + 'identity') self.mysql_connector = self.get_db_method_create_connection('identity', - identity_conf) + ident_conf) identity = resources['identity']( - identity_conf, + ident_conf, self) self.resources['identity'] = identity @@ -107,7 +118,8 @@ def init_resources(self, cloud_config): self.position, cloud_config, resource) - self.mysql_connector = self.get_db_method_create_connection(resource, - resource_config) + self.mysql_connector = \ + self.get_db_method_create_connection(resource, + resource_config) self.resources[resource] = resources[resource]( resource_config, self) diff --git a/cloud/os2os.py b/cloud/os2os.py index 0a93cfa4..ac3fc03e 100644 --- a/cloud/os2os.py +++ b/cloud/os2os.py @@ -15,13 +15,13 @@ import cloud import cloud_ferry -from cloudferrylib.base.action import copy_var, rename_info, merge, is_end_iter, get_info_iter +from cloudferrylib.base.action import copy_var, rename_info, \ + merge, is_end_iter, get_info_iter from cloudferrylib.os.actions import identity_transporter from cloudferrylib.scheduler import scheduler from cloudferrylib.scheduler import namespace from cloudferrylib.scheduler import cursor from cloudferrylib.os.image import glance_image -from cloudferrylib.os.storage import cinder_storage from cloudferrylib.os.network import neutron from cloudferrylib.os.identity import keystone from cloudferrylib.os.compute import nova_compute @@ -120,20 +120,23 @@ def migrate(self, scenario=None): # "rollback" - is cursor that points to tasks must be processed # in case of "migration" failure if not scenario: - process_migration = {"migration": cursor.Cursor(self.process_migrate())} + process_migration = { + "migration": cursor.Cursor(self.process_migrate())} else: scenario.init_tasks(self.init) scenario.load_scenario() - process_migration = {k: cursor.Cursor(v) for k, v in scenario.get_net().items()} - scheduler_migr = scheduler.Scheduler(namespace=namespace_scheduler, **process_migration) + process_migration = {k: cursor.Cursor(v) + for k, v in scenario.get_net().items()} + scheduler_migr = scheduler.Scheduler(namespace=namespace_scheduler, + **process_migration) scheduler_migr.start() def process_migrate(self): check_environment = self.check_environment() task_resources_transporting = self.transport_resources() transport_instances_and_dependency_resources = self.migrate_instances() - return (check_environment >> - task_resources_transporting >> + return (check_environment >> + task_resources_transporting >> transport_instances_and_dependency_resources) def check_environment(self): @@ -165,16 +168,27 @@ def migrate_instances(self): name_result = 'info_result' name_backup = 'info_backup' name_iter = 'info_iter' - save_result = self.save_result(name_data, name_result, name_result, 'instances') + save_result = self.save_result(name_data, + name_result, + name_result, + 'instances') trans_one_inst = self.migrate_process_instance() - init_iteration_instance = self.init_iteration_instance(name_data, name_backup, name_iter) - act_check_needed_compute_resources = check_needed_compute_resources.CheckNeededComputeResources(self.init) + init_iteration_instance = self.init_iteration_instance(name_data, + name_backup, + name_iter) + act_check_needed_compute_resources = \ + check_needed_compute_resources.CheckNeededComputeResources( + self.init) act_get_filter = get_filter.GetFilter(self.init) - act_get_info_inst = get_info_instances.GetInfoInstances(self.init, cloud='src_cloud') + act_get_info_inst = \ + get_info_instances.GetInfoInstances(self.init, + cloud='src_cloud') act_cleanup_images = cleanup_images.CleanupImages(self.init) get_next_instance = get_info_iter.GetInfoIter(self.init) check_for_instance = check_instances.CheckInstances(self.init) - rename_info_iter = rename_info.RenameInfo(self.init, name_result, name_data) + rename_info_iter = rename_info.RenameInfo(self.init, + name_result, + name_data) is_instances = is_end_iter.IsEndIter(self.init) transport_instances_and_dependency_resources = \ @@ -192,12 +206,14 @@ def migrate_instances(self): return transport_instances_and_dependency_resources def init_iteration_instance(self, data, name_backup, name_iter): - init_iteration_instance = copy_var.CopyVar(self.init, data, name_backup, True) >>\ - create_reference.CreateReference(self.init, data, name_iter) + init_iteration_instance = \ + copy_var.CopyVar(self.init, data, name_backup, True) >>\ + create_reference.CreateReference(self.init, data, name_iter) return init_iteration_instance def migration_images(self): - act_get_info_images = get_info_images.GetInfoImages(self.init, cloud='src_cloud') + act_get_info_images = get_info_images.GetInfoImages(self.init, + cloud='src_cloud') act_deploy_images = copy_g2g.CopyFromGlanceToGlance(self.init) return act_get_info_images >> act_deploy_images @@ -206,40 +222,64 @@ def save_result(self, data1, data2, result, resources_name): def transport_volumes_by_instance(self): act_copy_g2g_vols = copy_g2g.CopyFromGlanceToGlance(self.init) - act_convert_c_to_v = convert_compute_to_volume.ConvertComputeToVolume(self.init, cloud='src_cloud') - act_convert_v_to_i = convert_volume_to_image.ConvertVolumeToImage(self.init, cloud='src_cloud') - act_convert_i_to_v = convert_image_to_volume.ConvertImageToVolume(self.init, cloud='dst_cloud') - act_convert_v_to_c = convert_volume_to_compute.ConvertVolumeToCompute(self.init, cloud='dst_cloud') + act_convert_c_to_v = \ + convert_compute_to_volume.ConvertComputeToVolume(self.init, + cloud='src_cloud') + act_convert_v_to_i = \ + convert_volume_to_image.ConvertVolumeToImage(self.init, + cloud='src_cloud') + act_convert_i_to_v = \ + convert_image_to_volume.ConvertImageToVolume(self.init, + cloud='dst_cloud') + act_convert_v_to_c = \ + convert_volume_to_compute.ConvertVolumeToCompute(self.init, + cloud='dst_cloud') task_convert_c_to_v_to_i = act_convert_c_to_v >> act_convert_v_to_i task_convert_i_to_v_to_c = act_convert_i_to_v >> act_convert_v_to_c - return task_convert_c_to_v_to_i >> act_copy_g2g_vols >> task_convert_i_to_v_to_c + return task_convert_c_to_v_to_i >> act_copy_g2g_vols >>\ + task_convert_i_to_v_to_c def transport_volumes_by_instance_via_ssh(self): - act_convert_c_to_v = convert_compute_to_volume.ConvertComputeToVolume(self.init, cloud='src_cloud') - act_rename_inst_vol_src = create_reference.CreateReference(self.init, 'storage_info', - 'src_storage_info') - act_convert_v_to_c = convert_volume_to_compute.ConvertVolumeToCompute(self.init, cloud='dst_cloud') - act_rename_inst_vol_dst = create_reference.CreateReference(self.init, 'storage_info', - 'dst_storage_info') - act_inst_vol_data_map = prepare_volumes_data_map.PrepareVolumesDataMap(self.init, - 'src_storage_info', - 'dst_storage_info') - act_deploy_inst_volumes = deploy_volumes.DeployVolumes(self.init, cloud='dst_cloud') - - act_inst_vol_transport_data = task_transfer.TaskTransfer(self.init, - 'SSHCephToCeph', - input_info='storage_info') - - act_deploy_snapshots = deploy_snapshots.DeployVolSnapshots(self.init, cloud='dst_cloud') - act_convert_v_to_c + act_convert_c_to_v = \ + convert_compute_to_volume.ConvertComputeToVolume(self.init, + cloud='src_cloud') + act_rename_inst_vol_src = \ + create_reference.CreateReference(self.init, + 'storage_info', + 'src_storage_info') + act_convert_v_to_c = \ + convert_volume_to_compute.ConvertVolumeToCompute(self.init, + cloud='dst_cloud') + act_rename_inst_vol_dst = \ + create_reference.CreateReference(self.init, 'storage_info', + 'dst_storage_info') + act_inst_vol_data_map = \ + prepare_volumes_data_map.PrepareVolumesDataMap(self.init, + 'src_storage_info', + 'dst_storage_info') + act_deploy_inst_volumes = \ + deploy_volumes.DeployVolumes(self.init, cloud='dst_cloud') + + act_inst_vol_transport_data = \ + task_transfer.TaskTransfer(self.init, + 'SSHCephToCeph', + input_info='storage_info') + + act_deploy_snapshots = \ + deploy_snapshots.DeployVolSnapshots( + self.init, + cloud='dst_cloud') - act_convert_v_to_c is_snapshots = is_option.IsOption(self.init, 'keep_volume_snapshots') task_get_inst_vol_info = act_convert_c_to_v >> act_rename_inst_vol_src - task_deploy_inst_vol = act_deploy_inst_volumes >> act_rename_inst_vol_dst + task_deploy_inst_vol = act_deploy_inst_volumes >> \ + act_rename_inst_vol_dst return task_get_inst_vol_info >> \ - task_deploy_inst_vol >> act_inst_vol_data_map >> \ - (is_snapshots | act_deploy_snapshots | act_inst_vol_transport_data) >> \ - act_convert_v_to_c + task_deploy_inst_vol >> act_inst_vol_data_map >> \ + (is_snapshots | act_deploy_snapshots | + act_inst_vol_transport_data) >> \ + act_convert_v_to_c def transport_available_volumes_via_ssh(self): is_volume_snapshots = is_option.IsOption(self.init, @@ -247,68 +287,87 @@ def transport_available_volumes_via_ssh(self): final_action = fake_action.FakeAction(self.init) - act_get_info_available_volumes = get_info_volumes.GetInfoVolumes(self.init, - cloud='src_cloud', - search_opts={'status': 'available'}) - act_rename_vol_src = create_reference.CreateReference(self.init, - 'storage_info', - 'src_storage_info') - task_get_available_vol_info = act_get_info_available_volumes >> act_rename_vol_src + act_get_info_available_volumes = \ + get_info_volumes.GetInfoVolumes(self.init, + cloud='src_cloud', + search_opts={ + 'status': 'available'}) + act_rename_vol_src = \ + create_reference.CreateReference(self.init, + 'storage_info', + 'src_storage_info') + task_get_available_vol_info = \ + act_get_info_available_volumes >> act_rename_vol_src act_deploy_vol = deploy_volumes.DeployVolumes(self.init, cloud='dst_cloud') - act_rename_vol_dst = create_reference.CreateReference(self.init, - 'storage_info', - 'dst_storage_info') + act_rename_vol_dst = \ + create_reference.CreateReference(self.init, + 'storage_info', + 'dst_storage_info') task_deploy_available_volumes = act_deploy_vol >> act_rename_vol_dst - act_vol_data_map = prepare_volumes_data_map.PrepareVolumesDataMap(self.init, - 'src_storage_info', - 'dst_storage_info') + act_vol_data_map = \ + prepare_volumes_data_map.PrepareVolumesDataMap(self.init, + 'src_storage_info', + 'dst_storage_info') act_vol_transport_data = \ - task_transfer.TaskTransfer(self.init, - 'SSHCephToCeph', - input_info='storage_info') - final_action + task_transfer.TaskTransfer( + self.init, + 'SSHCephToCeph', + input_info='storage_info') - final_action act_deploy_vol_snapshots = \ - deploy_snapshots.DeployVolSnapshots(self.init,cloud='dst_cloud') - final_action + deploy_snapshots.DeployVolSnapshots( + self.init, cloud='dst_cloud') - final_action return task_get_available_vol_info >> \ - task_deploy_available_volumes >> \ - act_vol_data_map >> \ - (is_volume_snapshots | act_deploy_vol_snapshots | act_vol_transport_data) \ - >> final_action + task_deploy_available_volumes >> \ + act_vol_data_map >> \ + (is_volume_snapshots | act_deploy_vol_snapshots | + act_vol_transport_data) \ + >> final_action def transport_object_storage(self): - act_get_objects_info = get_info_objects.GetInfoObjects(self.init, - cloud='src_cloud') - act_transfer_objects = copy_object2object.CopyFromObjectToObject(self.init, - src_cloud='src_cloud', - dst_cloud='dst_cloud') + act_get_objects_info = \ + get_info_objects.GetInfoObjects(self.init, + cloud='src_cloud') + act_transfer_objects = \ + copy_object2object.CopyFromObjectToObject(self.init, + src_cloud='src_cloud', + dst_cloud='dst_cloud') task_transfer_objects = act_get_objects_info >> act_transfer_objects return task_transfer_objects def transport_cold_data(self): - act_identity_trans = identity_transporter.IdentityTransporter(self.init) - task_transport_available_volumes = self.transport_available_volumes_via_ssh() + act_identity_trans = \ + identity_transporter.IdentityTransporter(self.init) + task_transport_available_volumes = \ + self.transport_available_volumes_via_ssh() task_transport_objects = self.transport_object_storage() task_transport_images = self.migration_images() return act_identity_trans >> \ - task_transport_available_volumes \ - >> task_transport_objects \ - >> task_transport_images + task_transport_available_volumes \ + >> task_transport_objects \ + >> task_transport_images def transport_resources(self): - act_identity_trans = identity_transporter.IdentityTransporter(self.init) + act_identity_trans = \ + identity_transporter.IdentityTransporter(self.init) task_images_trans = self.migration_images() - act_comp_res_trans = transport_compute_resources.TransportComputeResources(self.init) + act_comp_res_trans = \ + transport_compute_resources.TransportComputeResources(self.init) act_network_trans = networks_transporter.NetworkTransporter(self.init) - return act_identity_trans >> task_images_trans >> act_network_trans >> act_comp_res_trans + return act_identity_trans >> task_images_trans >> act_network_trans >>\ + act_comp_res_trans def migrate_images_by_instances(self): - act_conv_comp_img = convert_compute_to_image.ConvertComputeToImage(self.init, cloud='src_cloud') - act_conv_image_comp = convert_image_to_compute.ConvertImageToCompute(self.init) + act_conv_comp_img = \ + convert_compute_to_image.ConvertComputeToImage(self.init, + cloud='src_cloud') + act_conv_image_comp = \ + convert_image_to_compute.ConvertImageToCompute(self.init) act_copy_inst_images = copy_g2g.CopyFromGlanceToGlance(self.init) return act_conv_comp_img >> act_copy_inst_images >> act_conv_image_comp @@ -324,20 +383,37 @@ def migrate_resources_by_instance_via_ssh(self): def migrate_instance(self): act_map_com_info = map_compute_info.MapComputeInfo(self.init) - act_net_prep = prepare_networks.PrepareNetworks(self.init, cloud='dst_cloud') + act_net_prep = prepare_networks.PrepareNetworks(self.init, + cloud='dst_cloud') act_deploy_instances = transport_instance.TransportInstance(self.init) - act_i_to_f = load_compute_image_to_file.LoadComputeImageToFile(self.init, cloud='dst_cloud') - act_merge = merge_base_and_diff.MergeBaseDiff(self.init, cloud='dst_cloud') - act_convert_image = convert_file.ConvertFile(self.init, cloud='dst_cloud') - act_f_to_i = upload_file_to_image.UploadFileToImage(self.init, cloud='dst_cloud') - act_transfer_file = task_transfer.TaskTransfer(self.init, 'SSHCephToFile', - resource_name=utl.INSTANCES_TYPE, - resource_root_name=utl.DIFF_BODY) - act_f_to_i_after_transfer = upload_file_to_image.UploadFileToImage(self.init, cloud='dst_cloud') - act_is_not_trans_image = is_not_transport_image.IsNotTransportImage(self.init, cloud='src_cloud') - act_is_not_merge_diff = is_not_merge_diff.IsNotMergeDiff(self.init, cloud='src_cloud') - act_post_transport_instance = post_transport_instance.PostTransportInstance(self.init, cloud='dst_cloud') - act_transport_ephemeral = transport_ephemeral.TransportEphemeral(self.init, cloud='dst_cloud') + act_i_to_f = \ + load_compute_image_to_file.LoadComputeImageToFile( + self.init, + cloud='dst_cloud') + act_merge = \ + merge_base_and_diff.MergeBaseDiff(self.init, cloud='dst_cloud') + act_convert_image = convert_file.ConvertFile(self.init, + cloud='dst_cloud') + act_f_to_i = upload_file_to_image.UploadFileToImage(self.init, + cloud='dst_cloud') + act_transfer_file = \ + task_transfer.TaskTransfer(self.init, + 'SSHCephToFile', + resource_name=utl.INSTANCES_TYPE, + resource_root_name=utl.DIFF_BODY) + act_f_to_i_after_transfer = \ + upload_file_to_image.UploadFileToImage(self.init, + cloud='dst_cloud') + act_is_not_trans_image = \ + is_not_transport_image.IsNotTransportImage(self.init, + cloud='src_cloud') + act_is_not_merge_diff = \ + is_not_merge_diff.IsNotMergeDiff(self.init, cloud='src_cloud') + post_transport_instance.PostTransportInstance(self.init, + cloud='dst_cloud') + act_transport_ephemeral = \ + transport_ephemeral.TransportEphemeral(self.init, + cloud='dst_cloud') trans_file_to_file = task_transfer.TaskTransfer( self.init, 'SSHFileToFile', @@ -348,31 +424,44 @@ def migrate_instance(self): 'SSHFileToFile', resource_name=utl.INSTANCES_TYPE, resource_root_name=utl.DIFF_BODY) - act_is_not_copy_diff_file = is_not_copy_diff_file.IsNotCopyDiffFile(self.init) - process_merge_diff_and_base = act_i_to_f >> trans_file_to_file >> act_merge >> act_convert_image >> act_f_to_i + act_is_not_copy_diff_file = \ + is_not_copy_diff_file.IsNotCopyDiffFile(self.init) + process_merge_diff_and_base = act_i_to_f >> trans_file_to_file >> \ + act_merge >> act_convert_image >> act_f_to_i process_merge_diff_and_base = act_i_to_f - process_transport_image = act_transfer_file >> act_f_to_i_after_transfer + process_transport_image = act_transfer_file >> \ + act_f_to_i_after_transfer process_transport_image = act_transfer_file - act_pre_transport_instance = (act_is_not_trans_image | act_is_not_merge_diff) >> \ - process_transport_image >> \ - (act_is_not_merge_diff | act_deploy_instances) >> \ - process_merge_diff_and_base + act_pre_transport_instance = (act_is_not_trans_image | + act_is_not_merge_diff) >> \ + process_transport_image >> \ + (act_is_not_merge_diff | + act_deploy_instances) >> \ + process_merge_diff_and_base act_post_transport_instance = (act_is_not_copy_diff_file | - act_transport_ephemeral) >> act_trans_diff_file + act_transport_ephemeral) >> \ + act_trans_diff_file return act_net_prep >> \ - act_map_com_info >> \ - act_pre_transport_instance >> \ - act_deploy_instances >> \ - act_post_transport_instance >> \ - act_transport_ephemeral + act_map_com_info >> \ + act_pre_transport_instance >> \ + act_deploy_instances >> \ + act_post_transport_instance >> \ + act_transport_ephemeral def migrate_process_instance(self): - act_attaching = attach_used_volumes_via_compute.AttachVolumesCompute(self.init, cloud='dst_cloud') + act_attaching = \ + attach_used_volumes_via_compute.AttachVolumesCompute( + self.init, + cloud='dst_cloud') act_stop_vms = stop_vm.StopVms(self.init, cloud='src_cloud') act_start_vms = start_vm.StartVms(self.init, cloud='dst_cloud') - #transport_resource_inst = self.migrate_resources_by_instance_via_ssh() + # transport_resource_inst = \ + # self.migrate_resources_by_instance_via_ssh() transport_resource_inst = self.migrate_resources_by_instance() transport_inst = self.migrate_instance() - act_dissociate_floatingip = instance_floatingip_actions.DisassociateAllFloatingips(self.init, cloud='src_cloud') + act_dissociate_floatingip = \ + instance_floatingip_actions.DisassociateAllFloatingips( + self.init, + cloud='src_cloud') return act_stop_vms >> transport_resource_inst >> transport_inst >> \ - act_attaching >> act_dissociate_floatingip >> act_start_vms + act_attaching >> act_dissociate_floatingip >> act_start_vms diff --git a/cloudferrylib/__init__.py b/cloudferrylib/__init__.py index 2f3634b0..e69de29b 100644 --- a/cloudferrylib/__init__.py +++ b/cloudferrylib/__init__.py @@ -1,3 +0,0 @@ -import base -import scheduler -import os diff --git a/cloudferrylib/base/__init__.py b/cloudferrylib/base/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/base/__init__.py +++ b/cloudferrylib/base/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/base/action/copy_var.py b/cloudferrylib/base/action/copy_var.py index d8262b73..0254ccd2 100644 --- a/cloudferrylib/base/action/copy_var.py +++ b/cloudferrylib/base/action/copy_var.py @@ -33,4 +33,3 @@ def run(self, **kwargs): return { self.info_name: new_obj } - diff --git a/cloudferrylib/base/action/create_reference.py b/cloudferrylib/base/action/create_reference.py index b42ab70c..0de9535e 100644 --- a/cloudferrylib/base/action/create_reference.py +++ b/cloudferrylib/base/action/create_reference.py @@ -14,7 +14,6 @@ from cloudferrylib.base.action import action -import copy class CreateReference(action.Action): @@ -28,4 +27,3 @@ def run(self, **kwargs): return { self.info_name: kwargs[self.original_info_name] } - diff --git a/cloudferrylib/base/action/get_info_iter.py b/cloudferrylib/base/action/get_info_iter.py index d7689c20..daf86e0b 100644 --- a/cloudferrylib/base/action/get_info_iter.py +++ b/cloudferrylib/base/action/get_info_iter.py @@ -15,12 +15,12 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils as utl -import copy class GetInfoIter(action.Action): - def __init__(self, init, iter_info_name='info_iter', info_name='info', resource_name=utl.INSTANCES_TYPE): + def __init__(self, init, iter_info_name='info_iter', info_name='info', + resource_name=utl.INSTANCES_TYPE): self.iter_info_name = iter_info_name self.info_name = info_name self.resource_name = resource_name @@ -39,7 +39,3 @@ def run(self, **kwargs): self.iter_info_name: info, self.info_name: new_info } - - - - diff --git a/cloudferrylib/base/action/is_end_iter.py b/cloudferrylib/base/action/is_end_iter.py index 696f166b..6fa01694 100644 --- a/cloudferrylib/base/action/is_end_iter.py +++ b/cloudferrylib/base/action/is_end_iter.py @@ -33,8 +33,3 @@ def run(self, **kwargs): else: self.num_element = 0 return {} - - - - - diff --git a/cloudferrylib/base/action/is_option.py b/cloudferrylib/base/action/is_option.py index b29a7714..38a90cdf 100644 --- a/cloudferrylib/base/action/is_option.py +++ b/cloudferrylib/base/action/is_option.py @@ -19,6 +19,7 @@ PATH_ONE = 1 PATH_TWO = 2 + class IsOption(action.Action): def __init__(self, init, option_name): @@ -26,16 +27,10 @@ def __init__(self, init, option_name): super(IsOption, self).__init__(init) def run(self, **kwargs): - self.set_next_path(DEFAULT) # DEFAULT PATH + self.set_next_path(DEFAULT) # DEFAULT PATH option_value = self.cfg.migrate[self.option_name] if option_value: self.set_next_path(PATH_ONE) else: self.set_next_path(PATH_TWO) return {} - - - - - - diff --git a/cloudferrylib/base/action/merge.py b/cloudferrylib/base/action/merge.py index 060aaee5..4bf9338b 100644 --- a/cloudferrylib/base/action/merge.py +++ b/cloudferrylib/base/action/merge.py @@ -35,4 +35,3 @@ def run(self, **kwargs): return { self.result: data2 } - diff --git a/cloudferrylib/base/action/rename_info.py b/cloudferrylib/base/action/rename_info.py index f6a3bde7..4bd568b3 100644 --- a/cloudferrylib/base/action/rename_info.py +++ b/cloudferrylib/base/action/rename_info.py @@ -14,7 +14,6 @@ from cloudferrylib.base.action import action -import copy class RenameInfo(action.Action): @@ -29,4 +28,3 @@ def run(self, **kwargs): self.original_info_name: None, self.info_name: kwargs[self.original_info_name] } - diff --git a/cloudferrylib/base/image.py b/cloudferrylib/base/image.py index 80bffcaf..2ac764c0 100644 --- a/cloudferrylib/base/image.py +++ b/cloudferrylib/base/image.py @@ -39,20 +39,26 @@ def glance_image_download_cmd(config, image_id, destination_file): file=destination_file) -def glance_image_create_cmd(config, image_name, disk_format, filename, +def glance_image_create_cmd(config, image_name, disk_format, file_path, container_format="bare"): """Generates glance command which creates image based on arguments provided. Command output is filtered for 'id' :returns: Openstack CLI command""" - args = ("--name {image_name} " + if file_path.startswith("http"): + file_prefix = "location" + else: + file_prefix = "file" + args = ("image-create " + "--name {image_name} " "--disk-format={disk_format} " "--container-format={container_format} " - "--file {file}").format( + "--{file_prefix} {file_path}").format( image_name=image_name, disk_format=disk_format, container_format=container_format, - file=filename + file_prefix=file_prefix, + file_path=file_path ) return "{image_create} | grep '\'".format( - image_create=clients.os_cli_cmd(config, 'image-create', *args)) + image_create=clients.os_cli_cmd(config, 'glance', args)) diff --git a/cloudferrylib/base/network.py b/cloudferrylib/base/network.py index 9a82ad2b..0f4a91a7 100644 --- a/cloudferrylib/base/network.py +++ b/cloudferrylib/base/network.py @@ -38,4 +38,3 @@ def check_existing_port(self, network_id, mac): def get_security_groups(self): raise NotImplemented("it's base class") - diff --git a/cloudferrylib/base/objstorage.py b/cloudferrylib/base/objstorage.py index 3d7e474b..05c8ac8e 100644 --- a/cloudferrylib/base/objstorage.py +++ b/cloudferrylib/base/objstorage.py @@ -19,4 +19,3 @@ class ObjStorage(resource.Resource): def __init__(self): super(ObjStorage, self).__init__() - diff --git a/cloudferrylib/base/resource.py b/cloudferrylib/base/resource.py index 2e3718af..be408a1c 100644 --- a/cloudferrylib/base/resource.py +++ b/cloudferrylib/base/resource.py @@ -49,8 +49,9 @@ def wait_for_status(self, res_id, get_status, wait_status, timeout=60): time.sleep(delay) delay *= 2 else: - raise timeout_exception.TimeoutException(get_status(res_id).lower(), - wait_status, "Timeout exp") + raise timeout_exception.TimeoutException( + get_status(res_id).lower(), + wait_status, "Timeout exp") def try_wait_for_status(self, res_id, get_status, wait_status, timeout=60): try: diff --git a/cloudferrylib/os/__init__.py b/cloudferrylib/os/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/os/__init__.py +++ b/cloudferrylib/os/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/os/actions/attach_used_volumes.py b/cloudferrylib/os/actions/attach_used_volumes.py index 2ebde35f..36eec0f1 100644 --- a/cloudferrylib/os/actions/attach_used_volumes.py +++ b/cloudferrylib/os/actions/attach_used_volumes.py @@ -23,4 +23,4 @@ def run(self, storage_info={}, **kwargs): resource_storage = self.cloud.resources[utl.STORAGE_RESOURCE] for vol in storage_info[utl.VOLUMES_TYPE].itervalues(): resource_storage.attach_volume_to_instance(vol) - return {} \ No newline at end of file + return {} diff --git a/cloudferrylib/os/actions/attach_used_volumes_via_compute.py b/cloudferrylib/os/actions/attach_used_volumes_via_compute.py index 96e2dd68..f5f3416a 100644 --- a/cloudferrylib/os/actions/attach_used_volumes_via_compute.py +++ b/cloudferrylib/os/actions/attach_used_volumes_via_compute.py @@ -22,16 +22,16 @@ class AttachVolumesCompute(action.Action): def run(self, info, **kwargs): info = copy.deepcopy(info) - compute_resource = self.cloud.resources[utl.COMPUTE_RESOURCE] - storage_resource = self.cloud.resources[utl.STORAGE_RESOURCE] + compute_res = self.cloud.resources[utl.COMPUTE_RESOURCE] + storage_res = self.cloud.resources[utl.STORAGE_RESOURCE] for instance in info[utl.INSTANCES_TYPE].itervalues(): if not instance[utl.META_INFO].get(utl.VOLUME_BODY): continue for vol in instance[utl.META_INFO][utl.VOLUME_BODY]: - if storage_resource.get_status( + if storage_res.get_status( vol['volume']['id']) != 'in-use': - compute_resource.attach_volume_to_instance(instance, vol) - storage_resource.wait_for_status(vol['volume']['id'], - storage_resource.get_status, - 'in-use') + compute_res.attach_volume_to_instance(instance, vol) + storage_res.wait_for_status(vol['volume']['id'], + storage_res.get_status, + 'in-use') return {} diff --git a/cloudferrylib/os/actions/check_bandwidth.py b/cloudferrylib/os/actions/check_bandwidth.py index eb7e5c3d..126cc0fa 100644 --- a/cloudferrylib/os/actions/check_bandwidth.py +++ b/cloudferrylib/os/actions/check_bandwidth.py @@ -64,8 +64,11 @@ def run(self, **kwargs): with files.RemoteDir(runner, temp_dir_name): try: with utils.forward_agent(env.key_filename): - dd_command = cmd_cfg.dd_full('/dev/zero', remote_file_path, 1, - 0, test_file_size) + dd_command = cmd_cfg.dd_full('/dev/zero', + remote_file_path, + 1, + 0, + test_file_size) self.cloud.ssh_util.execute(dd_command) LOG.info("Checking upload speed... Wait please.") diff --git a/cloudferrylib/os/actions/check_cloud.py b/cloudferrylib/os/actions/check_cloud.py index 9773f65f..caa4fdc3 100644 --- a/cloudferrylib/os/actions/check_cloud.py +++ b/cloudferrylib/os/actions/check_cloud.py @@ -14,7 +14,7 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils as utl -from keystoneclient.openstack.common.apiclient import exceptions as keystone_exc +from keystoneclient.openstack.common.apiclient import exceptions as ks_exc from novaclient import exceptions as nova_exc from cloudferrylib.base import exception @@ -32,8 +32,8 @@ def run(self, **kwargs): compute_resource = self.cloud.resources[utl.COMPUTE_RESOURCE] volume_resource = self.cloud.resources[utl.STORAGE_RESOURCE] net_resource = self.cloud.resources[utl.NETWORK_RESOURCE] - admin_tenant_name = self.cloud.cloud_config.cloud.tenant - admin_tenant_id = ident_resource.get_tenant_id_by_name(admin_tenant_name) + adm_tenant_name = self.cloud.cloud_config.cloud.tenant + adm_tenant_id = ident_resource.get_tenant_id_by_name(adm_tenant_name) tenant_name = 'test_name' flavor_id = 'c0c0c0c0' err_message = 'Failed to create object in the cloud' @@ -65,8 +65,7 @@ def run(self, **kwargs): ident_resource._deploy_tenants(tenant) tenant_id = ident_resource.get_tenant_id_by_name(tenant_name) compute_resource._deploy_flavors(flavor, None) - except (keystone_exc.ClientException, - nova_exc.ClientException) as e: + except (ks_exc.ClientException, nova_exc.ClientException): LOG.error(err_message) raise exception.AbortMigrationError(err_message) migrate_image = image_res.create_image( @@ -75,7 +74,7 @@ def run(self, **kwargs): disk_format='qcow2', is_public=True, protected=False, - owner=admin_tenant_id, + owner=adm_tenant_id, size=4, properties={'user_name': 'test_user_name'}, data='test' @@ -106,7 +105,7 @@ def run(self, **kwargs): } } - subnet = net_resource.neutron_client.create_subnet(subnet_info) + net_resource.neutron_client.create_subnet(subnet_info) nics = [{'net-id': private_network_id}] @@ -123,7 +122,7 @@ def run(self, **kwargs): 'user_id': '1', 'boot_mode': utl.BOOT_FROM_IMAGE, 'availability_zone': 'nova', - 'tenant_name': admin_tenant_name + 'tenant_name': adm_tenant_name } } }, @@ -146,7 +145,7 @@ def run(self, **kwargs): volume_resource.cinder_client.volumes.delete(vol_new_ids.keys()[0]) network_info = { 'network': { - 'tenant_id': admin_tenant_id, + 'tenant_id': adm_tenant_id, 'admin_state_up': True, 'shared': True, 'name': 'test_net', @@ -157,7 +156,10 @@ def run(self, **kwargs): network_info)['network']['id'] net_resource.neutron_client.delete_network(new_net_id) vm_new_ids = compute_resource._deploy_instances(info) - if not vm_new_ids or not vol_new_ids or not migrate_image or not new_net_id: + if (not vm_new_ids or + not vol_new_ids or + not migrate_image or + not new_net_id): LOG.error(err_message) raise exception.AbortMigrationError(err_message) compute_resource.nova_client.servers.delete(vm_new_ids.keys()[0]) diff --git a/cloudferrylib/os/actions/check_filter.py b/cloudferrylib/os/actions/check_filter.py index 20d8afa7..9e9007ab 100644 --- a/cloudferrylib/os/actions/check_filter.py +++ b/cloudferrylib/os/actions/check_filter.py @@ -24,8 +24,8 @@ class CheckFilter(action.Action): def run(self, **kwargs): - """Check filter file and make sure all entries are present in source cloud. - + """Check filter file and make sure all entries are present in source + cloud. """ search_opts = kwargs.get('search_opts', {}) search_opts_img = kwargs.get('search_opts_img', {}) @@ -38,10 +38,13 @@ def run(self, **kwargs): try: img = image_resource.glance_client.images.get(img_id) if img: - LOG.debug('Filter config check: Image ID {} is OK'.format(img_id)) + LOG.debug('Filter config check: Image ID {} is OK' + .format(img_id)) except glance_exc.HTTPNotFound as e: - LOG.error('Filter config check: Image ID {} is not present in source cloud, ' - 'please update your filter config. Aborting.'.format(img_id)) + LOG.error('Filter config check: Image ID {} ' + 'is not present in source cloud, ' + 'please update your filter config. Aborting.' + .format(img_id)) raise e ident_resource = self.cloud.resources[utl.IDENTITY_RESOURCE] if search_opts_tenant and search_opts_tenant.get('tenant_id'): @@ -49,12 +52,16 @@ def run(self, **kwargs): for tenant_id in tenants: LOG.debug('Filtered tenant id: {}'.format(tenant_id)) try: - tenant = ident_resource.keystone_client.tenants.find(id=tenant_id) + tenant = ident_resource.keystone_client.tenants.find( + id=tenant_id) if tenant: - LOG.debug('Filter config check: Tenant ID {} is OK'.format(tenant_id)) + LOG.debug('Filter config check: Tenant ID {} is OK' + .format(tenant_id)) except keystone_exc.NotFound as e: - LOG.error('Filter config check: Tenant ID {} is not present in source cloud, ' - 'please update your filter config. Aborting.'.format(tenant_id)) + LOG.error('Filter config check: Tenant ID {} ' + 'is not present in source cloud, ' + 'please update your filter config. Aborting.' + .format(tenant_id)) raise e compute_resource = self.cloud.resources[utl.COMPUTE_RESOURCE] if search_opts and search_opts.get('id'): @@ -62,10 +69,14 @@ def run(self, **kwargs): for instance_id in instances: LOG.debug('Filtered instance id: {}'.format(instance_id)) try: - instance = compute_resource.nova_client.servers.get(instance_id) + instance = \ + compute_resource.nova_client.servers.get(instance_id) if instance: - LOG.debug('Filter config check: Instance ID {} is OK'.format(instance_id)) + LOG.debug('Filter config check: Instance ID {} is OK' + .format(instance_id)) except nova_exc.NotFound as e: - LOG.error('Filter config check: Instance ID {} is not present in source cloud, ' - 'please update your filter config. Aborting.'.format(instance_id)) + LOG.error('Filter config check: Instance ID {} ' + 'is not present in source cloud, ' + 'please update your filter config. Aborting.' + .format(instance_id)) raise e diff --git a/cloudferrylib/os/actions/check_instance_networks.py b/cloudferrylib/os/actions/check_instance_networks.py index b9beaaab..cf991ccf 100644 --- a/cloudferrylib/os/actions/check_instance_networks.py +++ b/cloudferrylib/os/actions/check_instance_networks.py @@ -49,6 +49,7 @@ def run(self, **kwargs): instance_names.append(compute_info['instance']['name']) if len(instance_names) > 0: - raise exceptions.EnvironmentError("Instances %s spawned " + raise exceptions.EnvironmentError( + "Instances %s spawned " "in external network directly. CloudFerry can't " "migrate it." % ", ".join(instance_names)) diff --git a/cloudferrylib/os/actions/check_needed_compute_resources.py b/cloudferrylib/os/actions/check_needed_compute_resources.py index cdd3d83d..09cc22d8 100644 --- a/cloudferrylib/os/actions/check_needed_compute_resources.py +++ b/cloudferrylib/os/actions/check_needed_compute_resources.py @@ -61,7 +61,8 @@ def check_in_use_flavor(self, objs, info): # is in this flavor, it will keep the old flavor id, can not be matched # to existing flavors src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] - src_flavor_ids = [flavor.id for flavor in src_compute.get_flavor_list()] + src_flavor_ids = \ + [flavor.id for flavor in src_compute.get_flavor_list()] dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] dst_flavors = dst_compute.get_flavor_list() dst_flavor_ids = [flavor.id for flavor in dst_flavors] @@ -82,5 +83,6 @@ def check_in_use_flavor(self, objs, info): ram=flav_details['memory_mb'], vcpus=flav_details['vcpus'], disk=flav_details['root_gb'], - ephemeral=flav_details['ephemeral_gb']) + ephemeral=flav_details[ + 'ephemeral_gb']) src_flavor_ids.append(_instance['flavor_id']) diff --git a/cloudferrylib/os/actions/check_networks.py b/cloudferrylib/os/actions/check_networks.py index 56e4272f..92a4b149 100644 --- a/cloudferrylib/os/actions/check_networks.py +++ b/cloudferrylib/os/actions/check_networks.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and# # limitations under the License. -import ipaddr + import collections -import exceptions + +import ipaddr + +from cloudferrylib.base import exception from cloudferrylib.base.action import action from cloudferrylib.utils import utils @@ -24,9 +27,12 @@ class CheckNetworks(action.Action): """ - Check subnets overlapping and raise EnvironmentError if needed. - It must be done before actual migration in check section. - The action use filter search opts and must run after act_get_filter action + Check networks segmentation ID and subnets overlapping. + + Raise exception (AbortMigrationError) if overlapping has been found. + It must be done before actual migration in the 'preparation' section. + The action uses filtered search opts and must be run after 'act_get_filter' + action. """ def run(self, **kwargs): @@ -36,12 +42,22 @@ def run(self, **kwargs): search_opts = kwargs.get('search_opts_tenant', {}) src_info = NetworkInfo(src_net.read_info(**search_opts)) dst_info = NetworkInfo(dst_net.read_info(**search_opts)) + dst_seg_ids = dst_info.get_segmentation_ids() + for network in src_info.get_networks(): dst_net = dst_info.by_hash.get(network.hash) if dst_net: - LOG.debug("src network %s, dst network %s" % + # Current network matches with network on DST + # Have the same networks on SRC and DST + LOG.debug("SRC network: '%s', DST network: '%s'" % (network.id, dst_net.id)) network.check_network_overlapping(dst_net) + else: + # Current network does not match with any network on DST + # Check Segmentation ID overlapping with DST + LOG.debug("Check segmentation ID for SRC network: '%s'", + network.id) + network.check_segmentation_id_overlapping(dst_seg_ids) class NetworkInfo(object): @@ -59,6 +75,40 @@ def __init__(self, info): def get_networks(self): return self.by_hash.values() + def get_segmentation_ids(self): + """Get busy segmentation IDs. + + We need to handle duplicates in segmentation ids. + Neutron has different validation rules for different network types. + + For 'gre' and 'vxlan' network types there is no strong requirement + for 'physical_network' attribute, if we want to have + 'segmentation_id', because traffic is encapsulated in L3 packets. + + For 'vlan' and 'flat' network types there is a strong requirement for + 'physical_network' attribute, if we want to have 'segmentation_id'. + + :result: Dictionary with busy segmentation IDs. + Hash is used with structure {"gre": [1, 2, ...], + "vlan": [1, 2, ...]} + """ + + used_seg_ids = {} + networks = self.by_id.values() + + for net in networks: + network_has_segmentation_id = ( + net.info["provider:physical_network"] or + (net.network_type in ['gre', 'vxlan'])) + + if network_has_segmentation_id: + if net.network_type not in used_seg_ids: + used_seg_ids[net.network_type] = [] + if net.seg_id: + used_seg_ids[net.network_type].append(net.seg_id) + + return used_seg_ids + class Network(object): def __init__(self, info): @@ -68,24 +118,27 @@ def __init__(self, info): self.id = info['id'] self.hash = info['res_hash'] + self.network_type = self.info['provider:network_type'] + self.seg_id = self.info["provider:segmentation_id"] + def add_subnet(self, info): self.subnets.append(info) self.subnets_hash.add(info['res_hash']) def check_network_overlapping(self, network): for subnet in network.subnets: - LOG.debug("work on src subnet %s" % subnet['id']) + LOG.debug("Work with SRC subnet: '%s'" % subnet['id']) if self.is_subnet_eq(subnet): - LOG.debug("We have the same on dst by hash") + LOG.debug("We have the same subnet on DST by hash") continue overlapping_subnet = self.get_overlapping_subnet(subnet) if overlapping_subnet: - message = ("Subnet %s in network %s on src overlap " - "subnet %s in network %s on dst" % ( - overlapping_subnet, self.id, - subnet['id'], network.id)) + message = ("Subnet '%s' in network '%s' on SRC overlaps with " + "subnet '%s' in network '%s' on DST" % ( + overlapping_subnet, self.id, + subnet['id'], network.id)) LOG.error(message) - raise exceptions.EnvironmentError(message) + raise exception.AbortMigrationError(message) def is_subnet_eq(self, subnet): return subnet['res_hash'] in self.subnets_hash @@ -94,5 +147,22 @@ def get_overlapping_subnet(self, subnet): cidr = ipaddr.IPNetwork(subnet['cidr']) for self_subnet in self.subnets: self_cidr = ipaddr.IPNetwork(self_subnet['cidr']) - if (cidr.Contains(self_cidr) or self_cidr.Contains(cidr)): + if cidr.Contains(self_cidr) or self_cidr.Contains(cidr): return self_subnet['id'] + + def check_segmentation_id_overlapping(self, dst_seg_ids): + """ + Check if segmentation ID of current network overlaps with destination. + + :param dst_seg_ids: Dictionary with busy segmentation IDs on DST + """ + + if self.network_type not in dst_seg_ids: + return + + if self.seg_id in dst_seg_ids[self.network_type]: + message = ("Segmentation ID '%s' (network type = '%s', " + "network ID = '%s') is already busy on the destination " + "cloud.") % (self.seg_id, self.network_type, self.id) + LOG.error(message) + raise exception.AbortMigrationError(message) diff --git a/cloudferrylib/os/actions/check_openstack_apis.py b/cloudferrylib/os/actions/check_openstack_apis.py index 852d5a1c..ffa35b76 100644 --- a/cloudferrylib/os/actions/check_openstack_apis.py +++ b/cloudferrylib/os/actions/check_openstack_apis.py @@ -17,14 +17,18 @@ from cloudferrylib.utils import utils as utl from neutronclient.common import exceptions as neutron_exc from glanceclient import exc as glance_exc -from keystoneclient.openstack.common.apiclient import exceptions as keystone_exc +from keystoneclient.openstack.common.apiclient import exceptions as ks_exc from cinderclient import exceptions as cinder_exc from novaclient import exceptions as nova_exc LOG = utl.get_log(__name__) -def check(os_api_call, os_api_type, position, *os_api_call_args, **os_api_call_kwargs): +def check(os_api_call, + os_api_type, + position, + *os_api_call_args, + **os_api_call_kwargs): try: LOG.info("Checking %s APIs availability on %s.", os_api_type, position.upper()) @@ -32,7 +36,7 @@ def check(os_api_call, os_api_type, position, *os_api_call_args, **os_api_call_k except (neutron_exc.NeutronException, glance_exc.BaseException, glance_exc.ClientException, - keystone_exc.ClientException, + ks_exc.ClientException, cinder_exc.ClientException, nova_exc.ClientException) as e: message = ('{os_api_type} APIs on {position} check failed with: ' @@ -76,13 +80,15 @@ def run(self, **kwargs): storage_resource = self.cloud.resources[utl.STORAGE_RESOURCE] check(storage_resource.cinder_client.volumes.list, 'Cinder volumes', self.cloud.position) - check(storage_resource.cinder_client.volume_snapshots.list, 'Cinder snapshots', + check(storage_resource.cinder_client.volume_snapshots.list, + 'Cinder snapshots', self.cloud.position) class CheckNetworkingAPIs(action.Action): def run(self, **kwargs): - """Check networking backend by getting network/subnets/routers lists.""" + """Check networking backend + by getting network/subnets/routers lists.""" neutron = self.cloud.resources[utl.NETWORK_RESOURCE] check(neutron.neutron_client.list_networks, 'Neutron networks', self.cloud.position) diff --git a/cloudferrylib/os/actions/check_ssh.py b/cloudferrylib/os/actions/check_ssh.py index 72038df5..4d19ecb9 100644 --- a/cloudferrylib/os/actions/check_ssh.py +++ b/cloudferrylib/os/actions/check_ssh.py @@ -51,8 +51,10 @@ def check_access(self, node): runner = remote_runner.RemoteRunner(node, cfg.ssh_user, password=cfg.ssh_sudo_password) gateway = self.cloud.getIpSsh() + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + try: - with settings(gateway=gateway): + with settings(gateway=gateway, connection_attempts=ssh_attempts): runner.run('echo') except Exception as error: LOG.error("SSH connection from '%s' to '%s' failed with error: " diff --git a/cloudferrylib/os/actions/cinder_database_manipulation.py b/cloudferrylib/os/actions/cinder_database_manipulation.py index 1ce47587..a238c945 100644 --- a/cloudferrylib/os/actions/cinder_database_manipulation.py +++ b/cloudferrylib/os/actions/cinder_database_manipulation.py @@ -41,8 +41,8 @@ class GetVolumesDb(CinderDatabaseInteraction): def run(self, *args, **kwargs): search_opts = kwargs.get('search_opts_tenant', {}) - return {NAMESPACE_CINDER_CONST: \ - self.get_resource().read_db_info(**search_opts)} + return {NAMESPACE_CINDER_CONST: + self.get_resource().read_db_info(**search_opts)} class WriteVolumesDb(CinderDatabaseInteraction): diff --git a/cloudferrylib/os/actions/cleanup_images.py b/cloudferrylib/os/actions/cleanup_images.py index ed2a624f..7c28c211 100644 --- a/cloudferrylib/os/actions/cleanup_images.py +++ b/cloudferrylib/os/actions/cleanup_images.py @@ -37,7 +37,8 @@ def run(self, info, **kwargs): for vol in volumes: if utl.IMAGE_BODY in vol[utl.META_INFO]: - image_checksum = vol[utl.META_INFO][utl.IMAGE_BODY]['checksum'] + image_checksum = \ + vol[utl.META_INFO][utl.IMAGE_BODY]['checksum'] if image_checksum not in checksum_list: checksum_list.append(image_checksum) diff --git a/cloudferrylib/os/actions/convert_compute_to_image.py b/cloudferrylib/os/actions/convert_compute_to_image.py index 6894dc1c..e5d748e0 100644 --- a/cloudferrylib/os/actions/convert_compute_to_image.py +++ b/cloudferrylib/os/actions/convert_compute_to_image.py @@ -71,7 +71,8 @@ def run(self, info=None, **kwargs): images_body[image_id][utl.META_INFO][ utl.INSTANCE_BODY] = [instance] else: - LOG.warning("No boot image for instance, need to re-create it") + LOG.warning("No boot image for instance, " + "need to re-create it") missing_images[instance_id] = image_id else: compute_ignored_images[instance_id] = instance diff --git a/cloudferrylib/os/actions/convert_compute_to_volume.py b/cloudferrylib/os/actions/convert_compute_to_volume.py index 9d0b21d8..5bd5cc79 100644 --- a/cloudferrylib/os/actions/convert_compute_to_volume.py +++ b/cloudferrylib/os/actions/convert_compute_to_volume.py @@ -22,11 +22,11 @@ class ConvertComputeToVolume(action.Action): def run(self, info=None, **kwargs): - compute_info = copy.deepcopy(info) + info = copy.deepcopy(info) storage_info = {utl.VOLUMES_TYPE: {}} ignored = {} resource_storage = self.cloud.resources[utl.STORAGE_RESOURCE] - for instance_id, instance in compute_info[utl.INSTANCES_TYPE].iteritems(): + for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): volumes_exists = True if not instance[utl.INSTANCE_BODY]['volumes']: if 'volume' in instance['meta']: diff --git a/cloudferrylib/os/actions/convert_file.py b/cloudferrylib/os/actions/convert_file.py index 1f484983..0eea833c 100644 --- a/cloudferrylib/os/actions/convert_file.py +++ b/cloudferrylib/os/actions/convert_file.py @@ -13,7 +13,8 @@ def run(self, info=None, **kwargs): if image_res.config.image.convert_to_raw: return {} for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): - image_id = info[INSTANCES][instance_id][utl.INSTANCE_BODY]['image_id'] + image_id = \ + info[INSTANCES][instance_id][utl.INSTANCE_BODY]['image_id'] images = image_res.read_info(image_id=image_id) image = images[utl.IMAGES_TYPE][image_id] disk_format = image[utl.IMAGE_BODY]['disk_format'] @@ -24,8 +25,9 @@ def run(self, info=None, **kwargs): @staticmethod def convert_file_to_raw(host, disk_format, filepath): - with settings(host_string=host): + with settings(host_string=host, + connection_attempts=env.connection_attempts): with forward_agent(env.key_filename): run("qemu-img convert -f %s -O raw %s %s.tmp" % (disk_format, filepath, filepath)) - run("mv -f %s.tmp %s" % (filepath, filepath)) \ No newline at end of file + run("mv -f %s.tmp %s" % (filepath, filepath)) diff --git a/cloudferrylib/os/actions/convert_file_to_image.py b/cloudferrylib/os/actions/convert_file_to_image.py index 398936ef..c368d190 100644 --- a/cloudferrylib/os/actions/convert_file_to_image.py +++ b/cloudferrylib/os/actions/convert_file_to_image.py @@ -1,13 +1,10 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils -from cloudferrylib.utils import remote_runner class ConvertFileToImage(action.Action): - def run(self, file_path=None, image_format=None, image_name=None, **kwargs): + def run(self, file_path=None, img_fmt=None, img_name=None, **kwargs): image_resource = self.cloud.resources[utils.IMAGE_RESOURCE] - cfg = self.cloud.cloud_config.cloud - runner = remote_runner.RemoteRunner(cfg.host, cfg.ssh_user) - return image_resource.glance_img_create(runner, image_name, image_format, file_path) + return image_resource.glance_img_create(img_name, img_fmt, file_path) diff --git a/cloudferrylib/os/actions/convert_image_to_file.py b/cloudferrylib/os/actions/convert_image_to_file.py index ffd4d9b0..06c370d2 100644 --- a/cloudferrylib/os/actions/convert_image_to_file.py +++ b/cloudferrylib/os/actions/convert_image_to_file.py @@ -8,7 +8,9 @@ class ConvertImageToFile(action.Action): def run(self, image_id=None, base_filename=None, **kwargs): cfg = self.cloud.cloud_config.cloud - with settings(host_string=cfg.host): + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + + with settings(host_string=cfg.host, connection_attempts=ssh_attempts): with forward_agent(env.key_filename): cmd = image.glance_image_download_cmd(cfg, image_id, base_filename) diff --git a/cloudferrylib/os/actions/convert_volume_to_compute.py b/cloudferrylib/os/actions/convert_volume_to_compute.py index 5eaabe67..3aea6970 100644 --- a/cloudferrylib/os/actions/convert_volume_to_compute.py +++ b/cloudferrylib/os/actions/convert_volume_to_compute.py @@ -31,7 +31,8 @@ def run(self, storage_info, compute_ignored={}, **kwargs): instances[instance_id] = volume['meta']['instance'] instances[instance_id]['meta']['volume'] = [] volume['meta'].pop('instance') - instances[instance_id] = self.map_volume(instances[instance_id], volume) + instances[instance_id] = self.map_volume(instances[instance_id], + volume) for inst in instances.itervalues(): for vol in inst['instance']['volumes']: volumes_old[vol['id']]['volume']['device'] = vol['device'] @@ -43,4 +44,4 @@ def map_volume(instance, volume): for vol_old in instance['instance']['volumes']: if volume['old_id'] == vol_old['id']: vol_old['id'] = volume['volume']['id'] - return instance \ No newline at end of file + return instance diff --git a/cloudferrylib/os/actions/convert_volume_to_image.py b/cloudferrylib/os/actions/convert_volume_to_image.py index 68a43023..2fd7fa5a 100644 --- a/cloudferrylib/os/actions/convert_volume_to_image.py +++ b/cloudferrylib/os/actions/convert_volume_to_image.py @@ -18,6 +18,7 @@ from cloudferrylib.utils import utils import copy + LOG = utils.get_log(__name__) CEPH = 'ceph' ACTIVE = 'active' @@ -53,7 +54,9 @@ def run(self, storage_info={}, **kwargs): vol['id'], force=True, image_name=vol['id'], container_format=self.container_format, disk_format=self.disk_format) - resource_image.wait_for_status(image_id, resource_image.get_status, ACTIVE) + resource_image.wait_for_status(image_id, + resource_image.get_status, + ACTIVE) resource_image.patch_image(resource_image.get_backend(), image_id) image_vol = resource_image.read_info(image_id=image_id) img_new = { diff --git a/cloudferrylib/os/actions/copy_g2g.py b/cloudferrylib/os/actions/copy_g2g.py index 3a844fa0..542557b3 100644 --- a/cloudferrylib/os/actions/copy_g2g.py +++ b/cloudferrylib/os/actions/copy_g2g.py @@ -28,9 +28,9 @@ def run(self, images_info=None, **kwargs): dst_image = self.dst_cloud.resources[utl.IMAGE_RESOURCE] if not images_info: - action_get_im = get_info_images.GetInfoImages(self.init, cloud='src_cloud') + action_get_im = get_info_images.GetInfoImages(self.init, + cloud='src_cloud') images_info = action_get_im.run() new_info = dst_image.deploy(images_info) return {'images_info': new_info} - diff --git a/cloudferrylib/os/actions/deploy_snapshots.py b/cloudferrylib/os/actions/deploy_snapshots.py index 92293856..7a32dd31 100644 --- a/cloudferrylib/os/actions/deploy_snapshots.py +++ b/cloudferrylib/os/actions/deploy_snapshots.py @@ -45,38 +45,51 @@ def run(self, storage_info=None, identity_info=None, **kwargs): for snap in snapshots_list: if snapshots_list.index(snap) == 0: - act_snap_transfer = snap_transfer.SnapTransfer(self.init, - ssh_ceph_to_ceph.SSHCephToCeph, 1) + act_snap_transfer = \ + snap_transfer.SnapTransfer( + self.init, + ssh_ceph_to_ceph.SSHCephToCeph, + 1) else: snap_num = snapshots_list.index(snap) - snap['prev_snapname'] = snapshots_list[snap_num - 1]['name'] - act_snap_transfer = snap_transfer.SnapTransfer(self.init, - ssh_ceph_to_ceph.SSHCephToCeph, 2) - - + snap['prev_snapname'] = \ + snapshots_list[snap_num - 1]['name'] + act_snap_transfer = \ + snap_transfer.SnapTransfer( + self.init, + ssh_ceph_to_ceph.SSHCephToCeph, + 2) act_snap_transfer.run(volume=vol_info, snapshot_info=snap) - new_snapshot = volume_resource.create_snapshot(volume_id=vol_id, - display_name=snap['display_name'], - display_description=snap['display_description']) + volume_resource.create_snapshot( + volume_id=vol_id, + display_name=snap['display_name'], + display_description=snap['display_description']) - act_snap_transfer = snap_transfer.SnapTransfer(self.init, - ssh_ceph_to_ceph.SSHCephToCeph, 3) - act_snap_transfer.run(volume=vol_info, snapshot_info=snapshots_list[-1]) + act_snap_transfer = snap_transfer.SnapTransfer( + self.init, + ssh_ceph_to_ceph.SSHCephToCeph, + 3) + act_snap_transfer.run(volume=vol_info, + snapshot_info=snapshots_list[-1]) for snap in snapshots_list: if volume_resource.config.storage.host: - act_delete_redundant_snap = rbd_util.RbdUtil(cloud=self.cloud, - config_migrate=self.cfg.migrate, - host=vol_info[utl.HOST_DST]) - act_delete_redundant_snap.snap_rm(vol_info[utl.PATH_DST], - snap['name']) + act_delete_redundant_snap = \ + rbd_util.RbdUtil(cloud=self.cloud, + config_migrate=self.cfg.migrate, + host=vol_info[utl.HOST_DST]) + act_delete_redundant_snap.snap_rm( + vol_info[utl.PATH_DST], + snap['name']) else: - act_delete_redundant_snap = rbd_util.RbdUtil(cloud=self.cloud, - config_migrate=self.cfg.migrate) - act_delete_redundant_snap.snap_rm(vol_info[utl.PATH_DST], - snap['name'], vol_info[utl.HOST_DST]) + act_delete_redundant_snap = \ + rbd_util.RbdUtil(cloud=self.cloud, + config_migrate=self.cfg.migrate) + act_delete_redundant_snap.snap_rm( + vol_info[utl.PATH_DST], + snap['name'], vol_info[utl.HOST_DST]) else: one_volume_info = { @@ -87,10 +100,10 @@ def run(self, storage_info=None, identity_info=None, **kwargs): } } - - act_transport_vol_data = task_transfer.TaskTransfer(self.init, - 'SSHCephToCeph', - input_info='one_volume_info') + act_transport_vol_data = \ + task_transfer.TaskTransfer(self.init, + 'SSHCephToCeph', + input_info='one_volume_info') act_transport_vol_data.run(**one_volume_info) diff --git a/cloudferrylib/os/actions/deploy_volumes.py b/cloudferrylib/os/actions/deploy_volumes.py index 8e4f543f..df6e9c1c 100644 --- a/cloudferrylib/os/actions/deploy_volumes.py +++ b/cloudferrylib/os/actions/deploy_volumes.py @@ -48,7 +48,3 @@ def run(self, storage_info={}, identity_info={}, **kwargs): return { 'storage_info': storage_info_new } - - - - diff --git a/cloudferrylib/os/actions/failure_act.py b/cloudferrylib/os/actions/failure_act.py index 7eee6864..fe1e60bf 100644 --- a/cloudferrylib/os/actions/failure_act.py +++ b/cloudferrylib/os/actions/failure_act.py @@ -22,6 +22,7 @@ class DivisionByZero(action.Action): def run(self, *args, **kwargs): - LOG.debug("Dividing by zero (special case to fail migration execution)") + LOG.debug("Dividing by zero " + "(special case to fail migration execution)") a = 1 / 0 return a diff --git a/cloudferrylib/os/actions/image_snapshot.py b/cloudferrylib/os/actions/image_snapshot.py index 987ddda2..1b355dac 100644 --- a/cloudferrylib/os/actions/image_snapshot.py +++ b/cloudferrylib/os/actions/image_snapshot.py @@ -21,6 +21,7 @@ NAMESPACE = 'IMAGE_SNAPSHOT' + class ImageSnapshotBasic(action.Action): """Base class that contains common to image_snapshot methods""" diff --git a/cloudferrylib/os/actions/is_not_copy_diff_file.py b/cloudferrylib/os/actions/is_not_copy_diff_file.py index 12396960..c74e7edd 100644 --- a/cloudferrylib/os/actions/is_not_copy_diff_file.py +++ b/cloudferrylib/os/actions/is_not_copy_diff_file.py @@ -21,6 +21,8 @@ ISCSI = 'iscsi' PATH_COPY_DIFF = 0 DEFAULT = 1 +INST = utl.INSTANCES_TYPE +BODY = utl.INSTANCE_BODY class IsNotCopyDiffFile(action.Action): @@ -31,9 +33,10 @@ def run(self, info=None, **kwargs): dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend backend_ephem_drv_dst = dst_compute.config.compute.backend - instance_boot = info[utl.INSTANCES_TYPE].values()[0][utl.INSTANCE_BODY]['boot_mode'] - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == ISCSI): + instance_boot = \ + info[INST].values()[0][BODY]['boot_mode'] + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == ISCSI)): self.set_next_path(PATH_COPY_DIFF) - return { - } + return {} diff --git a/cloudferrylib/os/actions/is_not_merge_diff.py b/cloudferrylib/os/actions/is_not_merge_diff.py index 705f6542..98342fce 100644 --- a/cloudferrylib/os/actions/is_not_merge_diff.py +++ b/cloudferrylib/os/actions/is_not_merge_diff.py @@ -21,6 +21,8 @@ ISCSI = 'iscsi' PATH_MERGE_DIFF_IMAGE = 0 DEFAULT = 1 +INST = utl.INSTANCES_TYPE +BODY = utl.INSTANCE_BODY class IsNotMergeDiff(action.Action): @@ -31,9 +33,11 @@ def run(self, info=None, **kwargs): dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend backend_ephem_drv_dst = dst_compute.config.compute.backend - instance_boot = info[utl.INSTANCES_TYPE].values()[0][utl.INSTANCE_BODY]['boot_mode'] - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == CEPH): + instance_boot = \ + info[INST].values()[0][BODY]['boot_mode'] + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == CEPH)): self.set_next_path(PATH_MERGE_DIFF_IMAGE) return { } diff --git a/cloudferrylib/os/actions/is_not_transport_image.py b/cloudferrylib/os/actions/is_not_transport_image.py index 5c188c33..9c48cc17 100644 --- a/cloudferrylib/os/actions/is_not_transport_image.py +++ b/cloudferrylib/os/actions/is_not_transport_image.py @@ -21,6 +21,8 @@ ISCSI = 'iscsi' PATH_TRANSPORT_IMAGE = 0 DEFAULT = 1 +INST = utl.INSTANCES_TYPE +BODY = utl.INSTANCE_BODY class IsNotTransportImage(action.Action): @@ -29,8 +31,10 @@ def run(self, info=None, **kwargs): info = copy.deepcopy(info) src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend - instance_boot = info[utl.INSTANCES_TYPE].values()[0][utl.INSTANCE_BODY]['boot_mode'] - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == CEPH): + instance_boot = \ + info[INST].values()[0][BODY]['boot_mode'] + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == CEPH)): self.set_next_path(PATH_TRANSPORT_IMAGE) return { } diff --git a/cloudferrylib/os/actions/live_migration.py b/cloudferrylib/os/actions/live_migration.py index c93d8ed0..9d66a8d6 100644 --- a/cloudferrylib/os/actions/live_migration.py +++ b/cloudferrylib/os/actions/live_migration.py @@ -96,8 +96,10 @@ def run(self, info=None, **kwargs): src_host = instance_host(src_instance) dst_host = instance_host(dst_instance) - src_runner = remote_runner.RemoteRunner(src_host, cfglib.CONF.src.ssh_user) - dst_runner = remote_runner.RemoteRunner(dst_host, cfglib.CONF.dst.ssh_user) + src_runner = remote_runner.RemoteRunner(src_host, + cfglib.CONF.src.ssh_user) + dst_runner = remote_runner.RemoteRunner(dst_host, + cfglib.CONF.dst.ssh_user) src_libvirt = libvirt.Libvirt(src_runner) dst_libvirt = libvirt.Libvirt(dst_runner) @@ -116,14 +118,24 @@ def run(self, info=None, **kwargs): dst_backing_file = dst_libvirt.get_backing_file(new_id) src_backing_file = src_libvirt.get_backing_file(old_id) migration_backing_file = os.path.join( - libvirt.nova_instances_path, '_base', 'migration_disk_{}'.format(old_id)) + libvirt.nova_instances_path, + '_base', + 'migration_disk_{}'.format(old_id)) dst_compute.wait_for_status(new_id, dst_compute.get_status, 'active') - with files.RemoteTempFile(src_runner, "migrate-{}".format(old_id), src_vm_xml.dump()) as migration_file,\ - files.RemoteSymlink(src_runner, src_backing_file, migration_backing_file),\ - files.RemoteSymlink(dst_runner, dst_backing_file, migration_backing_file),\ + with files.RemoteTempFile(src_runner, + "migrate-{}".format(old_id), + src_vm_xml.dump()) as migration_file,\ + files.RemoteSymlink(src_runner, + src_backing_file, + migration_backing_file),\ + files.RemoteSymlink(dst_runner, + dst_backing_file, + migration_backing_file),\ ubuntu.StopNovaCompute(dst_runner),\ - libvirt.QemuBackingFileMover(src_libvirt.runner, migration_backing_file, old_id): + libvirt.QemuBackingFileMover(src_libvirt.runner, + migration_backing_file, + old_id): destroyer = libvirt.DestNovaInstanceDestroyer(dst_libvirt, dst_compute, @@ -131,7 +143,9 @@ def run(self, info=None, **kwargs): dst_instance.id) try: destroyer.do() - src_libvirt.live_migrate(src_virsh_name, dst_host, migration_file.filename) + src_libvirt.live_migrate(src_virsh_name, + dst_host, + migration_file.filename) except remote_runner.RemoteExecutionError: destroyer.undo() finally: diff --git a/cloudferrylib/os/actions/load_compute_image_to_file.py b/cloudferrylib/os/actions/load_compute_image_to_file.py index 961b6839..59bb2b34 100644 --- a/cloudferrylib/os/actions/load_compute_image_to_file.py +++ b/cloudferrylib/os/actions/load_compute_image_to_file.py @@ -20,13 +20,17 @@ class LoadComputeImageToFile(action.Action): def run(self, info=None, **kwargs): cfg = self.cloud.cloud_config.cloud + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): inst = info[utl.INSTANCES_TYPE][instance_id][utl.INSTANCE_BODY] image_id = inst['image_id'] base_file = "/tmp/%s" % ("temp%s_base" % instance_id) diff_file = "/tmp/%s" % ("temp%s" % instance_id) - with settings(host_string=cfg.host): + + with settings(host_string=cfg.host, + connection_attempts=ssh_attempts): with forward_agent(env.key_filename): cmd = image.glance_image_download_cmd(cfg, image_id, base_file) diff --git a/cloudferrylib/os/actions/map_compute_info.py b/cloudferrylib/os/actions/map_compute_info.py index c30138f7..6ffd987a 100644 --- a/cloudferrylib/os/actions/map_compute_info.py +++ b/cloudferrylib/os/actions/map_compute_info.py @@ -16,6 +16,7 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import utils as utl import copy + INSTANCES = 'instances' DIFF = 'diff' @@ -27,28 +28,28 @@ class MapComputeInfo(action.Action): def run(self, info=None, **kwargs): - new_compute_info = copy.deepcopy(info) + new_info = copy.deepcopy(info) src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] src_flavors_dict = \ - {flavor.id: flavor.name for flavor in src_compute.get_flavor_list()} + {flav.id: flav.name for flav in src_compute.get_flavor_list()} dst_flavors_dict = \ - {flavor.name: flavor.id for flavor in dst_compute.get_flavor_list()} + {flav.name: flav.id for flav in dst_compute.get_flavor_list()} - for instance_id, instance in new_compute_info[utl.INSTANCES_TYPE].iteritems(): + for instance_id, instance in new_info[utl.INSTANCES_TYPE].iteritems(): _instance = instance['instance'] if _instance['flavor_id'] in src_flavors_dict: flavor_name = src_flavors_dict[_instance['flavor_id']] _instance['flavor_id'] = dst_flavors_dict[flavor_name] - #TODO: path_dst is probably non used code, need functional testing + # TODO: path_dst is probably non used code, need functional testing self.dst_cloud.cloud_config.cloud.temp = '-' path_dst = "%s/%s" % (self.dst_cloud.cloud_config.cloud.temp, "temp%s_base" % instance_id) instance[DIFF][PATH_DST] = path_dst instance[DIFF][HOST_DST] = self.dst_cloud.getIpSsh() return { - 'info': new_compute_info + 'info': new_info } diff --git a/cloudferrylib/os/actions/merge_base_and_diff.py b/cloudferrylib/os/actions/merge_base_and_diff.py index da602bd9..99ae1f91 100644 --- a/cloudferrylib/os/actions/merge_base_and_diff.py +++ b/cloudferrylib/os/actions/merge_base_and_diff.py @@ -1,6 +1,5 @@ from fabric.api import run, settings, env from cloudferrylib.base.action import action -from cloudferrylib.utils import forward_agent from cloudferrylib.utils import utils as utl INSTANCES = 'instances' @@ -16,11 +15,15 @@ def run(self, info=None, **kwargs): self.rebase_diff_file(host, base_file, diff_file) self.commit_diff_file(host, diff_file) - def rebase_diff_file(self, host, base_file, diff_file): + @staticmethod + def rebase_diff_file(host, base_file, diff_file): cmd = "qemu-img rebase -u -b %s %s" % (base_file, diff_file) - with settings(host_string=host): + with settings(host_string=host, + connection_attempts=env.connection_attempts): run(cmd) - def commit_diff_file(self, host, diff_file): - with settings(host_string=host): - run("qemu-img commit %s" % diff_file) \ No newline at end of file + @staticmethod + def commit_diff_file(host, diff_file): + with settings(host_string=host, + connection_attempts=env.connection_attempts): + run("qemu-img commit %s" % diff_file) diff --git a/cloudferrylib/os/actions/post_transport_instance.py b/cloudferrylib/os/actions/post_transport_instance.py index 148e374d..f77f21c3 100644 --- a/cloudferrylib/os/actions/post_transport_instance.py +++ b/cloudferrylib/os/actions/post_transport_instance.py @@ -15,22 +15,9 @@ import copy -from fabric.api import env -from fabric.api import run -from fabric.api import settings - from cloudferrylib.base.action import action -from cloudferrylib.os.actions import convert_file_to_image -from cloudferrylib.os.actions import convert_image_to_file -from cloudferrylib.os.actions import convert_volume_to_image -from cloudferrylib.os.actions import copy_g2g from cloudferrylib.os.actions import task_transfer -from cloudferrylib.utils import utils as utl, forward_agent - -from cloudferrylib.utils.drivers import ssh_ceph_to_ceph -from cloudferrylib.utils.drivers import ssh_ceph_to_file -from cloudferrylib.utils.drivers import ssh_file_to_file -from cloudferrylib.utils.drivers import ssh_file_to_ceph +from cloudferrylib.utils import utils as utl CLOUD = 'cloud' @@ -66,7 +53,7 @@ class PostTransportInstance(action.Action): def run(self, info=None, **kwargs): info = copy.deepcopy(info) - #Init before run + # Init before run src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] dst_compute = self.dst_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend @@ -76,7 +63,7 @@ def run(self, info=None, **kwargs): } } - #Get next one instance + # Get next one instance for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): instance_boot = instance[utl.INSTANCE_BODY]['boot_mode'] one_instance = { @@ -84,9 +71,12 @@ def run(self, info=None, **kwargs): instance_id: instance } } - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == ISCSI): - self.copy_diff_file(self.src_cloud, self.dst_cloud, one_instance) + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == ISCSI)): + self.copy_diff_file(self.src_cloud, + self.dst_cloud, + one_instance) new_info[utl.INSTANCES_TYPE].update( one_instance[utl.INSTANCES_TYPE]) @@ -95,7 +85,13 @@ def run(self, info=None, **kwargs): 'info': new_info } - def copy_data_via_ssh(self, src_cloud, dst_cloud, info, body, resources, types): + def copy_data_via_ssh(self, + src_cloud, + dst_cloud, + info, + body, + resources, + types): dst_storage = dst_cloud.resources[resources] src_compute = src_cloud.resources[resources] src_backend = src_compute.config.compute.backend @@ -113,4 +109,4 @@ def copy_diff_file(self, src_cloud, dst_cloud, info): info, utl.DIFF_BODY, utl.COMPUTE_RESOURCE, - utl.INSTANCES_TYPE) \ No newline at end of file + utl.INSTANCES_TYPE) diff --git a/cloudferrylib/os/actions/pre_transport_instance.py b/cloudferrylib/os/actions/pre_transport_instance.py index e7254700..6849f388 100644 --- a/cloudferrylib/os/actions/pre_transport_instance.py +++ b/cloudferrylib/os/actions/pre_transport_instance.py @@ -22,8 +22,6 @@ from cloudferrylib.base.action import action from cloudferrylib.os.actions import convert_file_to_image from cloudferrylib.os.actions import convert_image_to_file -from cloudferrylib.os.actions import convert_volume_to_image -from cloudferrylib.os.actions import copy_g2g from cloudferrylib.os.actions import task_transfer from cloudferrylib.utils import utils as utl, forward_agent @@ -66,7 +64,7 @@ class PreTransportInstance(action.Action): def run(self, info=None, **kwargs): info = copy.deepcopy(info) - #Init before run + # Init before run src_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] dst_compute = self.src_cloud.resources[utl.COMPUTE_RESOURCE] backend_ephem_drv_src = src_compute.config.compute.backend @@ -76,7 +74,7 @@ def run(self, info=None, **kwargs): } } - #Get next one instance + # Get next one instance for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): instance_boot = instance[utl.INSTANCE_BODY]['boot_mode'] one_instance = { @@ -84,12 +82,16 @@ def run(self, info=None, **kwargs): instance_id: instance } } - #Pre processing deploy - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == CEPH): + # Pre processing deploy + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == CEPH)): self.transport_image(self.dst_cloud, one_instance, instance_id) - if (instance_boot == utl.BOOT_FROM_IMAGE) and (backend_ephem_drv_src == ISCSI) \ - and (backend_ephem_drv_dst == CEPH): - self.transport_diff_and_merge(self.dst_cloud, one_instance, instance_id) + if ((instance_boot == utl.BOOT_FROM_IMAGE) and + (backend_ephem_drv_src == ISCSI) and + (backend_ephem_drv_dst == CEPH)): + self.transport_diff_and_merge(self.dst_cloud, + one_instance, + instance_id) new_info[utl.INSTANCES_TYPE].update( one_instance[utl.INSTANCES_TYPE]) @@ -97,15 +99,21 @@ def run(self, info=None, **kwargs): 'info': new_info } - def convert_file_to_image(self, dst_cloud, base_file, disk_format, instance_id): - converter = convert_file_to_image.ConvertFileToImage(self.init, cloud=dst_cloud) + def convert_file_to_image(self, + dst_cloud, + base_file, + disk_format, + instance_id): + converter = convert_file_to_image.ConvertFileToImage(self.init, + cloud=dst_cloud) dst_image_id = converter.run(file_path=base_file, image_format=disk_format, image_name="%s-image" % instance_id) return dst_image_id def convert_image_to_file(self, cloud, image_id, filename): - convertor = convert_image_to_file.ConvertImageToFile(self.init, cloud=cloud) + convertor = convert_image_to_file.ConvertImageToFile(self.init, + cloud=cloud) convertor.run(image_id=image_id, base_filename=filename) @@ -115,7 +123,8 @@ def merge_file(self, cloud, base_file, diff_file): self.commit_diff_file(host, diff_file) def transport_image(self, dst_cloud, info, instance_id): - path_dst = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, "temp%s_base" % instance_id) + path_dst = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, + "temp%s_base" % instance_id) info[INSTANCES][instance_id][DIFF][PATH_DST] = path_dst info[INSTANCES][instance_id][DIFF][HOST_DST] = dst_cloud.getIpSsh() @@ -126,8 +135,8 @@ def transport_image(self, dst_cloud, info, instance_id): resource_root_name=utl.DIFF_BODY) transporter.run(info=info) - converter = convert_file_to_image.ConvertFileToImage(self.init, 'dst_cloud') - + converter = convert_file_to_image.ConvertFileToImage(self.init, + 'dst_cloud') dst_image_id = converter.run(file_path=path_dst, image_format='raw', @@ -135,7 +144,8 @@ def transport_image(self, dst_cloud, info, instance_id): info[INSTANCES][instance_id][INSTANCE_BODY]['image_id'] = dst_image_id def transport_diff_and_merge(self, dst_cloud, info, instance_id): - convertor = convert_image_to_file.ConvertImageToFile(self.init, cloud='dst_cloud') + convertor = convert_image_to_file.ConvertImageToFile(self.init, + cloud='dst_cloud') transporter = task_transfer.TaskTransfer( self.init, TRANSPORTER_MAP[ISCSI][ISCSI], @@ -143,8 +153,10 @@ def transport_diff_and_merge(self, dst_cloud, info, instance_id): resource_root_name=utl.DIFF_BODY) image_id = info[INSTANCES][instance_id][utl.INSTANCE_BODY]['image_id'] - base_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, "temp%s_base" % instance_id) - diff_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, "temp%s" % instance_id) + base_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, + "temp%s_base" % instance_id) + diff_file = "%s/%s" % (dst_cloud.cloud_config.cloud.temp, + "temp%s" % instance_id) info[INSTANCES][instance_id][DIFF][PATH_DST] = diff_file info[INSTANCES][instance_id][DIFF][HOST_DST] = dst_cloud.getIpSsh() @@ -162,26 +174,35 @@ def transport_diff_and_merge(self, dst_cloud, info, instance_id): disk_format = image[utl.IMAGE_BODY]['disk_format'] if image_res.config.image.convert_to_raw: if disk_format.lower() != utl.RAW: - self.convert_file_to_raw(dst_cloud.cloud_config.cloud.host, disk_format, base_file) + self.convert_file_to_raw(dst_cloud.cloud_config.cloud.host, + disk_format, + base_file) disk_format = utl.RAW - converter = convert_file_to_image.ConvertFileToImage(self.init, cloud='dst_cloud') + converter = convert_file_to_image.ConvertFileToImage(self.init, + cloud='dst_cloud') dst_image_id = converter.run(file_path=base_file, image_format=disk_format, image_name="%s-image" % instance_id) info[INSTANCES][instance_id][INSTANCE_BODY]['image_id'] = dst_image_id - def convert_file_to_raw(self, host, disk_format, filepath): - with settings(host_string=host): + @staticmethod + def convert_file_to_raw(host, disk_format, filepath): + with settings(host_string=host, + connection_attempts=env.connection_attempts): with forward_agent(env.key_filename): run("qemu-img convert -f %s -O raw %s %s.tmp" % (disk_format, filepath, filepath)) run("mv -f %s.tmp %s" % (filepath, filepath)) - def rebase_diff_file(self, host, base_file, diff_file): + @staticmethod + def rebase_diff_file(host, base_file, diff_file): cmd = "qemu-img rebase -u -b %s %s" % (base_file, diff_file) - with settings(host_string=host): + with settings(host_string=host, + connection_attempts=env.connection_attempts): run(cmd) - def commit_diff_file(self, host, diff_file): - with settings(host_string=host): - run("qemu-img commit %s" % diff_file) + @staticmethod + def commit_diff_file(host, diff_file): + with settings(host_string=host, + connection_attempts=env.connection_attempts): + run("qemu-img commit %s" % diff_file) diff --git a/cloudferrylib/os/actions/prepare_volumes_data_map.py b/cloudferrylib/os/actions/prepare_volumes_data_map.py index 766fcbd8..d260c9b4 100644 --- a/cloudferrylib/os/actions/prepare_volumes_data_map.py +++ b/cloudferrylib/os/actions/prepare_volumes_data_map.py @@ -49,7 +49,8 @@ def run(self, **kwargs): utl.HOST_DST: dst_host, utl.PATH_DST: dst_path }) - volumes_data_map[dst_id][utl.META_INFO].update(src_volumes[src_id][utl.META_INFO]) + volumes_data_map[dst_id][utl.META_INFO].\ + update(src_volumes[src_id][utl.META_INFO]) volumes = { utl.VOLUMES_TYPE: volumes_data_map } diff --git a/cloudferrylib/os/actions/recreate_boot_image.py b/cloudferrylib/os/actions/recreate_boot_image.py index 602afab2..a5b5ddbb 100644 --- a/cloudferrylib/os/actions/recreate_boot_image.py +++ b/cloudferrylib/os/actions/recreate_boot_image.py @@ -15,7 +15,8 @@ from cloudferrylib.base.action import action from cloudferrylib.utils import files from cloudferrylib.utils import remote_runner -from cloudferrylib.utils.drivers.ssh_chunks import verified_file_copy, remote_md5_sum +from cloudferrylib.utils.drivers.ssh_chunks import verified_file_copy,\ + remote_md5_sum from cloudferrylib.utils import utils import copy import os @@ -43,12 +44,17 @@ def __init__(self, init, cloud=None): password=dst_password, sudo=True) - def run(self, images_info=None, compute_ignored_images={}, missing_images={}, **kwargs): + def run(self, + images_info=None, + compute_ignored_images={}, + missing_images={}, + **kwargs): """ Create boot image on destination based on root disk of instance. Use diff&base images, commit all changes from diff to base, copy base and add as a glance image. - Image ID from source is used as a name of new image because we can't get name of deleted image. + Image ID from source is used as a name of new image because we can't + get name of deleted image. :param images_info: dict with all images on source :param compute_ignored_images: not used, just resending to down level :param missing_images: dict with images that has been removed on source @@ -61,12 +67,15 @@ def run(self, images_info=None, compute_ignored_images={}, missing_images={}, ** for image_id_src, gl_image in images_info['images'].iteritems(): if image_id_src == img_id and not gl_image['image']: diff = gl_image['meta']['instance'][0]['diff']['path_src'] - img_src_host = gl_image['meta']['instance'][0]['diff']['host_src'] + img_src_host = \ + gl_image['meta']['instance'][0]['diff']['host_src'] if img_src_host != self.src_host: LOG.warning('Different host information. ' + - 'Image is located on host {img_src_host},' + - 'but host in the configuration file {src_host}.'.format(img_src_host, - self.src_host)) + 'Image is located on host ' + + '{img_src_host},' + + 'but host in the configuration file ' + + '{src_host}.'.format(img_src_host, + self.src_host)) continue new_img = self.process_image(img_id, diff) gl_image['image']['id'] = new_img['id'] @@ -79,7 +88,8 @@ def run(self, images_info=None, compute_ignored_images={}, missing_images={}, ** def process_image(self, img_id=None, diff=None): """ - Processing image file: copy from source to destination, create glance image + Processing image file: copy from source to destination, + create glance image :param img_id: image ID from source :param diff: diff file of root disk for instance :return: new image ID if image is created @@ -98,13 +108,23 @@ def process_image(self, img_id=None, diff=None): self.src_runner.run('cp {} {}'.format(base, base_file)) qemu_img_src.diff_rebase(base_file, diff_file, self.src_host) qemu_img_src.diff_commit(src_tmp_dir, diff_name, self.src_host) - verified_file_copy(self.src_runner, self.dst_runner, self.dst_user, - base_file, dst_base_file, self.dst_host, 1) + verified_file_copy(self.src_runner, + self.dst_runner, + self.dst_user, + base_file, + dst_base_file, + self.dst_host, + 1) else: - verified_file_copy(self.src_runner, self.dst_runner, self.dst_user, - diff_file, dst_base_file, self.dst_host, 1) + verified_file_copy(self.src_runner, + self.dst_runner, + self.dst_user, + diff_file, + dst_base_file, + self.dst_host, + 1) image_resource = self.dst_cloud.resources[utils.IMAGE_RESOURCE] - id = image_resource.glance_img_create(self.dst_runner, img_id, 'qcow2', + id = image_resource.glance_img_create(img_id, 'qcow2', dst_base_file) checksum = remote_md5_sum(self.dst_runner, dst_base_file) return {'id': id, 'checksum': checksum} diff --git a/cloudferrylib/os/actions/remote_execution.py b/cloudferrylib/os/actions/remote_execution.py index 3c7d9b3f..f96296c9 100644 --- a/cloudferrylib/os/actions/remote_execution.py +++ b/cloudferrylib/os/actions/remote_execution.py @@ -24,7 +24,9 @@ def __init__(self, cloud, host=None, int_host=None, config_migrate=None): self.host = host self.int_host = int_host self.config_migrate = config_migrate - self.remote_exec_obj = SshUtil(self.cloud, self.config_migrate, self.host) + self.remote_exec_obj = SshUtil(self.cloud, + self.config_migrate, + self.host) super(RemoteExecution, self).__init__({}) def run(self, command, **kwargs): diff --git a/cloudferrylib/os/actions/select_boot_volume.py b/cloudferrylib/os/actions/select_boot_volume.py index 30d0b85d..178bba35 100644 --- a/cloudferrylib/os/actions/select_boot_volume.py +++ b/cloudferrylib/os/actions/select_boot_volume.py @@ -9,4 +9,3 @@ def run(self, info=None, **kwargs): return { 'info': info_boot } - diff --git a/cloudferrylib/os/actions/set_volume_id.py b/cloudferrylib/os/actions/set_volume_id.py index 9b0b472a..a984981f 100644 --- a/cloudferrylib/os/actions/set_volume_id.py +++ b/cloudferrylib/os/actions/set_volume_id.py @@ -33,4 +33,3 @@ def run(self, info=None, **kwargs): meta_volume_array.append({'volume': volume}) instance['meta'] = {'volume': meta_volume_array} return {'info': info} - diff --git a/cloudferrylib/os/actions/transport_ephemeral.py b/cloudferrylib/os/actions/transport_ephemeral.py index 14430c0d..b7b16814 100644 --- a/cloudferrylib/os/actions/transport_ephemeral.py +++ b/cloudferrylib/os/actions/transport_ephemeral.py @@ -85,9 +85,11 @@ def run(self, info=None, **kwargs): 'info': new_info } - def delete_remote_file_on_compute(self, path_file, host_cloud, + @staticmethod + def delete_remote_file_on_compute(path_file, host_cloud, host_instance): - with settings(host_string=host_cloud): + with settings(host_string=host_cloud, + connection_attempts=env.connection_attempts): with forward_agent(env.key_filename): run("ssh -oStrictHostKeyChecking=no %s 'rm -rf %s'" % (host_instance, path_file)) @@ -138,7 +140,7 @@ def copy_ephemeral_ceph_to_iscsi(self, src_cloud, dst_cloud, info): qemu_img_dst = dst_cloud.qemu_img qemu_img_src = src_cloud.qemu_img - temp_path_src = temp_src+"/%s"+utl.DISK_EPHEM + temp_path_src = temp_src + "/%s" + utl.DISK_EPHEM for inst_id, inst in instances.iteritems(): path_src_id_temp = temp_path_src % inst_id diff --git a/cloudferrylib/os/actions/transport_instance.py b/cloudferrylib/os/actions/transport_instance.py index cc4505a6..1b19fba1 100644 --- a/cloudferrylib/os/actions/transport_instance.py +++ b/cloudferrylib/os/actions/transport_instance.py @@ -58,7 +58,7 @@ def run(self, info=None, **kwargs): } } - #Get next one instance + # Get next one instance for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): instance = self._replace_user_ids(instance) @@ -105,22 +105,22 @@ def prepare_ephemeral_drv(self, info, new_info, map_new_to_old_ids): instance_new = new_info[INSTANCES][new_id] ephemeral_path_dst = instance_new[EPHEMERAL][PATH_SRC] - instance_new[EPHEMERAL][PATH_DST] = ephemeral_path_dst + instance_new[EPHEMERAL][PATH_DST] = ephemeral_path_dst ephemeral_host_dst = instance_new[EPHEMERAL][HOST_SRC] instance_new[EPHEMERAL][HOST_DST] = ephemeral_host_dst - + diff_path_dst = instance_new[DIFF][PATH_SRC] - instance_new[DIFF][PATH_DST] = diff_path_dst + instance_new[DIFF][PATH_DST] = diff_path_dst diff_host_dst = instance_new[DIFF][HOST_SRC] instance_new[DIFF][HOST_DST] = diff_host_dst ephemeral_path_src = instance_old[EPHEMERAL][PATH_SRC] - instance_new[EPHEMERAL][PATH_SRC] = ephemeral_path_src + instance_new[EPHEMERAL][PATH_SRC] = ephemeral_path_src ephemeral_host_src = instance_old[EPHEMERAL][HOST_SRC] instance_new[EPHEMERAL][HOST_SRC] = ephemeral_host_src - + diff_path_src = instance_old[DIFF][PATH_SRC] - instance_new[DIFF][PATH_SRC] = diff_path_src + instance_new[DIFF][PATH_SRC] = diff_path_src diff_host_src = instance_old[DIFF][HOST_SRC] instance_new[DIFF][HOST_SRC] = diff_host_src diff --git a/cloudferrylib/os/actions/upload_file_to_image.py b/cloudferrylib/os/actions/upload_file_to_image.py index e3be7942..9058295a 100644 --- a/cloudferrylib/os/actions/upload_file_to_image.py +++ b/cloudferrylib/os/actions/upload_file_to_image.py @@ -31,7 +31,9 @@ class UploadFileToImage(action.Action): def run(self, info=None, **kwargs): cfg = self.cloud.cloud_config.cloud + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts img_res = self.cloud.resources[utl.IMAGE_RESOURCE] + for instance_id, instance in info[utl.INSTANCES_TYPE].iteritems(): # init inst_body = info[INSTANCES][instance_id][utl.INSTANCE_BODY] @@ -43,7 +45,8 @@ def run(self, info=None, **kwargs): if img_res.config.image.convert_to_raw: image_format = utl.RAW # action - with settings(host_string=cfg.host): + with settings(host_string=cfg.host, + connection_attempts=ssh_attempts): cmd = image.glance_image_create_cmd(cfg, image_name, image_format, base_file) out = run(cmd) diff --git a/cloudferrylib/os/actions/utils.py b/cloudferrylib/os/actions/utils.py index b899b3c2..dd79634c 100644 --- a/cloudferrylib/os/actions/utils.py +++ b/cloudferrylib/os/actions/utils.py @@ -20,27 +20,38 @@ LOG = utils.get_log(__name__) -def transfer_file_to_file(src_cloud, dst_cloud, host_src, host_dst, path_src, path_dst, cfg_migrate): +def transfer_file_to_file(src_cloud, + dst_cloud, + host_src, + host_dst, + path_src, + path_dst, + cfg_migrate): # TODO: Delete after transport_db_via_ssh action rewriting LOG.debug("| | copy file") ssh_ip_src = src_cloud.getIpSsh() ssh_ip_dst = dst_cloud.getIpSsh() - with settings(host_string=ssh_ip_src): + with settings(host_string=ssh_ip_src, + connection_attempts=env.connection_attempts): with utils.forward_agent(cfg_migrate.key_filename): with utils.up_ssh_tunnel(host_dst, ssh_ip_dst, ssh_ip_src) as port: if cfg_migrate.file_compression == "dd": - run(("ssh -oStrictHostKeyChecking=no %s 'dd bs=1M if=%s' " + - "| ssh -oStrictHostKeyChecking=no -p %s localhost 'dd bs=1M of=%s'") % + run(("ssh -oStrictHostKeyChecking=no %s 'dd bs=1M " + + "if=%s' | ssh -oStrictHostKeyChecking=no " + + "-p %s localhost 'dd bs=1M of=%s'") % (host_src, path_src, port, path_dst)) elif cfg_migrate.file_compression == "gzip": - run(("ssh -oStrictHostKeyChecking=no %s 'gzip -%s -c %s' " + - "| ssh -oStrictHostKeyChecking=no -p %s localhost 'gunzip | dd bs=1M of=%s'") % + run(("ssh -oStrictHostKeyChecking=no " + + "%s 'gzip -%s -c %s' " + + "| ssh -oStrictHostKeyChecking=no -p %s localhost " + + "'gunzip | dd bs=1M of=%s'") % (host_src, cfg_migrate.level_compression, path_src, port, path_dst)) def delete_file_from_rbd(ssh_ip, file_path): - with settings(host_string=ssh_ip): + with settings(host_string=ssh_ip, + connection_attempts=env.connection_attempts): with utils.forward_agent(env.key_filename): run("rbd rm %s" % file_path) diff --git a/cloudferrylib/os/compute/instances.py b/cloudferrylib/os/compute/instances.py index 77ea2316..9a8d361e 100644 --- a/cloudferrylib/os/compute/instances.py +++ b/cloudferrylib/os/compute/instances.py @@ -69,8 +69,10 @@ def cobalt_live_migrate_vm(config, vm_id, dest_host): host_string = "{user}@{host}".format( user=config.cloud.ssh_user, host=config.cloud.ssh_host) - with settings(warn_only=True, host_string=host_string, - key_filename=config.migrate.key_filename): + with settings(warn_only=True, + host_string=host_string, + key_filename=config.migrate.key_filename, + connection_attempts=config.migrate.ssh_connection_attempts): migrate_cmd = clients.os_cli_cmd(config.cloud, "nova", "cobalt-migrate", vm_id, "--dest", dest_host) diff --git a/cloudferrylib/os/compute/nova_compute.py b/cloudferrylib/os/compute/nova_compute.py index ac5e3b51..d3878ba5 100644 --- a/cloudferrylib/os/compute/nova_compute.py +++ b/cloudferrylib/os/compute/nova_compute.py @@ -308,7 +308,7 @@ def convert_instance(instance, cfg, cloud): instance, instance_block_info, is_ceph_ephemeral=is_ceph, - disk=DISK+LOCAL) + disk=DISK + LOCAL) diff = { 'path_src': None, @@ -693,8 +693,7 @@ def change_status(self, status, instance=None, instance_id=None): 'status': lambda status: lambda instance: self.wait_for_status( instance_id, self.get_status, - status, - ) + status) } map_status = { 'paused': { diff --git a/cloudferrylib/os/identity/keystone.py b/cloudferrylib/os/identity/keystone.py index 0d7b2a30..63306e76 100644 --- a/cloudferrylib/os/identity/keystone.py +++ b/cloudferrylib/os/identity/keystone.py @@ -475,7 +475,8 @@ def _passwd_notification(self, email, name, password): def _deploy_roles(self, roles): LOG.info("Role deployment started...") - dst_roles = {role.name.lower(): role.id for role in self.get_roles_list()} + dst_roles = { + role.name.lower(): role.id for role in self.get_roles_list()} for _role in roles: role = _role['role'] if role['name'].lower() not in dst_roles: @@ -503,11 +504,13 @@ def _get_user_tenants_roles(self, tenant_list=None, user_list=None): if user_list is None: user_list = [] if not self.config.migrate.optimize_user_role_fetch: - user_tenants_roles = self._get_user_tenants_roles_by_api(tenant_list, - user_list) + user_tenants_roles = \ + self._get_user_tenants_roles_by_api(tenant_list, + user_list) else: - user_tenants_roles = self._get_user_tenants_roles_by_db(tenant_list, - user_list) + user_tenants_roles = \ + self._get_user_tenants_roles_by_db(tenant_list, + user_list) return user_tenants_roles def _get_roles_sql_request(self): @@ -515,16 +518,17 @@ def _get_roles_sql_request(self): try: is_project_metadata = self.mysql_connector.execute( "SHOW TABLES LIKE 'user_project_metadata'").rowcount - if is_project_metadata: #for grizzly case + if is_project_metadata: # for grizzly case return self.mysql_connector.execute( "SELECT * FROM user_project_metadata") is_assignment = self.mysql_connector.execute( "SHOW TABLES LIKE 'assignment'").rowcount - if is_assignment: #for icehouse case + if is_assignment: # for icehouse case res_raw = self.mysql_connector.execute( "SELECT * FROM assignment") res_tmp = {} - for type_record, actor_id, project_id, role_id, inher_tmp in res_raw: + for (type_record, actor_id, project_id, + role_id, inher_tmp) in res_raw: if (actor_id, project_id) not in res_tmp: res_tmp[(actor_id, project_id)] = {'roles': []} res_tmp[(actor_id, project_id)]['roles'].append(role_id) @@ -609,9 +613,12 @@ def _upload_user_tenant_roles(self, user_tenants_roles, users, tenants): continue for _tenant in tenants: tenant = _tenant['tenant'] - user_roles_objs = _get_user_roles_cached(_user['meta']['new_id'], - _tenant['meta']['new_id']) - exists_roles = [dst_roles[role] if not hasattr(role, 'name') else role.name + user_roles_objs = _get_user_roles_cached( + _user['meta']['new_id'], + _tenant['meta']['new_id']) + exists_roles = [dst_roles[role] if not hasattr(role, + 'name') + else role.name for role in user_roles_objs] for _role in user_tenants_roles[user['name']][tenant['name']]: role = _role['role'] diff --git a/cloudferrylib/os/image/glance_image.py b/cloudferrylib/os/image/glance_image.py index c7dc8e13..903e8123 100644 --- a/cloudferrylib/os/image/glance_image.py +++ b/cloudferrylib/os/image/glance_image.py @@ -29,6 +29,7 @@ from cloudferrylib.base import image from cloudferrylib.utils import file_like_proxy from cloudferrylib.utils import utils as utl +from cloudferrylib.utils import remote_runner LOG = utl.get_log(__name__) @@ -50,6 +51,8 @@ def __init__(self, config, cloud): self.filter_image = [] # get mysql settings self.mysql_connector = cloud.mysql_connector('glance') + self.runner = remote_runner.RemoteRunner(self.host, + self.config.cloud.ssh_user) super(GlanceImage, self).__init__(config) @property @@ -89,7 +92,8 @@ def get_image_list(self): filters = {'is_public': None} img_list = get_img_list(filters=filters) # getting images if tenant is member - for img in self.glance_client.image_members.list(member=self.filter_tenant_id): + for img in self.glance_client.image_members.list( + member=self.filter_tenant_id): for i in img_list: if i.id == img.image_id: LOG.debug("append image(by member) ID {}".format(i.id)) @@ -278,11 +282,26 @@ def make_image_info(self, glance_image, info): if glance_image.status == "active": gl_image = self.convert(glance_image, self.cloud) - info['images'][glance_image.id] = {'image': gl_image, - 'meta': {}, - } - LOG.debug("find image with ID {}({})".format(glance_image.id, - glance_image.name)) + command = ("SELECT value FROM image_locations " + "WHERE image_id=\"{}\" AND deleted=\"0\";" + .format(glance_image.id)) + res = self.mysql_connector.execute(command) + img_loc = None + for row in res: + if img_loc is not None: + LOG.warning("ignoring multi locations for image {}" + .format(glance_image.name)) + break + img_loc = row[0] + + info['images'][glance_image.id] = { + 'image': gl_image, + 'meta': { + 'img_loc': img_loc + }, + } + LOG.debug("find image with ID {}({})" + .format(glance_image.id, glance_image.name)) else: LOG.warning("image {img} was not migrated according to " "status = {status}, (expected status " @@ -345,6 +364,26 @@ def deploy(self, info): self.identity_client.keystone_client.users.find( username=metadata["user_name"]).id del metadata["user_name"] + if gl_image["image"]["checksum"] is None: + LOG.warning("re-creating image {} " + "from original source URL" + .format(gl_image["image"]["id"])) + if meta['img_loc'] is not None: + self.glance_img_create( + gl_image['image']['name'], + gl_image['image']['disk_format'] or "qcow2", + meta['img_loc'] + ) + recreated_image = utl.ext_dict( + name=gl_image["image"]["name"] + ) + migrate_images_list.append( + (recreated_image, gl_image['meta']) + ) + else: + raise exception.AbortMigrationError( + "image information has no original source URL") + continue LOG.debug("Creating image '{image}' ({image_id})".format( image=gl_image["image"]["name"], @@ -362,7 +401,8 @@ def deploy(self, info): name=gl_image['image']['name'], container_format=(gl_image['image']['container_format'] or "bare"), - disk_format=gl_image['image']['disk_format'] or "qcow2", + disk_format=(gl_image['image']['disk_format'] + or "qcow2"), is_public=gl_image['image']['is_public'], protected=gl_image['image']['protected'], owner=gl_image['image']['owner'], @@ -439,17 +479,20 @@ def get_status(self, res_id): return self.glance_client.images.get(res_id).status def patch_image(self, backend_storage, image_id): + ssh_attempts = self.cloud.cloud_config.migrate.ssh_connection_attempts + if backend_storage == 'ceph': image_from_glance = self.get_image_by_id(image_id) - with settings(host_string=self.cloud.getIpSsh()): + with settings(host_string=self.cloud.getIpSsh(), + connection_attempts=ssh_attempts): out = json.loads( run("rbd -p images info %s --format json" % image_id)) image_from_glance.update(size=out["size"]) - def glance_img_create(self, runner, image_name, image_format, file_path): + def glance_img_create(self, img_name, img_format, file_path): cfg = self.cloud.cloud_config.cloud - cmd = image.glance_image_create_cmd(cfg, image_name, image_format, + cmd = image.glance_image_create_cmd(cfg, img_name, img_format, file_path) - out = runner.run(cmd) + out = self.runner.run(cmd) image_id = out.split("|")[2].replace(' ', '') return image_id diff --git a/cloudferrylib/os/network/__init__.py b/cloudferrylib/os/network/__init__.py index a3e51dd3..ba605e43 100644 --- a/cloudferrylib/os/network/__init__.py +++ b/cloudferrylib/os/network/__init__.py @@ -11,4 +11,3 @@ # implied. # See the License for the specific language governing permissions and# # limitations under the License. - diff --git a/cloudferrylib/os/network/neutron.py b/cloudferrylib/os/network/neutron.py index d5f7822a..9ac928bc 100644 --- a/cloudferrylib/os/network/neutron.py +++ b/cloudferrylib/os/network/neutron.py @@ -15,11 +15,10 @@ import ipaddr import netaddr -import collections from neutronclient.common import exceptions as neutron_exc from neutronclient.v2_0 import client as neutron_client -from cloudferrylib.base import network, exception +from cloudferrylib.base import network from cloudferrylib.os.identity import keystone as ksresource from cloudferrylib.utils import utils as utl @@ -881,7 +880,7 @@ def upload_lb_pools(self, pools, subnets): 'subnet_id': snet_id, 'protocol': pool['protocol'], 'lb_method': pool['lb_method'] - } + } } LOG.debug("Creating LB pool '%s'", pool['name']) pool['meta']['id'] = \ diff --git a/cloudferrylib/os/network/nova_network.py b/cloudferrylib/os/network/nova_network.py index 2ba3b0f8..342c06fd 100644 --- a/cloudferrylib/os/network/nova_network.py +++ b/cloudferrylib/os/network/nova_network.py @@ -92,8 +92,10 @@ def get_func_mac_address(self, instance): def get_mac_addresses(self, instance): compute_node = getattr(instance, nova_compute.INSTANCE_HOST_ATTRIBUTE) libvirt_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name') + ssh_attempts = self.config.migrate.ssh_connection_attempts - with settings(host_string=self.config['host']): + with settings(host_string=self.config['host'], + connection_attempts=ssh_attempts): with forward_agent(env.key_filename): cmd = "virsh dumpxml %s | grep 'mac address' | " \ "cut -d\\' -f2" % libvirt_name diff --git a/cloudferrylib/os/object_storage/__init__.py b/cloudferrylib/os/object_storage/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/os/object_storage/__init__.py +++ b/cloudferrylib/os/object_storage/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/os/object_storage/swift_storage.py b/cloudferrylib/os/object_storage/swift_storage.py index ba51df02..5285f521 100644 --- a/cloudferrylib/os/object_storage/swift_storage.py +++ b/cloudferrylib/os/object_storage/swift_storage.py @@ -40,15 +40,20 @@ def get_swift_conn(self, params=None): return conn.get_auth() def read_info(self, **kwargs): - info = {utl.OBJSTORAGE_RESOURCE: - {utl.CONTAINERS: {}}} + info = { + utl.OBJSTORAGE_RESOURCE: { + utl.CONTAINERS: {} + } + } account_info = self.get_account_info() info[utl.OBJSTORAGE_RESOURCE][utl.CONTAINERS] = account_info[1] for container_info in info[utl.OBJSTORAGE_RESOURCE][utl.CONTAINERS]: - container_info['objects'] = self.get_container(container_info['name'])[1] + container_info['objects'] = \ + self.get_container(container_info['name'])[1] for object_info in container_info['objects']: - resp, object_info['data'] = self.get_object(container_info['name'], - object_info['name']) + resp, object_info['data'] = \ + self.get_object(container_info['name'], + object_info['name']) return info def deploy(self, info, **kwargs): @@ -65,30 +70,35 @@ def get_account_info(self): return swift_client.get_account(self.storage_url, self.token) def get_container(self, container, *args): - return swift_client.get_container(self.storage_url, self.token, container, *args) + return swift_client.get_container(self.storage_url, + self.token, + container, *args) def get_object(self, container, obj_name, *args): - return swift_client.get_object(self.storage_url, self.token, container, obj_name, *args) - - def put_object(self, container, obj_name, content=None, content_type=None, *args): - return swift_client.put_object(self.storage_url, self.token, container, obj_name, content, content_type, *args) + return swift_client.get_object(self.storage_url, + self.token, + container, + obj_name, *args) + + def put_object(self, + container, + obj_name, + content=None, + content_type=None, + *args): + return swift_client.put_object(self.storage_url, + self.token, + container, + obj_name, + content, + content_type, + *args) def put_container(self, container, *args): - return swift_client.put_container(self.storage_url, self.token, container, *args) + return swift_client.put_container(self.storage_url, + self.token, + container, *args) def delete_container(self, container, *args): - return swift_client.delete_container(self.storage_url, self.token, container) - - - - - - - - - - - - - - + return swift_client.delete_container(self.storage_url, + self.token, container) diff --git a/cloudferrylib/os/storage/cinder_database.py b/cloudferrylib/os/storage/cinder_database.py index d2c86efa..30801b1a 100644 --- a/cloudferrylib/os/storage/cinder_database.py +++ b/cloudferrylib/os/storage/cinder_database.py @@ -98,9 +98,13 @@ def list_of_dicts_for_table(self, table): if DELETED in column_names: result = filter(lambda a: a.get(DELETED) == 0, result) if PROJECT_ID in column_names: - result = filter(lambda e: self._check_update_tenant_names(e, PROJECT_ID), result) + result = \ + filter(lambda e: self._check_update_tenant_names( + e, PROJECT_ID), result) if TENANT_ID in column_names: - result = filter(lambda e: self._check_update_tenant_names(e, TENANT_ID), result) + result = \ + filter(lambda e: self._check_update_tenant_names( + e, TENANT_ID), result) if STATUS in column_names: result = filter(lambda e: 'error' not in e[STATUS], result) if USER_ID in column_names: @@ -150,7 +154,8 @@ def get_key_and_auto_increment(cursor, table): auto_increment = cursor.fetchone().get("auto_increment") return primary_key, auto_increment - def filter_data(existing_data, data_to_be_added, primary_key, auto_increment): + def filter_data(existing_data, data_to_be_added, primary_key, + auto_increment): """ handle duplicates in database """ existing_hash = {i.get(primary_key): i for i in existing_data} unique_entries, duplicated_pk = [], [] @@ -171,7 +176,8 @@ def filter_data(existing_data, data_to_be_added, primary_key, auto_increment): # add entry to database without primary_key # primary key will be generated automaticaly duplicated_pk.append( - {i: candidate[i] for i in candidate if i != primary_key}) + {i: candidate[i] for i in candidate + if i != primary_key}) else: unique_entries.append(candidate) return unique_entries, duplicated_pk diff --git a/cloudferrylib/os/storage/cinder_storage.py b/cloudferrylib/os/storage/cinder_storage.py index efc4fbe2..1f687368 100644 --- a/cloudferrylib/os/storage/cinder_storage.py +++ b/cloudferrylib/os/storage/cinder_storage.py @@ -16,7 +16,6 @@ from cinderclient.v1 import client as cinder_client from cloudferrylib.base import storage -from cloudferrylib.utils import mysql_connector from cloudferrylib.utils import utils as utl LOG = utl.get_log(__name__) diff --git a/cloudferrylib/scheduler/base/begin_task.py b/cloudferrylib/scheduler/base/begin_task.py index c195bbcf..596bff1a 100644 --- a/cloudferrylib/scheduler/base/begin_task.py +++ b/cloudferrylib/scheduler/base/begin_task.py @@ -20,4 +20,4 @@ class BeginTask(Task): def __init__(self, **kwargs): self.__dict__.update(kwargs) - super(BeginTask, self).__init__() \ No newline at end of file + super(BeginTask, self).__init__() diff --git a/cloudferrylib/scheduler/base/end_task.py b/cloudferrylib/scheduler/base/end_task.py index 8bb911ee..a4e0d1b0 100644 --- a/cloudferrylib/scheduler/base/end_task.py +++ b/cloudferrylib/scheduler/base/end_task.py @@ -20,4 +20,4 @@ class EndTask(Task): def __init__(self, **kwargs): self.__dict__.update(kwargs) - super(EndTask, self).__init__() \ No newline at end of file + super(EndTask, self).__init__() diff --git a/cloudferrylib/scheduler/cursor.py b/cloudferrylib/scheduler/cursor.py index 6e0ad89a..44b55b17 100644 --- a/cloudferrylib/scheduler/cursor.py +++ b/cloudferrylib/scheduler/cursor.py @@ -33,7 +33,8 @@ def next(self): if self.threads: return self.threads.pop() if self.next_iter.next_element[0]: - if self.next_iter.num_element < len(self.next_iter.next_element): + if self.next_iter.num_element < len( + self.next_iter.next_element): self.__change_state_cursor(self.next_iter.num_element) else: self.__change_state_cursor(DEFAULT) diff --git a/cloudferrylib/scheduler/namespace.py b/cloudferrylib/scheduler/namespace.py index f3529570..752ba3c8 100644 --- a/cloudferrylib/scheduler/namespace.py +++ b/cloudferrylib/scheduler/namespace.py @@ -20,9 +20,10 @@ class Namespace: def __init__(self, vars={}): - if not CHILDREN in vars: + if CHILDREN not in vars: vars[CHILDREN] = dict() self.vars = vars def fork(self, is_deep_copy=False): - return Namespace(copy.copy(self.vars)) if not is_deep_copy else Namespace(copy.deepcopy(self.vars)) + return Namespace(copy.copy(self.vars)) if not is_deep_copy \ + else Namespace(copy.deepcopy(self.vars)) diff --git a/cloudferrylib/scheduler/scenario.py b/cloudferrylib/scheduler/scenario.py index bbec3303..59800c2d 100644 --- a/cloudferrylib/scheduler/scenario.py +++ b/cloudferrylib/scheduler/scenario.py @@ -47,7 +47,9 @@ def init_tasks(self, init={}): args = args[:-1] else: args_map = {} - tasks[task] = actions[tasks_file['tasks'][task][0]](init, *args, **args_map) + tasks[task] = actions[tasks_file['tasks'][task][0]](init, + *args, + **args_map) self.tasks = tasks def load_scenario(self, path_scenario=None): @@ -58,7 +60,8 @@ def load_scenario(self, path_scenario=None): self.namespace = migrate.get('namespace', {}) # "migration" yaml chain is responsible for migration self.migration = migrate.get("process") - # "preparation" yaml chain can be added to process pre-migration tasks + # "preparation" yaml chain can be added to process pre-migration + # tasks self.preparation = migrate.get("preparation") # "rollback" yaml chain can be added to rollback to previous state # in case of main chain failure @@ -110,11 +113,13 @@ def get_actions(self, mod): list_name_modules = map(lambda x: x.string.replace(".py", ""), modules_matches) - modules = [imp.load_source(name, path+'/%s.py' % name) for name in list_name_modules] + modules = [imp.load_source(name, path + '/%s.py' % name) + for name in list_name_modules] actions = {} for module in modules: for item in module.__dict__: - if inspect.isclass(module.__dict__[item]) and issubclass(module.__dict__[item], action.Action): + if inspect.isclass(module.__dict__[item]) and \ + issubclass(module.__dict__[item], action.Action): actions[item] = module.__dict__[item] return actions diff --git a/cloudferrylib/scheduler/task.py b/cloudferrylib/scheduler/task.py index a086a1a4..1c7d21af 100644 --- a/cloudferrylib/scheduler/task.py +++ b/cloudferrylib/scheduler/task.py @@ -65,7 +65,8 @@ def add_thread(self, other): def dual_link_with(self, other): other = other.go_start() self.next_element[0] = other - other.prev_element = self if not other.prev_element else other.prev_element + other.prev_element = self if not other.prev_element \ + else other.prev_element return other.go_end() @@ -103,5 +104,3 @@ def __repr__(self): class Task(BaseTask): pass - - diff --git a/cloudferrylib/scheduler/utils.py b/cloudferrylib/scheduler/utils.py index d48a9726..f6aaa6ef 100644 --- a/cloudferrylib/scheduler/utils.py +++ b/cloudferrylib/scheduler/utils.py @@ -22,4 +22,4 @@ def chain(net=None): if net: net.go_end() >> e_t b_t >> net.go_begin() - return b_t, e_t \ No newline at end of file + return b_t, e_t diff --git a/cloudferrylib/scheduler/utils/equ_instance.py b/cloudferrylib/scheduler/utils/equ_instance.py index 7c105b1d..e69b1e90 100644 --- a/cloudferrylib/scheduler/utils/equ_instance.py +++ b/cloudferrylib/scheduler/utils/equ_instance.py @@ -23,4 +23,4 @@ def __hash__(self): return hash(self.__class__.__name__) def __eq__(self, other): - return hash(self) == hash(other) \ No newline at end of file + return hash(self) == hash(other) diff --git a/cloudferrylib/utils/__init__.py b/cloudferrylib/utils/__init__.py index ca5e4a2b..e6a088e0 100644 --- a/cloudferrylib/utils/__init__.py +++ b/cloudferrylib/utils/__init__.py @@ -14,4 +14,4 @@ __author__ = 'mirrorcoder' -from utils import * \ No newline at end of file +from utils import * diff --git a/cloudferrylib/utils/console_cmd.py b/cloudferrylib/utils/console_cmd.py index b25a1793..b5d598e6 100644 --- a/cloudferrylib/utils/console_cmd.py +++ b/cloudferrylib/utils/console_cmd.py @@ -32,4 +32,4 @@ def __call__(self, *args): return BC(str(self) % args) def __str__(self): - return self.command \ No newline at end of file + return self.command diff --git a/cloudferrylib/utils/drivers/__init__.py b/cloudferrylib/utils/drivers/__init__.py index 8b137891..e69de29b 100644 --- a/cloudferrylib/utils/drivers/__init__.py +++ b/cloudferrylib/utils/drivers/__init__.py @@ -1 +0,0 @@ - diff --git a/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py b/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py index 8443afe7..099d0b8b 100644 --- a/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py +++ b/cloudferrylib/utils/drivers/ssh_ceph_to_ceph.py @@ -16,8 +16,6 @@ from fabric.api import env from fabric.api import settings -from cloudferrylib.os.actions import utils as action_utils - from cloudferrylib.utils import cmd_cfg from cloudferrylib.utils import driver_transporter from cloudferrylib.utils import rbd_util @@ -33,24 +31,28 @@ def transfer(self, data, snapshot=None, snapshot_type=1): else self.src_cloud.getIpSsh()) host_dst = (data.get('host_dst') if data.get('host_dst') else self.dst_cloud.getIpSsh()) - with settings(host_string=host_src), utils.forward_agent( - env.key_filename): + with (settings(host_string=host_src, + connection_attempts=env.connection_attempts), + utils.forward_agent(env.key_filename)): rbd_import_diff = rbd_util.RbdUtil.rbd_import_diff_cmd ssh_cmd = cmd_cfg.ssh_cmd ssh_rbd_import_diff = ssh_cmd(host_dst, rbd_import_diff) if snapshot: - process_params = [snapshot['name'], data['path_src'], '-', '-', data['path_dst']] + process_params = [snapshot['name'], data['path_src'], '-', '-', + data['path_dst']] if snapshot_type == 1: rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_snap_cmd elif snapshot_type == 2: - rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_from_snap_cmd + rbd_export_diff = \ + rbd_util.RbdUtil.rbd_export_diff_from_snap_cmd process_params.insert(0, snapshot['prev_snapname']) elif snapshot_type == 3: rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_from_cmd else: - raise ValueError("Unsupported snapshot type %s", snapshot_type) + raise ValueError("Unsupported snapshot type %s", + snapshot_type) else: rbd_export_diff = rbd_util.RbdUtil.rbd_export_diff_cmd process_params = [data['path_src'], '-', '-', data['path_dst']] diff --git a/cloudferrylib/utils/drivers/ssh_chunks.py b/cloudferrylib/utils/drivers/ssh_chunks.py index ad672052..4e88c667 100644 --- a/cloudferrylib/utils/drivers/ssh_chunks.py +++ b/cloudferrylib/utils/drivers/ssh_chunks.py @@ -69,7 +69,7 @@ def verified_file_copy(src_runner, dst_runner, dst_user, src_path, dst_path, """ copy_failed = True attempt = 0 - while copy_failed and attempt < num_retries+1: + while copy_failed and attempt < num_retries + 1: attempt += 1 try: LOG.info("Copying file '%s' to '%s', attempt '%d'", @@ -155,7 +155,7 @@ def transfer(self, data): src_md5 = remote_md5_sum(src_runner, src_path) - num_blocks = int(math.ceil(float(file_size)/block_size)) + num_blocks = int(math.ceil(float(file_size) / block_size)) for i in xrange(num_blocks): part = os.path.basename(src_path) + '.part{i}'.format(i=i) diff --git a/cloudferrylib/utils/drivers/ssh_file_to_ceph.py b/cloudferrylib/utils/drivers/ssh_file_to_ceph.py index 78dfef2b..7c5cf930 100644 --- a/cloudferrylib/utils/drivers/ssh_file_to_ceph.py +++ b/cloudferrylib/utils/drivers/ssh_file_to_ceph.py @@ -32,8 +32,9 @@ def transfer(self, data): ssh_ip_src = self.src_cloud.getIpSsh() ssh_ip_dst = self.dst_cloud.getIpSsh() action_utils.delete_file_from_rbd(ssh_ip_dst, data['path_dst']) - with settings(host_string=ssh_ip_src), utils.forward_agent( - env.key_filename): + with (settings(host_string=ssh_ip_src, + connection_attempts=env.connection_attempts), + utils.forward_agent(env.key_filename)): rbd_import = rbd_util.RbdUtil.rbd_import_cmd ssh_cmd_dst = cmd_cfg.ssh_cmd ssh_dst = ssh_cmd_dst(ssh_ip_dst, rbd_import) diff --git a/cloudferrylib/utils/drivers/ssh_file_to_file.py b/cloudferrylib/utils/drivers/ssh_file_to_file.py index c7a2ca21..e5ce7947 100644 --- a/cloudferrylib/utils/drivers/ssh_file_to_file.py +++ b/cloudferrylib/utils/drivers/ssh_file_to_file.py @@ -69,14 +69,16 @@ def transfer(self, data): self.src_cloud.ssh_util.execute(process) def transfer_direct(self, data): + ssh_attempts = self.cfg.migrate.ssh_connection_attempts LOG.debug("| | copy file") if self.cfg.src.ssh_user != 'root' or self.cfg.dst.ssh_user != 'root': LOG.critical("This operation needs 'sudo' access rights, that is " "currently not implemented in this driver. Please use" " 'CopyFilesBetweenComputeHosts' driver from " "cloudferrylib/utils/drivers/.") - with settings(host_string=data['host_src']), utils.forward_agent( - self.cfg.migrate.key_filename): + with (settings(host_string=data['host_src'], + connection_attempts=ssh_attempts), + utils.forward_agent(self.cfg.migrate.key_filename)): if self.cfg.migrate.file_compression == "dd": dd_dst = cmd_cfg.dd_cmd_of ssh_cmd_dst = cmd_cfg.ssh_cmd diff --git a/cloudferrylib/utils/file_like_proxy.py b/cloudferrylib/utils/file_like_proxy.py index 584072f4..c5bde36f 100644 --- a/cloudferrylib/utils/file_like_proxy.py +++ b/cloudferrylib/utils/file_like_proxy.py @@ -27,7 +27,8 @@ class FileLikeProxy: def __init__(self, transfer_object, callback, speed_limit='1mb'): - self.__callback = callback if callback else lambda size, length, obj_id, name: True + self.__callback = callback if callback \ + else lambda size, length, obj_id, name: True self.resp = transfer_object['resource'].get_ref_image( transfer_object['id']) self.length = ( diff --git a/cloudferrylib/utils/proxy_client.py b/cloudferrylib/utils/proxy_client.py index b89f6895..fb207a6e 100644 --- a/cloudferrylib/utils/proxy_client.py +++ b/cloudferrylib/utils/proxy_client.py @@ -80,6 +80,8 @@ def __call__(self, *args, **kwargs): def __getattr__(self, name): attr = getattr(self.client, name) - if inspect.ismethod(attr) or (type(attr) is method_wrapper) or is_wrapping(attr): + if inspect.ismethod(attr) or \ + (type(attr) is method_wrapper) or \ + is_wrapping(attr): return Proxy(attr, self.retry, self.wait_time) return attr diff --git a/cloudferrylib/utils/qemu_img.py b/cloudferrylib/utils/qemu_img.py index a8d82594..e330d81f 100644 --- a/cloudferrylib/utils/qemu_img.py +++ b/cloudferrylib/utils/qemu_img.py @@ -27,7 +27,8 @@ class QemuImg(ssh_util.SshUtil): commit_cd_cmd = cmd_cfg.cd_cmd & commit_cmd convert_cmd = cmd_cfg.qemu_img_cmd("convert %s") convert_full_image_cmd = cmd_cfg.cd_cmd & convert_cmd("-f %s -O %s %s %s") - backing_file_cmd = cmd_cfg.qemu_img_cmd("info %s") >> cmd_cfg.grep_cmd("\"backing file\"") + backing_file_cmd = \ + cmd_cfg.qemu_img_cmd("info %s") >> cmd_cfg.grep_cmd("\"backing file\"") rebase_cmd = cmd_cfg.qemu_img_cmd("rebase -u -b %s %s") convert_cmd = convert_cmd("-O %s %s %s") @@ -50,12 +51,13 @@ def convert_image(self, cmd2 = cmd_cfg.move_cmd(path_to_image, baseimage_tmp, baseimage) - return self.execute(cmd1, host_compute), \ - self.execute(cmd2, host_compute) + return \ + self.execute(cmd1, host_compute), self.execute(cmd2, host_compute) def detect_backing_file(self, dest_disk_ephemeral, host_instance): cmd = self.backing_file_cmd(dest_disk_ephemeral) - return self.parsing_output_backing(self.execute(cmd=cmd, host_exec=host_instance, ignore_errors=True)) + return self.parsing_output_backing( + self.execute(cmd=cmd, host_exec=host_instance, ignore_errors=True)) @staticmethod def parsing_output_backing(output): diff --git a/cloudferrylib/utils/rbd_util.py b/cloudferrylib/utils/rbd_util.py index 61cc3a76..afc141e4 100644 --- a/cloudferrylib/utils/rbd_util.py +++ b/cloudferrylib/utils/rbd_util.py @@ -24,12 +24,14 @@ class RbdUtil(ssh_util.SshUtil): rbd_export_cmd = cmd_cfg.rbd_cmd("export %s %s") rbd_export_diff_cmd = cmd_cfg.rbd_cmd("export-diff %s %s") rbd_export_diff_snap_cmd = cmd_cfg.rbd_cmd("export-diff --snap %s %s %s") - rbd_export_diff_from_snap_cmd = cmd_cfg.rbd_cmd("export-diff --from-snap %s --snap %s %s %s") - rbd_export_diff_from_cmd = cmd_cfg.rbd_cmd("export-diff --from-snap %s %s %s") + rbd_export_diff_from_snap_cmd = \ + cmd_cfg.rbd_cmd("export-diff --from-snap %s --snap %s %s %s") + rbd_export_diff_from_cmd = \ + cmd_cfg.rbd_cmd("export-diff --from-snap %s %s %s") rbd_info_cmd = cmd_cfg.rbd_cmd("-p %s info %s --format %s") rbd_snap_rm = cmd_cfg.rbd_cmd("snap rm %s@%s") - #exmaple pool=compute filename = %s_disk.local % instance_id + # exmaple pool=compute filename = %s_disk.local % instance_id def rm(self, pool, filename, int_host=None): cmd = self.rbd_rm_cmd(pool, filename) return self.execute(cmd, int_host) @@ -38,27 +40,27 @@ def snap_rm(self, volume_path, snapshot_name, int_host=None): cmd = self.rbd_snap_rm(volume_path, snapshot_name) return self.execute(cmd, int_host) - #example image-format=2 output="-" filename=%s_disk.local + # example image-format=2 output="-" filename=%s_disk.local def rbd_import(self, image_format, output, filename, int_host=None): cmd = self.rbd_import_cmd(image_format, output, filename) return self.execute(cmd, int_host) - #example output="-" ceph_path=%s_disk.local + # example output="-" ceph_path=%s_disk.local def rbd_import_diff(self, output, ceph_path, int_host=None): cmd = self.rbd_import_cmd(output, ceph_path) return self.execute(cmd, int_host) - #example filename=volume-id1 output=- + # example filename=volume-id1 output=- def rbd_export(self, filename, output, int_host=None): cmd = self.rbd_export_cmd(filename, output) return self.execute(cmd, int_host) - #example ceph_path=volume-id1 output=- + # example ceph_path=volume-id1 output=- def rbd_export_diff(self, ceph_path, output, int_host=None): cmd = self.rbd_export_cmd(ceph_path, output) return self.execute(cmd, int_host) - #pool=images filename=image_id format=json + # pool=images filename=image_id format=json def rbd_get_info(self, pool, filename, format_output='json', int_host=None): cmd = self.rbd_info_cmd(pool, filename, format_output) diff --git a/cloudferrylib/utils/remote_runner.py b/cloudferrylib/utils/remote_runner.py index f5a75c60..492a767d 100644 --- a/cloudferrylib/utils/remote_runner.py +++ b/cloudferrylib/utils/remote_runner.py @@ -28,7 +28,8 @@ class RemoteExecutionError(RuntimeError): class RemoteRunner(object): - def __init__(self, host, user, password=None, sudo=False, key=None, ignore_errors=False): + def __init__(self, host, user, password=None, sudo=False, key=None, + ignore_errors=False): self.host = host if key is None: key = cfglib.CONF.migrate.key_filename @@ -43,13 +44,16 @@ def run(self, cmd): if not self.ignore_errors: abort_exception = RemoteExecutionError + ssh_attempts = cfglib.CONF.migrate.ssh_connection_attempts + with settings(warn_only=self.ignore_errors, host_string=self.host, user=self.user, password=self.password, abort_exception=abort_exception, reject_unkown_hosts=False, - combine_stderr=False): + combine_stderr=False, + connection_attempts=ssh_attempts): with forward_agent(self.key): LOG.debug("running '%s' on '%s' host as user '%s'", cmd, self.host, self.user) diff --git a/cloudferrylib/utils/ssh_util.py b/cloudferrylib/utils/ssh_util.py index 1e465e4f..d7b452c8 100644 --- a/cloudferrylib/utils/ssh_util.py +++ b/cloudferrylib/utils/ssh_util.py @@ -24,13 +24,15 @@ def __init__(self, cloud, config_migrate, host=None): self.host = host if host else cloud.host self.config_migrate = config_migrate - def execute(self, cmd, internal_host=None, host_exec=None, ignore_errors=False): + def execute(self, cmd, internal_host=None, host_exec=None, + ignore_errors=False): host = host_exec if host_exec else self.host - runner = remote_runner.RemoteRunner(host, - self.cloud.ssh_user, - password=self.cloud.ssh_sudo_password, - sudo=False, - ignore_errors=ignore_errors) + runner = \ + remote_runner.RemoteRunner(host, + self.cloud.ssh_user, + password=self.cloud.ssh_sudo_password, + sudo=False, + ignore_errors=ignore_errors) if internal_host: return self.execute_on_inthost(runner, str(cmd), internal_host) else: diff --git a/cloudferrylib/utils/utils.py b/cloudferrylib/utils/utils.py index 49cab0db..97af27cf 100644 --- a/cloudferrylib/utils/utils.py +++ b/cloudferrylib/utils/utils.py @@ -96,6 +96,9 @@ up_ssh_tunnel = None +SSH_CMD = \ + "ssh -oStrictHostKeyChecking=no -L %s:%s:22 -R %s:localhost:%s %s -Nf" + class ext_dict(dict): def __getattr__(self, name): @@ -105,8 +108,8 @@ def __getattr__(self, name): def get_snapshots_list_repository(path=PATH_TO_SNAPSHOTS): - path_source = path+'/source' - path_dest = path+'/dest' + path_source = path + '/source' + path_dest = path + '/dest' s = os.listdir(path_source) s.sort() source = [{'path': '%s/%s' % (path_source, f), @@ -137,19 +140,22 @@ def convert_to_dict(obj, ident=0, limit_ident=6): ident += 1 if type(obj) in primitive: return obj - if isinstance(obj, inspect.types.InstanceType) or (type(obj) not in (list, tuple, dict)): + if isinstance(obj, inspect.types.InstanceType) or \ + (type(obj) not in (list, tuple, dict)): if ident <= limit_ident: try: obj = obj.convert_to_dict() - except AttributeError as e: + except AttributeError: try: t = obj.__dict__ t['_type_class'] = str(obj.__class__) obj = t - except AttributeError as e: - return str(obj.__class__ if hasattr(obj, '__class__') else type(obj)) + except AttributeError: + return str(obj.__class__ if hasattr(obj, '__class__') + else type(obj)) else: - return str(obj.__class__ if hasattr(obj, '__class__') else type(obj)) + return str(obj.__class__ if hasattr(obj, '__class__') + else type(obj)) if type(obj) is dict: res = {} for item in obj: @@ -238,13 +244,16 @@ def render(self, name_file, args): config.dictConfig(yaml.load(logging_config)) LOGGER = logging.getLogger("CF") + def configure_logging(level): # redefine default logging level LOGGER.setLevel(level) + def get_log(name): return LOGGER + class StackCallFunctions(object): def __init__(self): self.stack_call_functions = [] @@ -288,7 +297,8 @@ def decorator(func): @wraps(func) def inner(*args, **kwargs): stack_call_functions.append(func.__name__, args, kwargs) - log.info("%s> Step %s" % ("- - "*stack_call_functions.depth(), func.__name__)) + log.info("%s> Step %s" % ("- - " * stack_call_functions.depth(), + func.__name__)) res = func(*args, **kwargs) stack_call_functions.pop(res) return res @@ -298,7 +308,8 @@ def inner(*args, **kwargs): class forward_agent(object): """ - Forwarding ssh-key for access on to source and destination clouds via ssh + Forwarding ssh-key for access on to source and + destination clouds via ssh """ def __init__(self, key_file): @@ -306,7 +317,8 @@ def __init__(self, key_file): def _agent_already_running(self): with settings(hide('warnings', 'running', 'stdout', 'stderr'), - warn_only=True): + warn_only=True, + connection_attempts=env.connection_attempts): res = local("ssh-add -l", capture=True) if res.succeeded: @@ -339,7 +351,8 @@ def __exit__(self, type, value, traceback): class wrapper_singletone_ssh_tunnel: def __init__(self, interval_ssh="9000-9999", locker=Lock()): - self.interval_ssh = [int(interval_ssh.split('-')[0]), int(interval_ssh.split('-')[1])] + self.interval_ssh = [int(interval_ssh.split('-')[0]), + int(interval_ssh.split('-')[1])] self.busy_port = [] self.locker = locker @@ -359,39 +372,50 @@ def free_port(self, port): if port in self.busy_port: self.busy_port.remove(port) - def __call__(self, address_dest_compute, address_dest_controller, host, **kwargs): - return up_ssh_tunnel_class(address_dest_compute, - address_dest_controller, - host, - self.get_free_port, - self.free_port) + def __call__(self, address_dest_compute, address_dest_controller, host, + **kwargs): + return UpSshTunnelClass(address_dest_compute, + address_dest_controller, + host, + self.get_free_port, + self.free_port) -class up_ssh_tunnel_class: +class UpSshTunnelClass: """ Up ssh tunnel on dest controller node for transferring data """ - def __init__(self, address_dest_compute, address_dest_controller, host, callback_get, callback_free): + def __init__(self, address_dest_compute, address_dest_controller, host, + callback_get, callback_free): self.address_dest_compute = address_dest_compute self.address_dest_controller = address_dest_controller self.get_free_port = callback_get self.remove_port = callback_free self.host = host - self.cmd = "ssh -oStrictHostKeyChecking=no -L %s:%s:22 -R %s:localhost:%s %s -Nf" + self.cmd = SSH_CMD def __enter__(self): self.port = self.get_free_port() - with settings(host_string=self.host): - run(self.cmd % (self.port, self.address_dest_compute, self.port, self.port, + with settings(host_string=self.host, + connection_attempts=env.connection_attempts): + run(self.cmd % (self.port, + self.address_dest_compute, + self.port, + self.port, self.address_dest_controller) + " && sleep 2") return self.port def __exit__(self, type, value, traceback): - with settings(host_string=self.host): - run(("pkill -f '"+self.cmd+"'") % (self.port, self.address_dest_compute, self.port, self.port, - self.address_dest_controller)) + with settings(host_string=self.host, + connection_attempts=env.connection_attempts): + run(("pkill -f '" + self.cmd + "'") % + (self.port, + self.address_dest_compute, + self.port, + self.port, + self.address_dest_controller)) time.sleep(2) self.remove_port(self.port) @@ -402,17 +426,19 @@ def __init__(self, checksum_source, checksum_dest): self.checksum_dest = checksum_dest def __str__(self): - return repr("Checksum of image source = %s Checksum of image dest = %s" % + return repr("Checksum of image source = %s" + + "Checksum of image dest = %s" % (self.checksum_source, self.checksum_dest)) -def render_info(info_values, template_path="templates", template_file="info.html"): +def render_info(info_values, template_path="templates", + template_file="info.html"): info_env = Environment(loader=FileSystemLoader(template_path)) template = info_env.get_template(template_file) return template.render(info_values) -def write_info(rendered_info, info_file = "source_info.html"): +def write_info(rendered_info, info_file="source_info.html"): with open(info_file, "wb") as ifile: ifile.write(rendered_info) @@ -422,7 +448,8 @@ def get_libvirt_block_info(libvirt_name, init_host, compute_host, ssh_user, with settings(host_string=compute_host, user=ssh_user, password=ssh_sudo_password, - gateway=init_host): + gateway=init_host, + connection_attempts=env.connection_attempts): out = sudo("virsh domblklist %s" % libvirt_name) libvirt_output = out.split() return libvirt_output @@ -435,7 +462,8 @@ def find_element_by_in(list_values, word): def init_singletones(cfg): - globals()['up_ssh_tunnel'] = wrapper_singletone_ssh_tunnel(cfg.migrate.ssh_transfer_port) + globals()['up_ssh_tunnel'] = wrapper_singletone_ssh_tunnel( + cfg.migrate.ssh_transfer_port) def get_disk_path(instance, blk_list, is_ceph_ephemeral=False, disk=DISK): @@ -458,14 +486,15 @@ def get_disk_path(instance, blk_list, is_ceph_ephemeral=False, disk=DISK): def get_ips(init_host, compute_host, ssh_user): with settings(host_string=compute_host, user=ssh_user, - gateway=init_host): + gateway=init_host, + connection_attempts=env.connection_attempts): cmd = ("ifconfig | awk -F \"[: ]+\" \'/inet addr:/ " "{ if ($4 != \"127.0.0.1\") print $4 }\'") out = run(cmd) list_ips = [] for info in out.split(): try: - ip = ipaddr.IPAddress(info) + ipaddr.IPAddress(info) except ValueError: continue list_ips.append(info) diff --git a/condensation/process.py b/condensation/process.py index e0c0dfaf..f2e19a54 100644 --- a/condensation/process.py +++ b/condensation/process.py @@ -29,12 +29,13 @@ def process(nodes, flavors, vms, groups): # read files with nova data and node data LOG.info("started creating schedule for node condensation") - source = cloud.Cloud.from_dicts('source', nodes, flavors, vms, groups) + source = cloud.Cloud.from_dicts('source', nodes, flavors, vms, groups) source.migrate_to(cloud.Cloud('destination')) if __name__ == "__main__": process(nodes=condense_utils.read_file(cfglib.CONF.condense.nodes_file), - flavors=condense_utils.read_file(cfglib.CONF.condense.flavors_file), + flavors=condense_utils.read_file( + cfglib.CONF.condense.flavors_file), vms=condense_utils.read_file(cfglib.CONF.condense.vms_file), groups=condense_utils.read_file(cfglib.CONF.condense.groups_file)) diff --git a/configs/config.ini b/configs/config.ini index 6fe4726a..b79b41c2 100644 --- a/configs/config.ini +++ b/configs/config.ini @@ -163,6 +163,10 @@ mysqldump_host = # default optimize_user_role_fetch = [True|False] +# Number of times CloudFerry will attempt to connect when connecting to a new +# server via SSH. +ssh_connection_attempts = 3 + #============================================================================== # Mailing configuration diff --git a/data_storage.py b/data_storage.py index d54839b8..bcadfd7b 100644 --- a/data_storage.py +++ b/data_storage.py @@ -83,6 +83,7 @@ def get(key, connection): def delete(key, connection): return connection.delete(key) + @redis_socket_to_kwargs def delete_batch(keys, connection): pipe = connection.pipeline() @@ -90,6 +91,7 @@ def delete_batch(keys, connection): pipe.delete(key) pipe.execute() + @redis_socket_to_kwargs def keys(pattern, connection): return connection.keys(pattern) diff --git a/devlab/config.template b/devlab/config.template index 3880fb2c..d966bdf4 100755 --- a/devlab/config.template +++ b/devlab/config.template @@ -29,6 +29,7 @@ cinder_migration_strategy = cloudferrylib.os.storage.cinder_database.CinderStora skip_down_hosts = True migrate_user_quotas = False incloud_live_migration = nova +ssh_connection_attempts = 3 [src] type = os diff --git a/devlab/tests/config.py b/devlab/tests/config.py index 022790ea..1a768f65 100644 --- a/devlab/tests/config.py +++ b/devlab/tests/config.py @@ -1,5 +1,8 @@ img_url = 'http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img' +# Path to CloudFerry config relative to the root folder +cloud_ferry_conf = 'configuration.ini' + # Users to create/delete users = [ {'name': 'user1', 'password': 'passwd1', 'email': 'mail@example.com', @@ -95,7 +98,10 @@ {'name': 'image4', 'copy_from': img_url, 'container_format': 'bare', 'disk_format': 'qcow2', 'is_public': False}, {'name': 'image5', 'copy_from': img_url, 'container_format': 'bare', - 'disk_format': 'qcow2', 'is_public': False} + 'disk_format': 'qcow2', 'is_public': False}, + # When location field is specified, glance creates images without checksum + {'name': 'without_checksum', 'location': img_url, 'disk_format': 'qcow2', + 'container_format': 'bare'} ] # Images not to be migrated: diff --git a/devlab/tests/data_collector.py b/devlab/tests/data_collector.py index 109de0ac..be8dc749 100644 --- a/devlab/tests/data_collector.py +++ b/devlab/tests/data_collector.py @@ -133,7 +133,7 @@ def nova_collector(self, destination, *args): 'novaclient', arg, 'list') return collected_items - def neutron_collector(self, destination, *args): + def neutron_collector(self, destination, *args): """ Neutron data collector method. """ diff --git a/devlab/tests/functional_test.py b/devlab/tests/functional_test.py index e3594566..73ca5e23 100644 --- a/devlab/tests/functional_test.py +++ b/devlab/tests/functional_test.py @@ -14,11 +14,21 @@ import config import logging +import sys +import os import unittest from generate_load import Prerequisites from filtering_utils import FilteringUtils +def get_cf_root_folder(): + return os.path.dirname(os.path.dirname(os.path.split(__file__)[0])) + +sys.path.append(get_cf_root_folder()) +import cfglib +cfglib.init_config(os.path.join(get_cf_root_folder(), config.cloud_ferry_conf)) + + def suppress_dependency_logging(): suppressed_logs = ['iso8601.iso8601', 'keystoneclient.session', @@ -100,9 +110,10 @@ def filter_roles(self): def filter_vms(self): vms = config.vms [vms.extend(i['vms']) for i in config.tenants if 'vms' in i] + vms_names = [vm['name'] for vm in vms] opts = {'search_opts': {'all_tenants': 1}} return [i for i in self.src_cloud.novaclient.servers.list(**opts) - if i.name in vms] + if i.name in vms_names] def filter_flavors(self): flavors = [i['name'] for i in config.flavors] diff --git a/devlab/tests/generate_load.py b/devlab/tests/generate_load.py index 6755b7da..1dd712cb 100644 --- a/devlab/tests/generate_load.py +++ b/devlab/tests/generate_load.py @@ -130,6 +130,12 @@ def get_user_tenant_roles(self, user): tenant=self.get_tenant_id(tenant.name))) return user_tenant_roles + def get_ext_routers(self): + routers = self.neutronclient.list_routers()['routers'] + ext_routers = [router for router in routers + if router['external_gateway_info']] + return ext_routers + def check_vm_state(self, srv): srv = self.novaclient.servers.get(srv) return srv.status == 'ACTIVE' @@ -824,14 +830,6 @@ def wait_until_vms_all_deleted(): except Exception as e: print "Tenant %s failed to delete: %s" % (tenant['name'], repr(e)) - sgs = self.neutronclient.list_security_groups()['security_groups'] - for sg in sgs: - try: - self.neutronclient.delete_security_group(sg['id']) - except Exception as e: - print "Security group %s failed to delete: %s" % (sg['name'], - repr(e)) - volumes = self.config.cinder_volumes volumes += itertools.chain(*[tenant['cinder_volumes'] for tenant in self.config.tenants if 'cinder_volumes' diff --git a/devlab/tests/generate_load_real_env.py b/devlab/tests/generate_load_real_env.py index 05a4226d..6d2e70da 100644 --- a/devlab/tests/generate_load_real_env.py +++ b/devlab/tests/generate_load_real_env.py @@ -233,7 +233,7 @@ def clean_objects(self): parser = argparse.ArgumentParser( description='Script to generate load for Openstack and delete ' 'generated objects') - parser.add_argument('--clean', action='store_true', + parser.add_argument('--clean', action='store_true', help='clean objects described in real_env_conf.ini') args = parser.parse_args() if args.clean: diff --git a/devlab/tests/test_resource_migration.py b/devlab/tests/test_resource_migration.py index 0775c616..8e5040eb 100644 --- a/devlab/tests/test_resource_migration.py +++ b/devlab/tests/test_resource_migration.py @@ -12,6 +12,9 @@ class ResourceMigrationTests(functional_test.FunctionalTest): def validate_resource_parameter_in_dst(self, src_list, dst_list, resource_name, parameter): + if not src_list: + self.skipTest( + 'Nothing to migrate - source resources list is empty') name_attr = 'name' if resource_name == 'volume': name_attr = 'display_name' @@ -35,6 +38,9 @@ def validate_resource_parameter_in_dst(self, src_list, dst_list, def validate_neutron_resource_parameter_in_dst(self, src_list, dst_list, resource_name='networks', parameter='name'): + if not src_list[resource_name]: + self.skipTest( + 'Nothing to migrate - source resources list is empty') for i in src_list[resource_name]: for j in dst_list[resource_name]: if i['name'] != j['name']: @@ -396,3 +402,25 @@ def get_fips(client): self.fail("{num} floating IPs did not migrate to destination: " "{fips}".format(num=len(missing_fips), fips=pprint.pformat(missing_fips))) + + @unittest.skipUnless(functional_test.cfglib.CONF.migrate.change_router_ips, + 'Change router ips disabled in CloudFerry config') + def test_ext_router_ip_changed(self): + dst_routers = self.dst_cloud.get_ext_routers() + src_routers = self.src_cloud.get_ext_routers() + for dst_router in dst_routers: + for src_router in src_routers: + if dst_router['name'] != src_router['name']: + continue + src_gateway = self.src_cloud.neutronclient.list_ports( + device_id=src_router['id'], + device_owner='network:router_gateway')['ports'][0] + dst_gateway = self.dst_cloud.neutronclient.list_ports( + device_id=dst_router['id'], + device_owner='network:router_gateway')['ports'][0] + self.assertNotEqual( + src_gateway['fixed_ips'][0]['ip_address'], + dst_gateway['fixed_ips'][0]['ip_address'], + 'GW ip addresses of router "{0}" are same on src and dst:' + ' {1}'.format(dst_router['name'], + dst_gateway['fixed_ips'][0]['ip_address'])) diff --git a/devlab/tests/test_vm_migration.py b/devlab/tests/test_vm_migration.py index 3131792b..26567c5a 100644 --- a/devlab/tests/test_vm_migration.py +++ b/devlab/tests/test_vm_migration.py @@ -7,16 +7,15 @@ class VmMigration(functional_test.FunctionalTest): def setUp(self): - src_vms = [x.__dict__ for x in self.filter_vms()] - self.dst_vms = [x.__dict__ for x in - self.dst_cloud.novaclient.servers.list( - search_opts={'all_tenants': 1})] - src_vms = [vm for vm in src_vms if vm['status'] != 'ERROR'] + src_vms = self.filter_vms() + self.dst_vms = self.dst_cloud.novaclient.servers.list( + search_opts={'all_tenants': 1}) + src_vms = [vm for vm in src_vms if vm.status != 'ERROR'] self.dst_vm_indexes = [] for vm in src_vms: - if vm['name'] not in config.vms_not_in_filter: + if vm.name not in config.vms_not_in_filter: self.dst_vm_indexes.append( - [x['name'] for x in self.dst_vms].index(vm['name'])) + [x.name for x in self.dst_vms].index(vm.name)) with open('pre_migration_vm_states.json') as data_file: self.before_migr_states = json.load(data_file) self.filter_vms = self.filtering_utils.filter_vms(src_vms) @@ -26,16 +25,15 @@ def test_vms_not_in_filter_stay_active_on_src(self): vms_filtered_out = filter_results[1] original_states = self.before_migr_states for vm in vms_filtered_out: - self.assertTrue(vm['status'] == original_states[vm['name']]) + self.assertTrue(vm.status == original_states[vm.name]) def test_vm_not_in_filter_did_not_migrate(self): filter_results = self.filter_vms vms_filtered_out = filter_results[1] - dst_vms = [x.__dict__['name'] for x in - self.dst_cloud.novaclient.servers.list( - search_opts={'all_tenants': 1})] + dst_vms = [x.name for x in self.dst_cloud.novaclient.servers.list( + search_opts={'all_tenants': 1})] for vm in vms_filtered_out: - self.assertTrue(vm['name'] not in dst_vms, + self.assertTrue(vm.name not in dst_vms, 'VM migrated despite that it was not included in ' 'filter, VM info: \n{}'.format(vm)) @@ -46,28 +44,28 @@ def test_cold_migrate_vm_state(self): original_states.pop(vm_name) src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): - if src_vm['name'] in original_states.keys(): - if original_states[src_vm['name']] == 'ACTIVE' or \ - original_states[src_vm['name']] == 'VERIFY_RESIZE': + if src_vm.name in original_states.keys(): + if original_states[src_vm.name] == 'ACTIVE' or \ + original_states[src_vm.name] == 'VERIFY_RESIZE': self.assertTrue( - src_vm['status'] == 'SHUTOFF' and - self.dst_vms[vm_index]['status'] == 'ACTIVE') + src_vm.status == 'SHUTOFF' and + self.dst_vms[vm_index].status == 'ACTIVE') else: self.assertTrue( - src_vm['status'] == 'SHUTOFF' and - self.dst_vms[vm_index]['status'] == 'SHUTOFF') + src_vm.status == 'SHUTOFF' and + self.dst_vms[vm_index].status == 'SHUTOFF') else: - self.assertTrue(src_vm['status'] == 'SHUTOFF' and - self.dst_vms[vm_index]['status'] == 'ACTIVE') + self.assertTrue(src_vm.status == 'SHUTOFF' and + self.dst_vms[vm_index].status == 'ACTIVE') def test_cold_migrate_vm_ip(self): src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): - for src_net in src_vm['addresses']: - for src_net_addr, dst_net_addr in zip(src_vm['addresses'] + for src_net in src_vm.addresses: + for src_net_addr, dst_net_addr in zip(src_vm.addresses [src_net], self.dst_vms[vm_index] - ['addresses'][src_net]): + .addresses[src_net]): self.assertTrue(src_net_addr['addr'] == dst_net_addr['addr']) @@ -75,13 +73,13 @@ def test_cold_migrate_vm_security_groups(self): src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): dst_sec_group_names = [x['name'] for x in - self.dst_vms[vm_index]['security_groups']] - for security_group in src_vm['security_groups']: + self.dst_vms[vm_index].security_groups] + for security_group in src_vm.security_groups: self.assertTrue(security_group['name'] in dst_sec_group_names) @unittest.skip("Temporarily disabled: image's id changes after migrating") def test_cold_migrate_vm_image_id(self): src_vms = self.filter_vms[0] for src_vm, vm_index in zip(src_vms, self.dst_vm_indexes): - self.assertTrue(src_vm['image']['id'] == - self.dst_vms[vm_index]['image']['id']) + self.assertTrue(src_vm.image.id == + self.dst_vms[vm_index].image.id) diff --git a/fabfile.py b/fabfile.py index 37461ad4..fae438d0 100644 --- a/fabfile.py +++ b/fabfile.py @@ -53,6 +53,7 @@ def migrate(name_config=None, name_instance=None, debug=False): cfglib.init_config(name_config) utils.init_singletones(cfglib.CONF) env.key_filename = cfglib.CONF.migrate.key_filename + env.connection_attempts = cfglib.CONF.migrate.ssh_connection_attempts cloud = cloud_ferry.CloudFerry(cfglib.CONF) cloud.migrate(Scenario(path_scenario=cfglib.CONF.migrate.scenario, path_tasks=cfglib.CONF.migrate.tasks_mapping)) diff --git a/tests/cloudferrylib/os/actions/test_check_instance_networks.py b/tests/cloudferrylib/os/actions/test_check_instance_networks.py index c2ddefd8..0c44237a 100644 --- a/tests/cloudferrylib/os/actions/test_check_instance_networks.py +++ b/tests/cloudferrylib/os/actions/test_check_instance_networks.py @@ -52,7 +52,7 @@ def test_net_empty(self): action.run() def test_instance_empty(self): - action = self.get_action(FakeSubnets().add('1.1.1.1/24',True).toMap(), + action = self.get_action(FakeSubnets().add('1.1.1.1/24', True).toMap(), []) action.run() @@ -70,7 +70,7 @@ def test_negative(self): 'name-10.0.0.1', action.run) - def test_negative(self): + def test_negative_ext(self): action = self.get_action(FakeSubnets().add('10.0.0.0/24', True) .add('100.0.0.0/24', True).toMap(), ['10.0.0.1', '10.0.0.2', '10.0.0.4']) diff --git a/tests/cloudferrylib/os/actions/test_check_networks.py b/tests/cloudferrylib/os/actions/test_check_networks.py index 684f11d0..aa3c777d 100644 --- a/tests/cloudferrylib/os/actions/test_check_networks.py +++ b/tests/cloudferrylib/os/actions/test_check_networks.py @@ -13,18 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. + import mock -import exceptions -from tests import test +from cloudferrylib.base import exception from cloudferrylib.os.actions import check_networks +from tests import test class CheckNetworksTestCase(test.TestCase): - - def setUp(self): - super(CheckNetworksTestCase, self).setUp() - @staticmethod def get_action(src_net_info, dst_net_info): fake_src_net = mock.Mock() @@ -52,7 +49,10 @@ def test_all_empty(self): def test_empty_dst(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', @@ -63,13 +63,19 @@ def test_empty_dst(self): def test_equals_networks(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', 'id': 'sub1'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id2', @@ -78,7 +84,10 @@ def test_equals_networks(self): def test_equals_and_new_networks(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', @@ -88,7 +97,10 @@ def test_equals_and_new_networks(self): 'network_id': 'id1', 'id': 'sub2'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id2', @@ -97,32 +109,92 @@ def test_equals_and_new_networks(self): def test_diff(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 2, 'network_id': 'id1', 'id': 'sub1'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 3, 'network_id': 'id2', 'id': 'sub2'}]} action = self.get_action(src_net_info, dst_net_info) - self.assertRaises(exceptions.EnvironmentError, action.run) + self.assertRaises(exception.AbortMigrationError, action.run) def test_overlap(self): src_net_info = {'networks': [{'id': 'id1', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/28', 'res_hash': 2, 'network_id': 'id1', 'id': 'sub1'}]} dst_net_info = {'networks': [{'id': 'id2', - 'res_hash': 1}], + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'local', + 'provider:segmentation_id': None}], 'subnets': [{'cidr': '10.0.0.0/24', 'res_hash': 3, 'network_id': 'id2', 'id': 'sub2'}]} action = self.get_action(src_net_info, dst_net_info) - self.assertRaises(exceptions.EnvironmentError, action.run) + self.assertRaises(exception.AbortMigrationError, action.run) + + def test_check_segmentation_id_overlapping_no_dst_networks(self): + src_net_info = {'networks': [{'id': 'id1', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + dst_net_info = {'networks': [], + 'subnets': []} + + self.get_action(src_net_info, dst_net_info).run() + + def test_check_segmentation_id_overlapping_same_network(self): + src_net_info = {'networks': [{'id': 'id1', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + dst_net_info = {'networks': [{'id': 'id2', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + self.get_action(src_net_info, dst_net_info).run() + + def test_check_segmentation_id_overlapping_different_network(self): + src_net_info = {'networks': [{'id': 'id1', + 'res_hash': 1, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + dst_net_info = {'networks': [{'id': 'id2', + 'res_hash': 2, + "provider:physical_network": None, + 'provider:network_type': 'gre', + 'provider:segmentation_id': 200}], + 'subnets': []} + + action = self.get_action(src_net_info, dst_net_info) + self.assertRaises(exception.AbortMigrationError, action.run) diff --git a/tests/cloudferrylib/os/compute/test_libvirt.py b/tests/cloudferrylib/os/compute/test_libvirt.py index 0081a608..1804f328 100644 --- a/tests/cloudferrylib/os/compute/test_libvirt.py +++ b/tests/cloudferrylib/os/compute/test_libvirt.py @@ -84,7 +84,7 @@ def test_backing_file_returns_none_in_case_of_error(self): grizzly_xml = """ instance-00000008 - 7f9cfeab-05c6-4dcc-91d4-3074383ff73d + 7f9cfeab-05c6-4dcc 524288 524288 1 @@ -94,7 +94,7 @@ def test_backing_file_returns_none_in_case_of_error(self): OpenStack Nova 2013.1.5 72827a02-9e88-41b9-82de-ec2ae162a93c - 7f9cfeab-05c6-4dcc-91d4-3074383ff73d + 7f9cfeab-05c6-4dcc @@ -117,12 +117,12 @@ def test_backing_file_returns_none_in_case_of_error(self): /usr/bin/qemu-system-x86_64 - + -
+
-
+
@@ -131,7 +131,7 @@ def test_backing_file_returns_none_in_case_of_error(self): -
+
@@ -139,7 +139,7 @@ def test_backing_file_returns_none_in_case_of_error(self): -
+
@@ -147,31 +147,31 @@ def test_backing_file_returns_none_in_case_of_error(self): -
+
- + - + - +