diff --git a/core.py b/core.py index 7d678312..73f8a956 100644 --- a/core.py +++ b/core.py @@ -284,25 +284,25 @@ def analyze_fuction(self, function_type, opt): self.set_context(function_type, 'analyze', config) handler = AnalyzeFltTraceHandler(self.context) handler.handle() - elif function_type == 'analyze_sql': - self.set_context(function_type, 'analyze', config) - handler = AnalyzeSQLHandler(self.context) - handler.handle() - elif function_type == 'analyze_sql_review': - self.set_context(function_type, 'analyze', config) - handler = AnalyzeSQLReviewHandler(self.context) - handler.handle() - elif function_type == 'analyze_parameter_non_default': + elif function_type == 'analyze_parameter_default': self.set_context(function_type, 'analyze', config) - handler = AnalyzeParameterHandler(self.context, 'non_default') + handler = AnalyzeParameterHandler(self.context, 'default') handler.handle() elif function_type == 'analyze_parameter_diff': self.set_context_skip_cluster_conn(function_type, 'analyze', config) handler = AnalyzeParameterHandler(self.context, 'diff') handler.handle() - elif function_type == 'analyze_variable': + elif function_type == 'analyze_variable_diff': self.set_context(function_type, 'analyze', config) - handler = AnalyzeVariableHandler(self.context) + handler = AnalyzeVariableHandler(self.context, 'diff') + handler.handle() + elif function_type == 'analyze_sql': + self.set_context(function_type, 'analyze', config) + handler = AnalyzeSQLHandler(self.context) + handler.handle() + elif function_type == 'analyze_sql_review': + self.set_context(function_type, 'analyze', config) + handler = AnalyzeSQLReviewHandler(self.context) handler.handle() else: self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type)) diff --git a/diag_cmd.py b/diag_cmd.py index 06d83d65..cc54650a 100644 --- a/diag_cmd.py +++ b/diag_cmd.py @@ -697,20 +697,49 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) -class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): +class ObdiagAnalyzeParameterDefaultCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values') + super(ObdiagAnalyzeParameterDefaultCommand, self).__init__('default', 'Analyze the parameter to identify parameters with non-default values') self.parser.add_option('--file', type='string', help="specify initialization parameter file") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) def init(self, cmd, args): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) + super(ObdiagAnalyzeParameterDefaultCommand, self).init(cmd, args) self.parser.set_usage('%s [options]' % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + return obdiag.analyze_fuction('analyze_parameter_default', self.opts) + + +class ObdiagAnalyzeParameterCommand(MajorCommand): + def __init__(self): + super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') + self.register_command(ObdiagAnalyzeParameterDiffCommand()) + self.register_command(ObdiagAnalyzeParameterDefaultCommand()) + + +class ObdiagAnalyzeVariableDiffCommand(ObdiagOriginCommand): + def __init__(self): + super(ObdiagAnalyzeVariableDiffCommand, self).__init__('diff', 'Analyze and identify variables that have changed compared to the specified variable file') + self.parser.add_option('--file', type='string', help="specify initialization parameter file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeVariableDiffCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_variable_diff', self.opts) + + +class ObdiagAnalyzeVariableCommand(MajorCommand): + def __init__(self): + super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze oceanbase variables info') + self.register_command(ObdiagAnalyzeVariableDiffCommand()) class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand): @@ -764,29 +793,6 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_sql_review', self.opts) -class ObdiagAnalyzeParameterCommand(MajorCommand): - def __init__(self): - super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') - self.register_command(ObdiagAnalyzeParameterDiffCommand()) - self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) - - -class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): - def __init__(self): - super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_variable', self.opts) - - class ObdiagCheckCommand(ObdiagOriginCommand): def __init__(self): @@ -895,6 +901,8 @@ def __init__(self): self.register_command(ObdiagGatherObproxyLogCommand()) self.register_command(ObdiagGatherSceneCommand()) self.register_command(ObdiagGatherAshReportCommand()) + self.register_command(ObdiagGatherParameterCommand()) + self.register_command(ObdiagGatherVariableCommand()) self.register_command(ObdiagGatherTableDumpHandler()) self.register_command(ObdiagGatherParameterCommand()) self.register_command(ObdiagGatherVariableCommand()) @@ -914,6 +922,8 @@ def __init__(self): super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info') self.register_command(ObdiagAnalyzeLogCommand()) self.register_command(ObdiagAnalyzeFltTraceCommand()) + self.register_command(ObdiagAnalyzeParameterCommand()) + self.register_command(ObdiagAnalyzeVariableCommand()) self.register_command(ObdiagAnalyzeSQLCommand()) self.register_command(ObdiagAnalyzeSQLReviewCommand()) self.register_command(ObdiagAnalyzeParameterCommand()) diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 47b51480..a31ebd38 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -28,7 +28,7 @@ class AnalyzeParameterHandler(object): - def __init__(self, context, analyze_type='non_default'): + def __init__(self, context, analyze_type='default'): self.context = context self.stdio = self.context.stdio self.export_report_path = None @@ -64,8 +64,8 @@ def get_version(self): return observer_version def handle(self): - if self.analyze_type == 'non_default': - if not self.init_option_non_default(): + if self.analyze_type == 'default': + if not self.init_option_default(): self.stdio.error('init option failed') return False else: @@ -76,7 +76,23 @@ def handle(self): DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) self.execute() - def init_option_non_default(self): + def check_file_valid(self): + with open(self.parameter_file_name, 'r') as f: + header = f.readline() + flag = 1 + if header: + header = header.strip() + if not header: + flag = 0 + if not header.startswith('VERSION'): + flag = 0 + if not header.endswith('ISDEFAULT'): + flag = 0 + if flag == 0: + self.stdio.error('args --file [{0}] is not a valid parameter file, Please specify it again'.format(os.path.abspath(self.parameter_file_name))) + exit(-1) + + def init_option_default(self): options = self.context.options store_dir_option = Util.get_option(options, 'store_dir') offline_file_option = Util.get_option(options, 'file') @@ -97,6 +113,7 @@ def init_option_non_default(self): exit(-1) else: self.parameter_file_name = os.path.abspath(offline_file_option) + self.check_file_valid() return True def init_option_diff(self): @@ -121,9 +138,10 @@ def init_option_diff(self): exit(-1) else: self.parameter_file_name = os.path.abspath(offline_file_option) + self.check_file_valid() return True - def analyze_parameter_non_default(self): + def analyze_parameter_default(self): observer_version = self.get_version() if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"): if self.parameter_file_name is not None: @@ -144,7 +162,7 @@ def analyze_parameter_non_default(self): report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]]) fp.write(report_default_tb.get_string() + "\n") self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") else: if self.parameter_file_name is None: self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values") @@ -179,9 +197,9 @@ def analyze_parameter_non_default(self): fp.write(report_default_tb.get_string() + "\n") if not is_empty: self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") else: - self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.") + self.stdio.print("Analyze parameter default finished. All parameter values are the same as the default values.") def alalyze_parameter_diff(self): if self.parameter_file_name is None: @@ -240,7 +258,10 @@ def alalyze_parameter_diff(self): if len(value_list) > 0: report_diff_tb = PrettyTable(["name", "diff"]) report_diff_tb.align["task_report"] = "l" - report_diff_tb.title = 'TENANT_ID:' + tenant + if tenant == 'CLUSTER': + report_diff_tb.title = 'SCOPE:' + tenant + else: + report_diff_tb.title = 'SCOPE:TENANT-' + tenant for value_dict in value_list: value_str_list = [] for value in value_dict['value_list']: @@ -258,8 +279,8 @@ def alalyze_parameter_diff(self): def execute(self): try: - if self.analyze_type == 'non_default': - self.analyze_parameter_non_default() + if self.analyze_type == 'default': + self.analyze_parameter_default() elif self.analyze_type == 'diff': self.alalyze_parameter_diff() except Exception as e: diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index 478c3c3d..4058c868 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -26,11 +26,12 @@ class AnalyzeVariableHandler(object): - def __init__(self, context): + def __init__(self, context, analyze_type='diff'): self.context = context self.stdio = self.context.stdio self.export_report_path = None self.variable_file_name = None + self.analyze_type = analyze_type self.ob_cluster = self.context.cluster_config if self.context.get_variable("gather_timestamp", None): self.analyze_timestamp = self.context.get_variable("gather_timestamp") @@ -48,8 +49,8 @@ def __init__(self, context): database="oceanbase", ) except Exception as e: - self.stdio.error("failed to connect to database: {0}".format(e)) - raise OBDIAGFormatException("failed to connect to database: {0}".format(e)) + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) def handle(self): if not self.init_option(): @@ -59,6 +60,22 @@ def handle(self): DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) self.execute() + def check_file_valid(self): + with open(self.variable_file_name, 'r') as f: + header = f.readline() + flag = 1 + if header: + header = header.strip() + if not header: + flag = 0 + if not header.startswith('VERSION'): + flag = 0 + if not header.endswith('RECORD_TIME'): + flag = 0 + if flag == 0: + self.stdio.error('args --file [{0}] is not a valid variable file, Please specify it again'.format(os.path.abspath(self.variable_file_name))) + exit(-1) + def init_option(self): options = self.context.options store_dir_option = Util.get_option(options, 'store_dir') @@ -69,6 +86,7 @@ def init_option(self): exit(-1) else: self.variable_file_name = os.path.abspath(offline_file_option) + self.check_file_valid() else: self.stdio.error("an initialization variable file must be provided to find the parts where variables have changed.") exit(-1) @@ -87,7 +105,7 @@ def init_option(self): return True - def alalyze_variable(self): + def analyze_variable(self): sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' db_variable_info = self.obconn.execute_sql(sql) @@ -131,6 +149,6 @@ def alalyze_variable(self): def execute(self): try: - self.alalyze_variable() + self.analyze_variable() except Exception as e: self.stdio.error("variable info analyze failed, error message: {0}".format(e)) diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index ea553faf..187fb779 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -47,8 +47,8 @@ def __init__(self, context, gather_pack_dir='./'): database="oceanbase", ) except Exception as e: - self.stdio.error("failed to connect to database: {0}".format(e)) - raise OBDIAGFormatException("failed to connect to database: {0}".format(e)) + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) def handle(self): if not self.init_option(): @@ -110,8 +110,10 @@ def get_parameters_info(self): ''' parameter_info = self.obconn.execute_sql(sql) self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT'] with open(self.parameter_file_name, 'w', newline='') as file: writer = csv.writer(file) + writer.writerow(header) for row in parameter_info: if row[5] is None: tmp_row = [col for col in row] @@ -121,7 +123,7 @@ def get_parameters_info(self): writer.writerow(row) self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'") else: - self.stdio.warn("failed to retrieve the database version. Please check if the database connection is normal.") + self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.") def execute(self): try: diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index 34729a3b..6c49b538 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -46,8 +46,8 @@ def __init__(self, context, gather_pack_dir='./'): database="oceanbase", ) except Exception as e: - self.stdio.error("failed to connect to database: {0}".format(e)) - raise OBDIAGFormatException("failed to connect to database: {0}".format(e)) + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) def handle(self): if not self.init_option(): @@ -64,7 +64,7 @@ def init_option(self): store_dir_option = Util.get_option(options, 'store_dir') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) return True @@ -86,8 +86,10 @@ def get_variables_info(self): from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' variable_info = self.obconn.execute_sql(sql) self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME'] with open(self.variable_file_name, 'w', newline='') as file: writer = csv.writer(file) + writer.writerow(header) for row in variable_info: writer.writerow(row) self.stdio.print("Gather variables finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.variable_file_name) + Style.RESET_ALL + "'")