From 647c76055974235dc720c9d58d71c1415d8f14ec Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Fri, 5 Jul 2024 18:48:15 +0800 Subject: [PATCH 01/12] obdiag gather and analyze parameters/variables --- core.py | 22 +++ diag_cmd.py | 95 ++++++++- handler/analyzer/analyze_parameter.py | 267 ++++++++++++++++++++++++++ handler/analyzer/analyze_variable.py | 136 +++++++++++++ handler/gather/gather_parameters.py | 130 +++++++++++++ handler/gather/gather_variables.py | 99 ++++++++++ 6 files changed, 747 insertions(+), 2 deletions(-) create mode 100644 handler/analyzer/analyze_parameter.py create mode 100644 handler/analyzer/analyze_variable.py create mode 100644 handler/gather/gather_parameters.py create mode 100644 handler/gather/gather_variables.py diff --git a/core.py b/core.py index 4efdd5d4..c958c47b 100644 --- a/core.py +++ b/core.py @@ -32,6 +32,8 @@ from err import CheckStatus, SUG_SSH_FAILED from handler.analyzer.analyze_flt_trace import AnalyzeFltTraceHandler from handler.analyzer.analyze_log import AnalyzeLogHandler +from handler.analyzer.analyze_parameter import AnalyzeParameterHandler +from handler.analyzer.analyze_variable import AnalyzeVariableHandler from handler.checker.check_handler import CheckHandler from handler.checker.check_list import CheckListHandler from handler.gather.gather_log import GatherLogHandler @@ -44,6 +46,8 @@ from handler.gather.gather_plan_monitor import GatherPlanMonitorHandler from handler.gather.gather_scenes import GatherSceneHandler from handler.gather.scenes.list import GatherScenesListHandler +from handler.gather.gather_parameters import GatherParametersHandler +from handler.gather.gather_variables import GatherVariablesHandler from telemetry.telemetry import telemetry from update.update import UpdateHandler from colorama import Fore, Style @@ -230,6 +234,12 @@ def gather_function(self, function_type, opt): elif function_type == 'gather_ash_report': handler = GatherAshReportHandler(self.context) return handler.handle() + elif function_type == 'gather_parameters': + handler = GatherParametersHandler(self.context) + return handler.handle() + elif function_type == 'gather_variables': + handler = GatherVariablesHandler(self.context) + return handler.handle() else: self._call_stdio('error', 'Not support gather function: {0}'.format(function_type)) return False @@ -268,6 +278,18 @@ def analyze_fuction(self, function_type, opt): self.set_context(function_type, 'analyze', config) handler = AnalyzeFltTraceHandler(self.context) handler.handle() + elif function_type == 'analyze_parameter_non_default': + self.set_context(function_type, 'analyze', config) + handler = AnalyzeParameterHandler(self.context, 'non_default') + handler.handle() + elif function_type == 'analyze_parameter_diff': + self.set_context_skip_cluster_conn(function_type, 'analyze', config) + handler = AnalyzeParameterHandler(self.context, 'diff') + handler.handle() + elif function_type == 'analyze_variable': + self.set_context(function_type, 'analyze', config) + handler = AnalyzeVariableHandler(self.context) + handler.handle() else: self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type)) return False diff --git a/diag_cmd.py b/diag_cmd.py index 33d367c5..62a11918 100644 --- a/diag_cmd.py +++ b/diag_cmd.py @@ -374,6 +374,38 @@ def _do_command(self, obdiag): return obdiag.gather_function('gather_log', self.opts) +class ObdiagGatherParameterCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherParameterCommand, self).__init__('parameter', 'Gather oceanbase parameters from oceanbase database') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherParameterCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_parameters', self.opts) + + +class ObdiagGatherVariableCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherVariableCommand, self).__init__('variable', 'Gather oceanbase variables from oceanbase database') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherVariableCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_variables', self.opts) + + class ObdiagGatherSysStatCommand(ObdiagOriginCommand): def __init__(self): @@ -630,6 +662,63 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_flt_trace', self.opts) + +class ObdiagAnalyzeParameterDiffCommand(ObdiagOriginCommand): + def __init__(self): + super(ObdiagAnalyzeParameterDiffCommand, self).__init__('diff', 'Analyze the parameter configurations between observers and identify the parameters with different values among the observers') + self.parser.add_option('--file', type='string', help="specify initialization parameter file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeParameterDiffCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) + + +class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): + def __init__(self): + super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values') + self.parser.add_option('--file', type='string', help="specify initialization parameter file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + + +class ObdiagAnalyzeParameterCommand(MajorCommand): + def __init__(self): + super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') + self.register_command(ObdiagAnalyzeParameterDiffCommand()) + self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) + + +class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') + self.parser.add_option('--file', type='string', help="specify initialization parameter file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_variable', self.opts) + + class ObdiagCheckCommand(ObdiagOriginCommand): def __init__(self): @@ -738,7 +827,8 @@ def __init__(self): self.register_command(ObdiagGatherObproxyLogCommand()) self.register_command(ObdiagGatherSceneCommand()) self.register_command(ObdiagGatherAshReportCommand()) - + self.register_command(ObdiagGatherParameterCommand()) + self.register_command(ObdiagGatherVariableCommand()) class ObdiagGatherSceneCommand(MajorCommand): @@ -754,7 +844,8 @@ def __init__(self): super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info') self.register_command(ObdiagAnalyzeLogCommand()) self.register_command(ObdiagAnalyzeFltTraceCommand()) - + self.register_command(ObdiagAnalyzeParameterCommand()) + self.register_command(ObdiagAnalyzeVariableCommand()) class ObdiagRCACommand(MajorCommand): diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py new file mode 100644 index 00000000..271362b4 --- /dev/null +++ b/handler/analyzer/analyze_parameter.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/16 +@file: analyze_parameter.py +@desc: +""" +import os +from common.command import get_observer_version_by_sql +from common.tool import DirectoryUtil, TimeUtils, Util, StringUtils +from common.obdiag_exception import OBDIAGFormatException +from common.ob_connector import OBConnector +import csv +from prettytable import PrettyTable +import json +import datetime +from colorama import Fore, Style + + +class AnalyzeParameterHandler(object): + def __init__(self, context, analyze_type='non_default'): + self.context = context + self.stdio = self.context.stdio + self.export_report_path = None + self.parameter_file_name = None + self.ob_cluster = self.context.cluster_config + self.analyze_type = analyze_type + if self.context.get_variable("gather_timestamp", None): + self.analyze_timestamp = self.context.get_variable("gather_timestamp") + else: + self.analyze_timestamp = TimeUtils.get_current_us_timestamp() + self.observer_nodes = self.context.cluster_config.get("servers") + try: + self.obconn = OBConnector( + ip=self.ob_cluster.get("db_host"), + port=self.ob_cluster.get("db_port"), + username=self.ob_cluster.get("tenant_sys").get("user"), + password=self.ob_cluster.get("tenant_sys").get("password"), + stdio=self.stdio, + timeout=10000, + database="oceanbase", + ) + except Exception as e: + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) + + def get_version(self): + observer_version = "" + try: + observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) + except Exception as e: + self.stdio.warn("AnalyzeHandler Failed to get observer version:{0}".format(e)) + self.stdio.verbose("AnalyzeHandler.init get observer version: {0}".format(observer_version)) + return observer_version + + def handle(self): + if self.analyze_type == 'non_default': + if not self.init_option_non_default(): + self.stdio.error('init option failed') + return False + else: + if not self.init_option_diff(): + self.stdio.error('init option failed') + return False + self.stdio.verbose("Use {0} as pack dir.".format(self.export_report_path)) + DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) + self.execute() + + def init_option_non_default(self): + options = self.context.options + store_dir_option = Util.get_option(options, 'store_dir') + offline_file_option = Util.get_option(options, 'file') + if store_dir_option and store_dir_option != "./": + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.export_report_path = os.path.abspath(store_dir_option) + else: + store_dir_option = './parameter_report' + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('The report directory is not specified, and a "parameter_report" directory will be created in the current directory.'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.export_report_path = os.path.abspath(store_dir_option) + if offline_file_option: + if not os.path.exists(os.path.abspath(offline_file_option)): + self.stdio.error('args --file [{0}] not exist: No such file, Please specify it again'.format(os.path.abspath(offline_file_option))) + exit(-1) + else: + self.parameter_file_name = os.path.abspath(offline_file_option) + return True + + def init_option_diff(self): + options = self.context.options + store_dir_option = Util.get_option(options, 'store_dir') + offline_file_option = Util.get_option(options, 'file') + if store_dir_option and store_dir_option != "./": + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.export_report_path = os.path.abspath(store_dir_option) + else: + store_dir_option = './parameter_report' + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('The report directory is not specified, and a "parameter_report" directory will be created in the current directory.'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.export_report_path = os.path.abspath(store_dir_option) + + if offline_file_option: + if not os.path.exists(os.path.abspath(offline_file_option)): + self.stdio.error('args --file [{0}] not exist: No such file, Please specify it again'.format(os.path.abspath(offline_file_option))) + exit(-1) + else: + self.parameter_file_name = os.path.abspath(offline_file_option) + return True + + def analyze_parameter_non_default(self): + observer_version = self.get_version() + if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"): + if self.parameter_file_name is not None: + self.stdio.warn("the version of OceanBase is greater than 4.2.2, an initialization parameter file will be ignored") + sql = '''select substr(version(),8), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, +EDIT_LEVEL, now(),default_value,isdefault from GV$OB_PARAMETERS where isdefault='NO' order by 5,2,3,4,7''' + parameter_info = self.obconn.execute_sql(sql) + report_default_tb = PrettyTable(["IP", "PORT", "ZONE", "CLUSTER", "TENANT_ID", "NAME", "DEFAULT_VALUE", "CURRENT_VALUE"]) + now = datetime.datetime.now() + date_format = now.strftime("%Y-%m-%d-%H-%M-%S") + file_name = self.export_report_path + '/parameter_default_{0}.table'.format(date_format) + fp = open(file_name, 'a+', encoding="utf8") + for row in parameter_info: + if row[5] is None: + tenant_id = 'None' + else: + tenant_id = row[5] + report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]]) + fp.write(report_default_tb.get_string() + "\n") + self.stdio.print(report_default_tb.get_string()) + self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") + else: + if self.parameter_file_name is None: + self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values") + exit(-1) + else: + sql = '''select substr(version(),8), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, +EDIT_LEVEL, now(),'','' from GV$OB_PARAMETERS order by 5,2,3,4,7''' + db_parameter_info = self.obconn.execute_sql(sql) + db_parameter_dict = dict() + for row in db_parameter_info: + key = str(row[1]) + '-' + str(row[2]) + '-' + str(row[3]) + '-' + str(row[4]) + '-' + str(row[5]) + '-' + str(row[6]) + value = row[7] + db_parameter_dict[key] = value + file_parameter_dict = dict() + with open(self.parameter_file_name, 'r', newline='') as file: + reader = csv.reader(file) + for row in reader: + key = str(row[1]) + '-' + str(row[2]) + '-' + str(row[3]) + '-' + str(row[4]) + '-' + str(row[5]) + '-' + str(row[6]) + value = row[7] + file_parameter_dict[key] = value + report_default_tb = PrettyTable(["IP", "PORT", "ZONE", "CLUSTER", "TENANT_ID", "NAME", "DEFAULT_VALUE", "CURRENT_VALUE"]) + now = datetime.datetime.now() + date_format = now.strftime("%Y-%m-%d-%H-%M-%S") + file_name = self.export_report_path + '/parameter_default_{0}.table'.format(date_format) + fp = open(file_name, 'a+', encoding="utf8") + is_empty = True + for key in db_parameter_dict: + if key in file_parameter_dict and db_parameter_dict[key] != file_parameter_dict[key]: + col_list = key.split('-') + report_default_tb.add_row([col_list[0], col_list[0], col_list[2], col_list[3], col_list[4], col_list[5], file_parameter_dict[key], db_parameter_dict[key]]) + is_empty = False + fp.write(report_default_tb.get_string() + "\n") + if not is_empty: + self.stdio.print(report_default_tb.get_string()) + self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + else: + self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.") + + def alalyze_parameter_diff(self): + if self.parameter_file_name is None: + sql = '''select substr(version(),8), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, +EDIT_LEVEL, now(),'','' from GV$OB_PARAMETERS order by 5,2,3,4,7''' + parameter_info = self.obconn.execute_sql(sql) + else: + parameter_info = [] + with open(self.parameter_file_name, 'r', newline='') as file: + reader = csv.reader(file) + for row in reader: + parameter_info.append(row) + tenants_dict = dict() + for row in parameter_info: + if row[5] is None: + scope = 'CLUSTER' + else: + scope = row[5] + tenant_id = str(scope) + observer = str(row[1]) + ':' + str(row[2]) + name = row[6] + value = row[7] + if tenant_id not in tenants_dict: + tenants_dict[tenant_id] = [] + tenants_dict[tenant_id].append({'observer': observer, 'name': name, 'value': value}) + else: + tenants_dict[tenant_id].append({'observer': observer, 'name': name, 'value': value}) + diff_parameter_dict = dict() + for tenant, parameters_list in tenants_dict.items(): + diff_parameter_dict[tenant] = [] + parameter_dict = dict() + for parameter_info in parameters_list: + name = parameter_info['name'] + observer = parameter_info['observer'] + value = parameter_info['value'] + if name not in parameter_dict: + parameter_dict[name] = [] + parameter_dict[name].append({'observer': observer, 'value': value}) + else: + parameter_dict[name].append({'observer': observer, 'value': value}) + + for name, value_list in parameter_dict.items(): + if name in ['local_ip', 'observer_id', 'zone']: + continue + value_set = set() + for value_info in value_list: + value_set.add(value_info['value']) + if len(value_set) > 1: + diff_parameter_dict[tenant].append({'name': name, 'value_list': value_list}) + now = datetime.datetime.now() + date_format = now.strftime("%Y-%m-%d-%H-%M-%S") + file_name = self.export_report_path + '/parameter_diff_{0}.table'.format(date_format) + fp = open(file_name, 'a+', encoding="utf8") + is_empty = True + for tenant, value_list in diff_parameter_dict.items(): + if len(value_list) > 0: + report_diff_tb = PrettyTable(["name", "diff"]) + report_diff_tb.align["task_report"] = "l" + report_diff_tb.title = 'TENANT_ID:' + tenant + for value_dict in value_list: + value_str_list = [] + for value in value_dict['value_list']: + value_str = json.dumps(value) + value_str_list.append(value_str) + report_diff_tb.add_row([value_dict['name'], '\n'.join(value_str_list)]) + fp.write(report_diff_tb.get_string() + "\n") + self.stdio.print(report_diff_tb.get_string()) + is_empty = False + fp.close() + if not is_empty: + self.stdio.print("Analyze parameter diff finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + else: + self.stdio.print("Analyze parameter diff finished. All parameter settings are consistent among observers") + + + def execute(self): + try: + if self.analyze_type == 'non_default': + self.analyze_parameter_non_default() + elif self.analyze_type == 'diff': + self.alalyze_parameter_diff() + except Exception as e: + self.stdio.error("parameter info analyze failed, error message: {0}".format(e)) diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py new file mode 100644 index 00000000..1edb8149 --- /dev/null +++ b/handler/analyzer/analyze_variable.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/16 +@file: analyze_variable.py +@desc: +""" +import os +from common.tool import DirectoryUtil, TimeUtils, Util +from common.obdiag_exception import OBDIAGFormatException +from common.ob_connector import OBConnector +import csv +from prettytable import PrettyTable +import datetime +from colorama import Fore, Style + + +class AnalyzeVariableHandler(object): + def __init__(self, context): + self.context = context + self.stdio = self.context.stdio + self.export_report_path = None + self.variable_file_name = None + self.ob_cluster = self.context.cluster_config + if self.context.get_variable("gather_timestamp", None): + self.analyze_timestamp = self.context.get_variable("gather_timestamp") + else: + self.analyze_timestamp = TimeUtils.get_current_us_timestamp() + self.observer_nodes = self.context.cluster_config.get("servers") + try: + self.obconn = OBConnector( + ip=self.ob_cluster.get("db_host"), + port=self.ob_cluster.get("db_port"), + username=self.ob_cluster.get("tenant_sys").get("user"), + password=self.ob_cluster.get("tenant_sys").get("password"), + stdio=self.stdio, + timeout=10000, + database="oceanbase", + ) + except Exception as e: + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) + + def handle(self): + if not self.init_option(): + self.stdio.error('init option failed') + return False + self.stdio.verbose("Use {0} as pack dir.".format(self.export_report_path)) + DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) + self.execute() + + def init_option(self): + options = self.context.options + store_dir_option = Util.get_option(options, 'store_dir') + offline_file_option = Util.get_option(options, 'file') + if offline_file_option: + if not os.path.exists(os.path.abspath(offline_file_option)): + self.stdio.error('args --file [{0}] not exist: No such file, Please specify it again'.format(os.path.abspath(offline_file_option))) + exit(-1) + else: + self.variable_file_name = os.path.abspath(offline_file_option) + else: + self.stdio.error("an initialization variable file must be provided to find the parts where variables have changed.") + exit(-1) + + if store_dir_option and store_dir_option != "./": + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.export_report_path = os.path.abspath(store_dir_option) + else: + store_dir_option = './variable_report' + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('The report directory is not specified, and a "variable_report" directory will be created in the current directory.'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.export_report_path = os.path.abspath(store_dir_option) + + return True + + def alalyze_variable(self): + sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() + from __all_virtual_sys_variable order by 2, 4, 5''' + db_variable_info = self.obconn.execute_sql(sql) + db_variable_dict = dict() + for row in db_variable_info: + key = str(row[1]) + '-' + str(row[3]) + db_variable_dict[key] = str(row[5]) + file_variable_dict = dict() + last_gather_time = '' + with open(self.variable_file_name, 'r', newline='') as file: + reader = csv.reader(file) + for row in reader: + key = str(row[1]) + '-' + str(row[3]) + file_variable_dict[key] = str(row[5]) + if not last_gather_time: + last_gather_time = row[-1] + report_default_tb = PrettyTable(["VERSION", "TENANT_ID", "ZONE", "NAME", "LAST_VALUE", "CURRENT_VALUE"]) + changed_variables_dict = dict() + for key in db_variable_dict: + if key in file_variable_dict and db_variable_dict[key] != file_variable_dict[key]: + changed_variables_dict[key] = file_variable_dict[key] + is_empty = True + for k in changed_variables_dict: + for row in db_variable_info: + key = str(row[1]) + '-' + str(row[3]) + if k == key: + report_default_tb.add_row([row[0], row[1], row[2], row[3], changed_variables_dict[key], row[5]]) + is_empty = False + if not is_empty: + now = datetime.datetime.now() + date_format = now.strftime("%Y-%m-%d-%H-%M-%S") + file_name = self.export_report_path + '/variables_changed_{0}.table'.format(date_format) + fp = open(file_name, 'a+', encoding="utf8") + fp.write(report_default_tb.get_string() + "\n") + fp.close() + self.stdio.print(Fore.RED + "Since {0}, the following variables have changed：".format(last_gather_time) + Style.RESET_ALL) + self.stdio.print(report_default_tb.get_string()) + self.stdio.print("Analyze variables changed finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + else: + self.stdio.print("Analyze variables changed finished. Since {0}, No changes in variables".format(last_gather_time)) + + def execute(self): + try: + self.alalyze_variable() + except Exception as e: + self.stdio.error("variable info analyze failed, error message: {0}".format(e)) diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py new file mode 100644 index 00000000..b30ebfe9 --- /dev/null +++ b/handler/gather/gather_parameters.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/16 +@file: gather_parameters.py +@desc: +""" +import os +from common.command import get_observer_version_by_sql +from common.tool import DirectoryUtil, TimeUtils, Util, StringUtils +from common.obdiag_exception import OBDIAGFormatException +from common.ob_connector import OBConnector +import csv +from colorama import Fore, Style + + +class GatherParametersHandler(object): + def __init__(self, context, gather_pack_dir='./'): + self.context = context + self.stdio = self.context.stdio + self.gather_pack_dir = gather_pack_dir + self.parameter_file_name = None + self.ob_cluster = self.context.cluster_config + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") + else: + self.gather_timestamp = TimeUtils.get_current_us_timestamp() + self.observer_nodes = self.context.cluster_config.get("servers") + try: + self.obconn = OBConnector( + ip=self.ob_cluster.get("db_host"), + port=self.ob_cluster.get("db_port"), + username=self.ob_cluster.get("tenant_sys").get("user"), + password=self.ob_cluster.get("tenant_sys").get("password"), + stdio=self.stdio, + timeout=10000, + database="oceanbase", + ) + except Exception as e: + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) + + def handle(self): + if not self.init_option(): + self.stdio.error('init option failed') + return False + # example of the format of pack dir for this command: (gather_pack_dir)/gather_pack_20190610123344 + pack_dir_this_command = os.path.join(self.gather_pack_dir, "gather_parameters") + self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) + DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) + self.gather_pack_dir = pack_dir_this_command + self.execute() + + def init_option(self): + options = self.context.options + store_dir_option = Util.get_option(options, 'store_dir') + if store_dir_option and store_dir_option != "./": + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.gather_pack_dir = os.path.abspath(store_dir_option) + return True + + def get_version(self): + observer_version = "" + try: + observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) + except Exception as e: + self.stdio.warn("GatherHandler Failed to get observer version:{0}".format(e)) + self.stdio.verbose("GatherHandler.init get observer version: {0}".format(observer_version)) + return observer_version + + def get_cluster_name(self): + cluster_name = "" + try: + sql = '''select value from __all_virtual_tenant_parameter_stat t2 where name = 'cluster' ''' + cluster_info = self.obconn.execute_sql(sql) + cluster_name = cluster_info[0][0] + except Exception as e: + self.stdio.warn("RCAHandler Failed to get oceanbase cluster name:{0}".format(e)) + self.stdio.verbose("RCAHandler.init get oceanbase cluster name {0}".format(cluster_name)) + return cluster_name + + def get_parameters_info(self): + observer_version = self.get_version() + cluster_name = self.get_cluster_name() + if observer_version: + if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"): + sql = '''select substr(version(),8), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, +EDIT_LEVEL, now(), DEFAULT_VALUE,ISDEFAULT from GV$OB_PARAMETERS order by 5,2,3,4,7''' + elif StringUtils.compare_versions_greater(observer_version, "4.0.0.0"): + sql = '''select substr(version(),8), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, + EDIT_LEVEL, now(), '','' from GV$OB_PARAMETERS order by 5,2,3,4,7''' + else: + sql = '''select version(), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, + EDIT_LEVEL, now(), '','' from __all_virtual_tenant_parameter_info +union +select version(), svr_ip,svr_port,zone,scope,'None' tenant_id,name,value,section, + EDIT_LEVEL, now(), '','' from __all_virtual_sys_parameter_stat where scope='CLUSTER' +''' + parameter_info = self.obconn.execute_sql(sql) + self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + with open(self.parameter_file_name, 'w', newline='') as file: + writer = csv.writer(file) + for row in parameter_info: + if row[5] is None: + tmp_row = [col for col in row] + tmp_row[5] = 'None' + writer.writerow(tmp_row) + else: + writer.writerow(row) + self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'") + else: + self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.") + + def execute(self): + try: + self.get_parameters_info() + except Exception as e: + self.stdio.error("parameter info gather failed, error message: {0}".format(e)) diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py new file mode 100644 index 00000000..b67a1cab --- /dev/null +++ b/handler/gather/gather_variables.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/16 +@file: gather_variables.py +@desc: +""" +import os +from common.tool import DirectoryUtil, TimeUtils, Util +from common.obdiag_exception import OBDIAGFormatException +from common.ob_connector import OBConnector +import csv +from colorama import Fore, Style + + +class GatherVariablesHandler(object): + def __init__(self, context, gather_pack_dir='./'): + self.context = context + self.stdio = self.context.stdio + self.gather_pack_dir = gather_pack_dir + self.variable_file_name = None + self.ob_cluster = self.context.cluster_config + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") + else: + self.gather_timestamp = TimeUtils.get_current_us_timestamp() + self.observer_nodes = self.context.cluster_config.get("servers") + try: + self.obconn = OBConnector( + ip=self.ob_cluster.get("db_host"), + port=self.ob_cluster.get("db_port"), + username=self.ob_cluster.get("tenant_sys").get("user"), + password=self.ob_cluster.get("tenant_sys").get("password"), + stdio=self.stdio, + timeout=10000, + database="oceanbase", + ) + except Exception as e: + self.stdio.error("Failed to connect to database: {0}".format(e)) + raise OBDIAGFormatException("Failed to connect to database: {0}".format(e)) + + def handle(self): + if not self.init_option(): + self.stdio.error('init option failed') + return False + pack_dir_this_command = os.path.join(self.gather_pack_dir, "gather_variables") + self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) + DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) + self.gather_pack_dir = pack_dir_this_command + self.execute() + + def init_option(self): + options = self.context.options + store_dir_option = Util.get_option(options, 'store_dir') + if store_dir_option and store_dir_option != "./": + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.gather_pack_dir = os.path.abspath(store_dir_option) + return True + + def get_cluster_name(self): + cluster_name = "" + try: + sql = '''select value from __all_virtual_tenant_parameter_stat t2 where name = 'cluster' ''' + cluster_info = self.obconn.execute_sql(sql) + cluster_name = cluster_info[0][0] + except Exception as e: + self.stdio.warn("RCAHandler Failed to get oceanbase cluster name:{0}".format(e)) + self.stdio.verbose("RCAHandler.init get oceanbase cluster name {0}".format(cluster_name)) + return cluster_name + + def get_variables_info(self): + cluster_name = self.get_cluster_name() + sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() + from __all_virtual_sys_variable order by 2, 4, 5''' + variable_info = self.obconn.execute_sql(sql) + self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + with open(self.variable_file_name, 'w', newline='') as file: + writer = csv.writer(file) + for row in variable_info: + writer.writerow(row) + self.stdio.print("Gather variables finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.variable_file_name) + Style.RESET_ALL + "'") + + def execute(self): + try: + self.get_variables_info() + except Exception as e: + self.stdio.error("parameter info gather failed, error message: {0}".format(e)) From 19f72589822ecf5812d57c48c6558769e80b935d Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Fri, 5 Jul 2024 20:38:24 +0800 Subject: [PATCH 02/12] obdiag gather and analyze parameters/variables --- diag_cmd.py | 4 ++-- handler/analyzer/analyze_parameter.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/diag_cmd.py b/diag_cmd.py index 62a11918..4235da37 100644 --- a/diag_cmd.py +++ b/diag_cmd.py @@ -662,7 +662,6 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_flt_trace', self.opts) - class ObdiagAnalyzeParameterDiffCommand(ObdiagOriginCommand): def __init__(self): super(ObdiagAnalyzeParameterDiffCommand, self).__init__('diff', 'Analyze the parameter configurations between observers and identify the parameters with different values among the observers') @@ -703,7 +702,6 @@ def __init__(self): class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): - def __init__(self): super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') self.parser.add_option('--file', type='string', help="specify initialization parameter file") @@ -830,6 +828,7 @@ def __init__(self): self.register_command(ObdiagGatherParameterCommand()) self.register_command(ObdiagGatherVariableCommand()) + class ObdiagGatherSceneCommand(MajorCommand): def __init__(self): @@ -847,6 +846,7 @@ def __init__(self): self.register_command(ObdiagAnalyzeParameterCommand()) self.register_command(ObdiagAnalyzeVariableCommand()) + class ObdiagRCACommand(MajorCommand): def __init__(self): diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 271362b4..97ca13cc 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -256,7 +256,6 @@ def alalyze_parameter_diff(self): else: self.stdio.print("Analyze parameter diff finished. All parameter settings are consistent among observers") - def execute(self): try: if self.analyze_type == 'non_default': From 73ae942b1b9b20d750f6931d6434725758abb2ec Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Fri, 5 Jul 2024 20:44:30 +0800 Subject: [PATCH 03/12] obdiag gather and analyze parameters/variables --- .github/workflows/build_rpm.yml | 54 ++ cmd.py | 863 ++++++++++++++++++++++++++++++++ 2 files changed, 917 insertions(+) create mode 100644 .github/workflows/build_rpm.yml create mode 100644 cmd.py diff --git a/.github/workflows/build_rpm.yml b/.github/workflows/build_rpm.yml new file mode 100644 index 00000000..e5dc3715 --- /dev/null +++ b/.github/workflows/build_rpm.yml @@ -0,0 +1,54 @@ +name: build rpm + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + build-and-package-x86_64: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip setuptools wheel + pip3 install -r requirements3.txt + + - name: Set up RPM build env + run: | + sudo apt-get update && sudo apt-get install -y rpm alien fakeroot + + - name: Build package + run: | + pwd + ls -lh + export RELEASE=`date +%Y%m%d%H%M` + cat ./rpm/oceanbase-diagnostic-tool.spec + rpmbuild -bb ./rpm/oceanbase-diagnostic-tool.spec + + - name: Convert RPM to DEB + run: | + mkdir -p /home/runner/artifacts + fakeroot alien --scripts --to-deb /home/runner/rpmbuild/RPMS/x86_64/oceanbase-diagnostic-tool-*.rpm + + - name: 'Upload Artifacts' + uses: actions/upload-artifact@v3 + with: + name: obdiag-packages + path: | + /home/runner/rpmbuild/RPMS/x86_64/oceanbase-diagnostic-tool-*.rpm + /home/runner/work/oceanbase-diagnostic-tool/oceanbase-diagnostic-tool/oceanbase-diagnostic-tool_*.deb + retention-days: 3 + debug: true \ No newline at end of file diff --git a/cmd.py b/cmd.py new file mode 100644 index 00000000..311847f8 --- /dev/null +++ b/cmd.py @@ -0,0 +1,863 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@file: cmd.py +@desc: +""" + +from __future__ import absolute_import, division, print_function +from common.tool import Util + +import os +import sys +import textwrap +import re +from uuid import uuid1 as uuid, UUID +from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter +from core import ObdiagHome +from stdio import IO +from common.version import get_obdiag_version +from telemetry.telemetry import telemetry + +ROOT_IO = IO(1) +OBDIAG_HOME_PATH = os.path.join(os.getenv('HOME'), 'oceanbase-diagnostic-tool') + + +class OptionHelpFormatter(IndentedHelpFormatter): + + def format_option(self, option): + result = [] + opts = self.option_strings[option] + opt_width = self.help_position - self.current_indent - 2 + if len(opts) > opt_width: + opts = "%*s%s\n" % (self.current_indent, "", opts) + indent_first = self.help_position + else: + opts = "%*s%-*s " % (self.current_indent, "", opt_width, opts) + indent_first = 0 + result.append(opts) + if option.help: + help_text = self.expand_default(option) + help_lines = help_text.split('\n') + if len(help_lines) == 1: + help_lines = textwrap.wrap(help_text, self.help_width) + result.append("%*s%s\n" % (indent_first, "", help_lines[0])) + result.extend(["%*s%s\n" % (self.help_position, "", line) for line in help_lines[1:]]) + elif opts[-1] != "\n": + result.append("\n") + return "".join(result) + + +class AllowUndefinedOptionParser(OptionParser): + IS_TTY = sys.stdin.isatty() + + def __init__(self, usage=None, option_list=None, option_class=Option, version=None, conflict_handler="resolve", description=None, formatter=None, add_help_option=True, prog=None, epilog=None, allow_undefine=True, undefine_warn=True): + OptionParser.__init__(self, usage, option_list, option_class, version, conflict_handler, description, formatter, add_help_option, prog, epilog) + self.allow_undefine = allow_undefine + self.undefine_warn = undefine_warn + + def warn(self, msg, file=None): + if self.IS_TTY: + print("%s %s" % (IO.WARNING_PREV, msg)) + else: + print('warn: %s' % msg) + + def _process_long_opt(self, rargs, values): + try: + value = rargs[0] + OptionParser._process_long_opt(self, rargs, values) + except BadOptionError as e: + if self.allow_undefine: + key = e.opt_str + value = value[len(key) + 1 :] + setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) + self.undefine_warn and self.warn(e) + else: + raise e + + def _process_short_opts(self, rargs, values): + try: + value = rargs[0] + OptionParser._process_short_opts(self, rargs, values) + except BadOptionError as e: + if self.allow_undefine: + key = e.opt_str + value = value[len(key) + 1 :] + setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) + self.undefine_warn and self.warn(e) + else: + raise e + + +class BaseCommand(object): + + def __init__(self, name, summary): + self.name = name + self.summary = summary + self.args = [] + self.cmds = [] + self.opts = {} + self.prev_cmd = '' + self.is_init = False + self.hidden = False + self.has_trace = True + self.parser = AllowUndefinedOptionParser(add_help_option=True) + self.parser.add_option('-h', '--help', action='callback', callback=self._show_help, help='Show help and exit.') + self.parser.add_option('-v', '--verbose', action='callback', callback=self._set_verbose, help='Activate verbose output.') + + def _set_verbose(self, *args, **kwargs): + ROOT_IO.set_verbose_level(0xFFFFFFF) + + def init(self, cmd, args): + if self.is_init is False: + self.prev_cmd = cmd + self.args = args + self.is_init = True + self.parser.prog = self.prev_cmd + option_list = self.parser.option_list[2:] + option_list.append(self.parser.option_list[0]) + option_list.append(self.parser.option_list[1]) + self.parser.option_list = option_list + return self + + def parse_command(self): + self.opts, self.cmds = self.parser.parse_args(self.args) + return self.opts + + def do_command(self): + raise NotImplementedError + + def _show_help(self, *args, **kwargs): + ROOT_IO.print(self._mk_usage()) + self.parser.exit(0) + + def _mk_usage(self): + return self.parser.format_help(OptionHelpFormatter()) + + +class ObdiagOriginCommand(BaseCommand): + OBDIAG_PATH = OBDIAG_HOME_PATH + + @property + def enable_log(self): + return True + + def is_valid_time_format(self, time_string): + time_pattern = r'^\d{2}:\d{2}:\d{2}$' + return bool(re.match(time_pattern, time_string)) + + def preprocess_argv(self, argv): + """ + Preprocesses the command line arguments to ensure that date-time strings for --from and --to + options are properly quoted, even if they are originally provided without quotes. + """ + processed_argv = [] + from_index = None + to_index = None + for i, arg in enumerate(argv): + if arg == '--from': + from_index = i + 1 + elif arg == '--to': + to_index = i + 1 + + if from_index is not None and i == from_index: + next_arg = argv[i + 1] if i + 1 < len(argv) else None + if next_arg and self.is_valid_time_format(next_arg): + processed_argv.append(argv[i] + ' ' + next_arg) + from_index = None + i += 1 + else: + processed_argv.append(arg) + elif to_index is not None and i == to_index: + next_arg = argv[i + 1] if i + 1 < len(argv) else None + if next_arg and self.is_valid_time_format(next_arg): + processed_argv.append(argv[i] + ' ' + next_arg) + to_index = None + i += 1 + else: + processed_argv.append(arg) + else: + processed_argv.append(arg) + return processed_argv + + def parse_command(self): + self.args = self.preprocess_argv(self.args) + return super(ObdiagOriginCommand, self).parse_command() + + def do_command(self): + self.parse_command() + trace_id = uuid() + ret = False + try: + log_directory = os.path.join(os.path.expanduser("~"), ".obdiag", "log") + if not os.path.exists(log_directory): + os.makedirs(log_directory, exist_ok=True) + log_path = os.path.join(log_directory, 'obdiag.log') + if self.enable_log: + ROOT_IO.init_trace_logger(log_path, 'obdiag', trace_id) + ROOT_IO.track_limit += 1 + ROOT_IO.verbose('cmd: %s' % self.cmds) + ROOT_IO.verbose('opts: %s' % self.opts) + config_path = os.path.expanduser('~/.obdiag/config.yml') + custom_config = Util.get_option(self.opts, 'c') + if custom_config: + config_path = custom_config + obdiag = ObdiagHome(stdio=ROOT_IO, config_path=config_path) + obdiag.set_options(self.opts) + obdiag.set_cmds(self.cmds) + ret = self._do_command(obdiag) + telemetry.put_data() + except NotImplementedError: + ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd) + except SystemExit: + pass + except KeyboardInterrupt: + ROOT_IO.exception('Keyboard Interrupt') + except: + e = sys.exc_info()[1] + ROOT_IO.exception('Running Error: %s' % e) + if self.has_trace: + ROOT_IO.print('Trace ID: %s' % trace_id) + ROOT_IO.print('If you want to view detailed obdiag logs, please run: obdiag display-trace %s' % trace_id) + return ret + + def _do_command(self, obdiag): + raise NotImplementedError + + def get_white_ip_list(self): + if self.opts.white: + return self.opts.white.split(',') + ROOT_IO.warn("Security Risk: the whitelist is empty and anyone can request this program!") + if ROOT_IO.confirm("Do you want to continue?"): + return [] + wthite_ip_list = ROOT_IO.read("Please enter the whitelist, eq: '192.168.1.1'") + raise wthite_ip_list.split(',') + + +class DisplayTraceCommand(ObdiagOriginCommand): + + def __init__(self): + super(DisplayTraceCommand, self).__init__('display-trace', 'display trace_id log.') + self.has_trace = False + + @property + def enable_log(self): + return False + + def _do_command(self, obdiag): + from common.ssh import LocalClient + + if not self.cmds: + return self._show_help() + log_dir = os.path.expanduser('~/.obdiag/log') + trace_id = self.cmds[0] + ROOT_IO.verbose('Get log by trace_id') + try: + if UUID(trace_id).version != 1: + ROOT_IO.critical('%s is not trace id' % trace_id) + return False + except: + ROOT_IO.print('%s is not trace id' % trace_id) + return False + cmd = 'cd {} && grep -h "\[{}\]" $(ls -tr {}*) | sed "s/\[{}\] //g" '.format(log_dir, trace_id, log_dir, trace_id) + data = LocalClient.execute_command(cmd) + ROOT_IO.print(data.stdout) + return True + + +class MajorCommand(BaseCommand): + + def __init__(self, name, summary): + super(MajorCommand, self).__init__(name, summary) + self.commands = {} + + def _mk_usage(self): + if self.commands: + usage = ['%s [options]\n\nAvailable commands:\n' % self.prev_cmd] + commands = [x for x in self.commands.values() if not (hasattr(x, 'hidden') and x.hidden)] + commands.sort(key=lambda x: x.name) + for command in commands: + if command.hidden is False: + usage.append("%-12s %s\n" % (command.name, command.summary)) + self.parser.set_usage('\n'.join(usage)) + return super(MajorCommand, self)._mk_usage() + + def do_command(self): + if not self.is_init: + ROOT_IO.error('%s command not init' % self.prev_cmd) + raise SystemExit('command not init') + if len(self.args) < 1: + ROOT_IO.print('You need to give some commands.\n\nTry `obdiag --help` for more information.') + self._show_help() + return False + base, args = self.args[0], self.args[1:] + if base not in self.commands: + self.parse_command() + self._show_help() + return False + cmd = '%s %s' % (self.prev_cmd, base) + ROOT_IO.track_limit += 1 + if "main.py" in cmd: + telemetry.work_tag = False + telemetry.push_cmd_info("cmd: {0}. args:{1}".format(cmd, args)) + return self.commands[base].init(cmd, args).do_command() + + def register_command(self, command): + self.commands[command.name] = command + + +class ObdiagGatherAllCommand(ObdiagOriginCommand): + + def init(self, cmd, args): + super(ObdiagGatherAllCommand, self).init(cmd, args) + return self + + def __init__(self): + super(ObdiagGatherAllCommand, self).__init__('all', 'Gather oceanbase diagnostic info') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherAllCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_all', self.opts) + + +class ObdiagGatherLogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherLogCommand, self).__init__('log', 'Gather oceanbase logs from oceanbase machines') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherLogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_log', self.opts) + + +class ObdiagGatherSysStatCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSysStatCommand, self).__init__('sysstat', 'Gather Host information') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherSysStatCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_sysstat', self.opts) + + +class ObdiagGatherStackCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherStackCommand, self).__init__('stack', 'Gather stack') + + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherStackCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_obstack', self.opts) + + +class ObdiagGatherPerfCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherPerfCommand, self).__init__('perf', 'Gather perf') + + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('--scope', type='string', help="perf type constrains, choices=[sample, flame, pstack, all]", default='all') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherPerfCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_perf', self.opts) + + +class ObdiagGatherSlogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSlogCommand, self).__init__('slog', 'Gather slog') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherSlogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_slog', self.opts) + + +class ObdiagGatherClogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherClogCommand, self).__init__('clog', 'Gather clog') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherClogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_clog', self.opts) + + +class ObdiagGatherAwrCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherAwrCommand, self).__init__('awr', 'Gather ParalleSQL information') + self.parser.add_option('--cluster_name', type='string', help='cluster_name from ocp') + self.parser.add_option('--cluster_id', type='string', help='cluster_id from ocp') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherAwrCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_awr', self.opts) + + +class ObdiagGatherPlanMonitorCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information') + self.parser.add_option('--trace_id', type='string', help='sql trace id') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('--env', type='string', help='''env, eg: "{db_connect='-h127.0.0.1 -P2881 -utest@test -p****** -Dtest'}"''') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherPlanMonitorCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_plan_monitor', self.opts) + + +class ObdiagGatherObproxyLogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherObproxyLogCommand, self).__init__('obproxy_log', 'Gather obproxy log from obproxy machines') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherObproxyLogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_obproxy_log(self.opts) + + +class ObdiagGatherSceneListCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSceneListCommand, self).__init__('list', 'gather scene list') + + def init(self, cmd, args): + super(ObdiagGatherSceneListCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_scenes_list(self.opts) + + +class ObdiagGatherSceneRunCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSceneRunCommand, self).__init__('run', 'gather scene run') + self.parser.add_option('--scene', type='string', help="Specify the scene to be gather") + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherSceneRunCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_scenes_run', self.opts) + + +class ObdiagGatherAshReportCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherAshReportCommand, self).__init__('ash', 'Gather ash report') + self.parser.add_option('--trace_id', type='string', help="The TRACE.ID of the SQL to be sampled, if left blank or filled with NULL, indicates that TRACE.ID is not restricted.") + self.parser.add_option('--sql_id', type='string', help="The SQL.ID, if left blank or filled with NULL, indicates that SQL.ID is not restricted.") + # WAIT_CLASS + self.parser.add_option('--wait_class', type='string', help='Event types to be sampled.') + self.parser.add_option('--report_type', type='string', help='Report type, currently only supports text type.', default='TEXT') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherAshReportCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_ash_report', self.opts) + + +class ObdiagAnalyzeLogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeLogCommand, self).__init__('log', 'Analyze oceanbase log from online observer machines or offline oceanbase log files') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--log_level', type='string', help="oceanbase logs greater than or equal to this level will be analyze, choices=[DEBUG, TRACE, INFO, WDIAG, WARN, EDIAG, ERROR]") + self.parser.add_option('--files', action="append", type='string', help="specify files") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + + def init(self, cmd, args): + super(ObdiagAnalyzeLogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + offline_args_sign = '--files' + if self.args and (offline_args_sign in self.args): + return obdiag.analyze_fuction('analyze_log_offline', self.opts) + else: + return obdiag.analyze_fuction('analyze_log', self.opts) + + +class ObdiagAnalyzeFltTraceCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeFltTraceCommand, self).__init__('flt_trace', 'Analyze oceanbase trace.log from online observer machines or offline oceanbase trace.log files') + self.parser.add_option('--flt_trace_id', type='string', help="flt trace id, . format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") + self.parser.add_option('--files', action="append", help="specify files") + self.parser.add_option('--top', type='string', help="top leaf span", default=5) + self.parser.add_option('--recursion', type='string', help="Maximum number of recursion", default=8) + self.parser.add_option('--output', type='string', help="Print the result to the maximum output line on the screen", default=60) + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeFltTraceCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_flt_trace', self.opts) + + +class ObdiagAnalyzeParameterCommand(MajorCommand): + + def __init__(self): + super(ObdiagAnalyzeParameterCommand, self).__init__('parameters', 'Analyze parameters info') + self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) + self.register_command(ObdiagAnalyzeParameterDiffCommand()) + + +class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'list non-default value') + self.parser.add_option('--file', type='string', help="specify initialization parameter file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + + +class ObdiagAnalyzeParameterDiffCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeParameterDiffCommand, self).__init__('diff', 'list inconsistent values among different observers') + self.parser.add_option('--file', type='string', help="specify parameter file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeParameterDiffCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) + + +class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeVariableCommand, self).__init__('variables', 'Analyze oceanbase variable from online observer machines to offline oceanbase variable files') + self.parser.add_option('--file', type='string', help="specify variable file") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_variable', self.opts) + + +class ObdiagCheckCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagCheckCommand, self).__init__('check', 'check oceanbase cluster') + self.parser.add_option('--cases', type='string', help="check observer's cases on package_file") + self.parser.add_option('--obproxy_cases', type='string', help="check obproxy's cases on package_file") + self.parser.add_option('--store_dir', type='string', help='the dir to store check result, current dir by default.', default='./check_report/') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagCheckCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + if 'list' in self.args: + obdiag.check_list(self.opts) + return + return obdiag.check(self.opts) + + +class ObdiagRCARunCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagRCARunCommand, self).__init__('run', 'root cause analysis') + self.parser.add_option('--scene', type='string', help="rca scene name. The argument is required.") + self.parser.add_option('--store_dir', type='string', help='the dir to store rca result, current dir by default.', default='./rca/') + self.parser.add_option('--input_parameters', type='string', help='input parameters of scene') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagRCARunCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.rca_run(self.opts) + + +class ObdiagRCAListCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagRCAListCommand, self).__init__('list', 'show list of rca list') + + def init(self, cmd, args): + super(ObdiagRCAListCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.rca_list(self.opts) + + +class ObdiagConfigCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagConfigCommand, self).__init__('config', 'Quick build config') + self.parser.add_option('-h', type='string', help="database host") + self.parser.add_option('-u', type='string', help='sys_user', default='root@sys') + self.parser.add_option('-p', type='string', help="password", default='') + self.parser.add_option('-P', type='string', help="port") + + def init(self, cmd, args): + super(ObdiagConfigCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.config(self.opts) + + +class ObdiagUpdateCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagUpdateCommand, self).__init__('update', 'Update cheat files') + self.parser.add_option('--file', type='string', help="obdiag update cheat file path. Please note that you need to ensure the reliability of the files on your own.") + self.parser.add_option( + '--force', + action='store_true', + help='You can force online upgrades by adding --force in the command', + ) + + def init(self, cmd, args): + super(ObdiagUpdateCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.update(self.opts) + + +class ObdiagGatherParametersCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherParametersCommand, self).__init__('parameters', 'Gather parameters info') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherParametersCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_parameters', self.opts) + + +class ObdiagGatherVariablesCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherVariablesCommand, self).__init__('variables', 'Gather variables info') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherVariablesCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_variables', self.opts) + + +class ObdiagGatherCommand(MajorCommand): + + def __init__(self): + super(ObdiagGatherCommand, self).__init__('gather', 'Gather oceanbase diagnostic info') + self.register_command(ObdiagGatherAllCommand()) + self.register_command(ObdiagGatherLogCommand()) + self.register_command(ObdiagGatherSysStatCommand()) + self.register_command(ObdiagGatherStackCommand()) + self.register_command(ObdiagGatherPerfCommand()) + self.register_command(ObdiagGatherSlogCommand()) + self.register_command(ObdiagGatherClogCommand()) + self.register_command(ObdiagGatherPlanMonitorCommand()) + self.register_command(ObdiagGatherAwrCommand()) + self.register_command(ObdiagGatherObproxyLogCommand()) + self.register_command(ObdiagGatherSceneCommand()) + self.register_command(ObdiagGatherAshReportCommand()) + self.register_command(ObdiagGatherParametersCommand()) + self.register_command(ObdiagGatherVariablesCommand()) + + +class ObdiagGatherSceneCommand(MajorCommand): + + def __init__(self): + super(ObdiagGatherSceneCommand, self).__init__('scene', 'Gather scene diagnostic info') + self.register_command(ObdiagGatherSceneListCommand()) + self.register_command(ObdiagGatherSceneRunCommand()) + + +class ObdiagAnalyzeCommand(MajorCommand): + + def __init__(self): + super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info') + self.register_command(ObdiagAnalyzeLogCommand()) + self.register_command(ObdiagAnalyzeFltTraceCommand()) + self.register_command(ObdiagAnalyzeParameterCommand()) + self.register_command(ObdiagAnalyzeVariableCommand()) + + +class ObdiagRCACommand(MajorCommand): + + def __init__(self): + super(ObdiagRCACommand, self).__init__('rca', 'root cause analysis') + self.register_command(ObdiagRCARunCommand()) + self.register_command(ObdiagRCAListCommand()) + + +class MainCommand(MajorCommand): + + def __init__(self): + super(MainCommand, self).__init__('obdiag', '') + self.register_command(DisplayTraceCommand()) + self.register_command(ObdiagGatherCommand()) + self.register_command(ObdiagAnalyzeCommand()) + self.register_command(ObdiagCheckCommand()) + self.register_command(ObdiagRCACommand()) + self.register_command(ObdiagConfigCommand()) + self.register_command(ObdiagUpdateCommand()) + self.parser.version = get_obdiag_version() + self.parser._add_version_option() From eb5109ff07cddcd415244efb3b54f8914d48ab37 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Mon, 8 Jul 2024 12:02:25 +0800 Subject: [PATCH 04/12] obdiag gather and analyze parameters/variables --- .github/workflows/build_rpm.yml | 54 -- cmd.py | 863 --------------------------- handler/analyzer/analyze_variable.py | 2 +- handler/gather/gather_parameters.py | 6 +- handler/gather/gather_variables.py | 4 +- 5 files changed, 6 insertions(+), 923 deletions(-) delete mode 100644 .github/workflows/build_rpm.yml delete mode 100644 cmd.py diff --git a/.github/workflows/build_rpm.yml b/.github/workflows/build_rpm.yml deleted file mode 100644 index e5dc3715..00000000 --- a/.github/workflows/build_rpm.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: build rpm - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - build-and-package-x86_64: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.8' - - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip setuptools wheel - pip3 install -r requirements3.txt - - - name: Set up RPM build env - run: | - sudo apt-get update && sudo apt-get install -y rpm alien fakeroot - - - name: Build package - run: | - pwd - ls -lh - export RELEASE=`date +%Y%m%d%H%M` - cat ./rpm/oceanbase-diagnostic-tool.spec - rpmbuild -bb ./rpm/oceanbase-diagnostic-tool.spec - - - name: Convert RPM to DEB - run: | - mkdir -p /home/runner/artifacts - fakeroot alien --scripts --to-deb /home/runner/rpmbuild/RPMS/x86_64/oceanbase-diagnostic-tool-*.rpm - - - name: 'Upload Artifacts' - uses: actions/upload-artifact@v3 - with: - name: obdiag-packages - path: | - /home/runner/rpmbuild/RPMS/x86_64/oceanbase-diagnostic-tool-*.rpm - /home/runner/work/oceanbase-diagnostic-tool/oceanbase-diagnostic-tool/oceanbase-diagnostic-tool_*.deb - retention-days: 3 - debug: true \ No newline at end of file diff --git a/cmd.py b/cmd.py deleted file mode 100644 index 311847f8..00000000 --- a/cmd.py +++ /dev/null @@ -1,863 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -* -# Copyright (c) 2022 OceanBase -# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -# See the Mulan PSL v2 for more details. - -""" -@file: cmd.py -@desc: -""" - -from __future__ import absolute_import, division, print_function -from common.tool import Util - -import os -import sys -import textwrap -import re -from uuid import uuid1 as uuid, UUID -from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter -from core import ObdiagHome -from stdio import IO -from common.version import get_obdiag_version -from telemetry.telemetry import telemetry - -ROOT_IO = IO(1) -OBDIAG_HOME_PATH = os.path.join(os.getenv('HOME'), 'oceanbase-diagnostic-tool') - - -class OptionHelpFormatter(IndentedHelpFormatter): - - def format_option(self, option): - result = [] - opts = self.option_strings[option] - opt_width = self.help_position - self.current_indent - 2 - if len(opts) > opt_width: - opts = "%*s%s\n" % (self.current_indent, "", opts) - indent_first = self.help_position - else: - opts = "%*s%-*s " % (self.current_indent, "", opt_width, opts) - indent_first = 0 - result.append(opts) - if option.help: - help_text = self.expand_default(option) - help_lines = help_text.split('\n') - if len(help_lines) == 1: - help_lines = textwrap.wrap(help_text, self.help_width) - result.append("%*s%s\n" % (indent_first, "", help_lines[0])) - result.extend(["%*s%s\n" % (self.help_position, "", line) for line in help_lines[1:]]) - elif opts[-1] != "\n": - result.append("\n") - return "".join(result) - - -class AllowUndefinedOptionParser(OptionParser): - IS_TTY = sys.stdin.isatty() - - def __init__(self, usage=None, option_list=None, option_class=Option, version=None, conflict_handler="resolve", description=None, formatter=None, add_help_option=True, prog=None, epilog=None, allow_undefine=True, undefine_warn=True): - OptionParser.__init__(self, usage, option_list, option_class, version, conflict_handler, description, formatter, add_help_option, prog, epilog) - self.allow_undefine = allow_undefine - self.undefine_warn = undefine_warn - - def warn(self, msg, file=None): - if self.IS_TTY: - print("%s %s" % (IO.WARNING_PREV, msg)) - else: - print('warn: %s' % msg) - - def _process_long_opt(self, rargs, values): - try: - value = rargs[0] - OptionParser._process_long_opt(self, rargs, values) - except BadOptionError as e: - if self.allow_undefine: - key = e.opt_str - value = value[len(key) + 1 :] - setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) - self.undefine_warn and self.warn(e) - else: - raise e - - def _process_short_opts(self, rargs, values): - try: - value = rargs[0] - OptionParser._process_short_opts(self, rargs, values) - except BadOptionError as e: - if self.allow_undefine: - key = e.opt_str - value = value[len(key) + 1 :] - setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) - self.undefine_warn and self.warn(e) - else: - raise e - - -class BaseCommand(object): - - def __init__(self, name, summary): - self.name = name - self.summary = summary - self.args = [] - self.cmds = [] - self.opts = {} - self.prev_cmd = '' - self.is_init = False - self.hidden = False - self.has_trace = True - self.parser = AllowUndefinedOptionParser(add_help_option=True) - self.parser.add_option('-h', '--help', action='callback', callback=self._show_help, help='Show help and exit.') - self.parser.add_option('-v', '--verbose', action='callback', callback=self._set_verbose, help='Activate verbose output.') - - def _set_verbose(self, *args, **kwargs): - ROOT_IO.set_verbose_level(0xFFFFFFF) - - def init(self, cmd, args): - if self.is_init is False: - self.prev_cmd = cmd - self.args = args - self.is_init = True - self.parser.prog = self.prev_cmd - option_list = self.parser.option_list[2:] - option_list.append(self.parser.option_list[0]) - option_list.append(self.parser.option_list[1]) - self.parser.option_list = option_list - return self - - def parse_command(self): - self.opts, self.cmds = self.parser.parse_args(self.args) - return self.opts - - def do_command(self): - raise NotImplementedError - - def _show_help(self, *args, **kwargs): - ROOT_IO.print(self._mk_usage()) - self.parser.exit(0) - - def _mk_usage(self): - return self.parser.format_help(OptionHelpFormatter()) - - -class ObdiagOriginCommand(BaseCommand): - OBDIAG_PATH = OBDIAG_HOME_PATH - - @property - def enable_log(self): - return True - - def is_valid_time_format(self, time_string): - time_pattern = r'^\d{2}:\d{2}:\d{2}$' - return bool(re.match(time_pattern, time_string)) - - def preprocess_argv(self, argv): - """ - Preprocesses the command line arguments to ensure that date-time strings for --from and --to - options are properly quoted, even if they are originally provided without quotes. - """ - processed_argv = [] - from_index = None - to_index = None - for i, arg in enumerate(argv): - if arg == '--from': - from_index = i + 1 - elif arg == '--to': - to_index = i + 1 - - if from_index is not None and i == from_index: - next_arg = argv[i + 1] if i + 1 < len(argv) else None - if next_arg and self.is_valid_time_format(next_arg): - processed_argv.append(argv[i] + ' ' + next_arg) - from_index = None - i += 1 - else: - processed_argv.append(arg) - elif to_index is not None and i == to_index: - next_arg = argv[i + 1] if i + 1 < len(argv) else None - if next_arg and self.is_valid_time_format(next_arg): - processed_argv.append(argv[i] + ' ' + next_arg) - to_index = None - i += 1 - else: - processed_argv.append(arg) - else: - processed_argv.append(arg) - return processed_argv - - def parse_command(self): - self.args = self.preprocess_argv(self.args) - return super(ObdiagOriginCommand, self).parse_command() - - def do_command(self): - self.parse_command() - trace_id = uuid() - ret = False - try: - log_directory = os.path.join(os.path.expanduser("~"), ".obdiag", "log") - if not os.path.exists(log_directory): - os.makedirs(log_directory, exist_ok=True) - log_path = os.path.join(log_directory, 'obdiag.log') - if self.enable_log: - ROOT_IO.init_trace_logger(log_path, 'obdiag', trace_id) - ROOT_IO.track_limit += 1 - ROOT_IO.verbose('cmd: %s' % self.cmds) - ROOT_IO.verbose('opts: %s' % self.opts) - config_path = os.path.expanduser('~/.obdiag/config.yml') - custom_config = Util.get_option(self.opts, 'c') - if custom_config: - config_path = custom_config - obdiag = ObdiagHome(stdio=ROOT_IO, config_path=config_path) - obdiag.set_options(self.opts) - obdiag.set_cmds(self.cmds) - ret = self._do_command(obdiag) - telemetry.put_data() - except NotImplementedError: - ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd) - except SystemExit: - pass - except KeyboardInterrupt: - ROOT_IO.exception('Keyboard Interrupt') - except: - e = sys.exc_info()[1] - ROOT_IO.exception('Running Error: %s' % e) - if self.has_trace: - ROOT_IO.print('Trace ID: %s' % trace_id) - ROOT_IO.print('If you want to view detailed obdiag logs, please run: obdiag display-trace %s' % trace_id) - return ret - - def _do_command(self, obdiag): - raise NotImplementedError - - def get_white_ip_list(self): - if self.opts.white: - return self.opts.white.split(',') - ROOT_IO.warn("Security Risk: the whitelist is empty and anyone can request this program!") - if ROOT_IO.confirm("Do you want to continue?"): - return [] - wthite_ip_list = ROOT_IO.read("Please enter the whitelist, eq: '192.168.1.1'") - raise wthite_ip_list.split(',') - - -class DisplayTraceCommand(ObdiagOriginCommand): - - def __init__(self): - super(DisplayTraceCommand, self).__init__('display-trace', 'display trace_id log.') - self.has_trace = False - - @property - def enable_log(self): - return False - - def _do_command(self, obdiag): - from common.ssh import LocalClient - - if not self.cmds: - return self._show_help() - log_dir = os.path.expanduser('~/.obdiag/log') - trace_id = self.cmds[0] - ROOT_IO.verbose('Get log by trace_id') - try: - if UUID(trace_id).version != 1: - ROOT_IO.critical('%s is not trace id' % trace_id) - return False - except: - ROOT_IO.print('%s is not trace id' % trace_id) - return False - cmd = 'cd {} && grep -h "\[{}\]" $(ls -tr {}*) | sed "s/\[{}\] //g" '.format(log_dir, trace_id, log_dir, trace_id) - data = LocalClient.execute_command(cmd) - ROOT_IO.print(data.stdout) - return True - - -class MajorCommand(BaseCommand): - - def __init__(self, name, summary): - super(MajorCommand, self).__init__(name, summary) - self.commands = {} - - def _mk_usage(self): - if self.commands: - usage = ['%s [options]\n\nAvailable commands:\n' % self.prev_cmd] - commands = [x for x in self.commands.values() if not (hasattr(x, 'hidden') and x.hidden)] - commands.sort(key=lambda x: x.name) - for command in commands: - if command.hidden is False: - usage.append("%-12s %s\n" % (command.name, command.summary)) - self.parser.set_usage('\n'.join(usage)) - return super(MajorCommand, self)._mk_usage() - - def do_command(self): - if not self.is_init: - ROOT_IO.error('%s command not init' % self.prev_cmd) - raise SystemExit('command not init') - if len(self.args) < 1: - ROOT_IO.print('You need to give some commands.\n\nTry `obdiag --help` for more information.') - self._show_help() - return False - base, args = self.args[0], self.args[1:] - if base not in self.commands: - self.parse_command() - self._show_help() - return False - cmd = '%s %s' % (self.prev_cmd, base) - ROOT_IO.track_limit += 1 - if "main.py" in cmd: - telemetry.work_tag = False - telemetry.push_cmd_info("cmd: {0}. args:{1}".format(cmd, args)) - return self.commands[base].init(cmd, args).do_command() - - def register_command(self, command): - self.commands[command.name] = command - - -class ObdiagGatherAllCommand(ObdiagOriginCommand): - - def init(self, cmd, args): - super(ObdiagGatherAllCommand, self).init(cmd, args) - return self - - def __init__(self): - super(ObdiagGatherAllCommand, self).__init__('all', 'Gather oceanbase diagnostic info') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherAllCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_all', self.opts) - - -class ObdiagGatherLogCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherLogCommand, self).__init__('log', 'Gather oceanbase logs from oceanbase machines') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherLogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_log', self.opts) - - -class ObdiagGatherSysStatCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherSysStatCommand, self).__init__('sysstat', 'Gather Host information') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherSysStatCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_sysstat', self.opts) - - -class ObdiagGatherStackCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherStackCommand, self).__init__('stack', 'Gather stack') - - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherStackCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_obstack', self.opts) - - -class ObdiagGatherPerfCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherPerfCommand, self).__init__('perf', 'Gather perf') - - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('--scope', type='string', help="perf type constrains, choices=[sample, flame, pstack, all]", default='all') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherPerfCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_perf', self.opts) - - -class ObdiagGatherSlogCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherSlogCommand, self).__init__('slog', 'Gather slog') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherSlogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_slog', self.opts) - - -class ObdiagGatherClogCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherClogCommand, self).__init__('clog', 'Gather clog') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherClogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_clog', self.opts) - - -class ObdiagGatherAwrCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherAwrCommand, self).__init__('awr', 'Gather ParalleSQL information') - self.parser.add_option('--cluster_name', type='string', help='cluster_name from ocp') - self.parser.add_option('--cluster_id', type='string', help='cluster_id from ocp') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherAwrCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_awr', self.opts) - - -class ObdiagGatherPlanMonitorCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information') - self.parser.add_option('--trace_id', type='string', help='sql trace id') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('--env', type='string', help='''env, eg: "{db_connect='-h127.0.0.1 -P2881 -utest@test -p****** -Dtest'}"''') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherPlanMonitorCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_plan_monitor', self.opts) - - -class ObdiagGatherObproxyLogCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherObproxyLogCommand, self).__init__('obproxy_log', 'Gather obproxy log from obproxy machines') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherObproxyLogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.gather_obproxy_log(self.opts) - - -class ObdiagGatherSceneListCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherSceneListCommand, self).__init__('list', 'gather scene list') - - def init(self, cmd, args): - super(ObdiagGatherSceneListCommand, self).init(cmd, args) - return self - - def _do_command(self, obdiag): - return obdiag.gather_scenes_list(self.opts) - - -class ObdiagGatherSceneRunCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherSceneRunCommand, self).__init__('run', 'gather scene run') - self.parser.add_option('--scene', type='string', help="Specify the scene to be gather") - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherSceneRunCommand, self).init(cmd, args) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_scenes_run', self.opts) - - -class ObdiagGatherAshReportCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherAshReportCommand, self).__init__('ash', 'Gather ash report') - self.parser.add_option('--trace_id', type='string', help="The TRACE.ID of the SQL to be sampled, if left blank or filled with NULL, indicates that TRACE.ID is not restricted.") - self.parser.add_option('--sql_id', type='string', help="The SQL.ID, if left blank or filled with NULL, indicates that SQL.ID is not restricted.") - # WAIT_CLASS - self.parser.add_option('--wait_class', type='string', help='Event types to be sampled.') - self.parser.add_option('--report_type', type='string', help='Report type, currently only supports text type.', default='TEXT') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherAshReportCommand, self).init(cmd, args) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_ash_report', self.opts) - - -class ObdiagAnalyzeLogCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagAnalyzeLogCommand, self).__init__('log', 'Analyze oceanbase log from online observer machines or offline oceanbase log files') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--log_level', type='string', help="oceanbase logs greater than or equal to this level will be analyze, choices=[DEBUG, TRACE, INFO, WDIAG, WARN, EDIAG, ERROR]") - self.parser.add_option('--files', action="append", type='string', help="specify files") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - - def init(self, cmd, args): - super(ObdiagAnalyzeLogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - offline_args_sign = '--files' - if self.args and (offline_args_sign in self.args): - return obdiag.analyze_fuction('analyze_log_offline', self.opts) - else: - return obdiag.analyze_fuction('analyze_log', self.opts) - - -class ObdiagAnalyzeFltTraceCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagAnalyzeFltTraceCommand, self).__init__('flt_trace', 'Analyze oceanbase trace.log from online observer machines or offline oceanbase trace.log files') - self.parser.add_option('--flt_trace_id', type='string', help="flt trace id, . format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") - self.parser.add_option('--files', action="append", help="specify files") - self.parser.add_option('--top', type='string', help="top leaf span", default=5) - self.parser.add_option('--recursion', type='string', help="Maximum number of recursion", default=8) - self.parser.add_option('--output', type='string', help="Print the result to the maximum output line on the screen", default=60) - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeFltTraceCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_flt_trace', self.opts) - - -class ObdiagAnalyzeParameterCommand(MajorCommand): - - def __init__(self): - super(ObdiagAnalyzeParameterCommand, self).__init__('parameters', 'Analyze parameters info') - self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) - self.register_command(ObdiagAnalyzeParameterDiffCommand()) - - -class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'list non-default value') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) - - -class ObdiagAnalyzeParameterDiffCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagAnalyzeParameterDiffCommand, self).__init__('diff', 'list inconsistent values among different observers') - self.parser.add_option('--file', type='string', help="specify parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeParameterDiffCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) - - -class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagAnalyzeVariableCommand, self).__init__('variables', 'Analyze oceanbase variable from online observer machines to offline oceanbase variable files') - self.parser.add_option('--file', type='string', help="specify variable file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_variable', self.opts) - - -class ObdiagCheckCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagCheckCommand, self).__init__('check', 'check oceanbase cluster') - self.parser.add_option('--cases', type='string', help="check observer's cases on package_file") - self.parser.add_option('--obproxy_cases', type='string', help="check obproxy's cases on package_file") - self.parser.add_option('--store_dir', type='string', help='the dir to store check result, current dir by default.', default='./check_report/') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagCheckCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - if 'list' in self.args: - obdiag.check_list(self.opts) - return - return obdiag.check(self.opts) - - -class ObdiagRCARunCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagRCARunCommand, self).__init__('run', 'root cause analysis') - self.parser.add_option('--scene', type='string', help="rca scene name. The argument is required.") - self.parser.add_option('--store_dir', type='string', help='the dir to store rca result, current dir by default.', default='./rca/') - self.parser.add_option('--input_parameters', type='string', help='input parameters of scene') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagRCARunCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.rca_run(self.opts) - - -class ObdiagRCAListCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagRCAListCommand, self).__init__('list', 'show list of rca list') - - def init(self, cmd, args): - super(ObdiagRCAListCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.rca_list(self.opts) - - -class ObdiagConfigCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagConfigCommand, self).__init__('config', 'Quick build config') - self.parser.add_option('-h', type='string', help="database host") - self.parser.add_option('-u', type='string', help='sys_user', default='root@sys') - self.parser.add_option('-p', type='string', help="password", default='') - self.parser.add_option('-P', type='string', help="port") - - def init(self, cmd, args): - super(ObdiagConfigCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.config(self.opts) - - -class ObdiagUpdateCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagUpdateCommand, self).__init__('update', 'Update cheat files') - self.parser.add_option('--file', type='string', help="obdiag update cheat file path. Please note that you need to ensure the reliability of the files on your own.") - self.parser.add_option( - '--force', - action='store_true', - help='You can force online upgrades by adding --force in the command', - ) - - def init(self, cmd, args): - super(ObdiagUpdateCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.update(self.opts) - - -class ObdiagGatherParametersCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherParametersCommand, self).__init__('parameters', 'Gather parameters info') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherParametersCommand, self).init(cmd, args) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_parameters', self.opts) - - -class ObdiagGatherVariablesCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagGatherVariablesCommand, self).__init__('variables', 'Gather variables info') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagGatherVariablesCommand, self).init(cmd, args) - return self - - def _do_command(self, obdiag): - return obdiag.gather_function('gather_variables', self.opts) - - -class ObdiagGatherCommand(MajorCommand): - - def __init__(self): - super(ObdiagGatherCommand, self).__init__('gather', 'Gather oceanbase diagnostic info') - self.register_command(ObdiagGatherAllCommand()) - self.register_command(ObdiagGatherLogCommand()) - self.register_command(ObdiagGatherSysStatCommand()) - self.register_command(ObdiagGatherStackCommand()) - self.register_command(ObdiagGatherPerfCommand()) - self.register_command(ObdiagGatherSlogCommand()) - self.register_command(ObdiagGatherClogCommand()) - self.register_command(ObdiagGatherPlanMonitorCommand()) - self.register_command(ObdiagGatherAwrCommand()) - self.register_command(ObdiagGatherObproxyLogCommand()) - self.register_command(ObdiagGatherSceneCommand()) - self.register_command(ObdiagGatherAshReportCommand()) - self.register_command(ObdiagGatherParametersCommand()) - self.register_command(ObdiagGatherVariablesCommand()) - - -class ObdiagGatherSceneCommand(MajorCommand): - - def __init__(self): - super(ObdiagGatherSceneCommand, self).__init__('scene', 'Gather scene diagnostic info') - self.register_command(ObdiagGatherSceneListCommand()) - self.register_command(ObdiagGatherSceneRunCommand()) - - -class ObdiagAnalyzeCommand(MajorCommand): - - def __init__(self): - super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info') - self.register_command(ObdiagAnalyzeLogCommand()) - self.register_command(ObdiagAnalyzeFltTraceCommand()) - self.register_command(ObdiagAnalyzeParameterCommand()) - self.register_command(ObdiagAnalyzeVariableCommand()) - - -class ObdiagRCACommand(MajorCommand): - - def __init__(self): - super(ObdiagRCACommand, self).__init__('rca', 'root cause analysis') - self.register_command(ObdiagRCARunCommand()) - self.register_command(ObdiagRCAListCommand()) - - -class MainCommand(MajorCommand): - - def __init__(self): - super(MainCommand, self).__init__('obdiag', '') - self.register_command(DisplayTraceCommand()) - self.register_command(ObdiagGatherCommand()) - self.register_command(ObdiagAnalyzeCommand()) - self.register_command(ObdiagCheckCommand()) - self.register_command(ObdiagRCACommand()) - self.register_command(ObdiagConfigCommand()) - self.register_command(ObdiagUpdateCommand()) - self.parser.version = get_obdiag_version() - self.parser._add_version_option() diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index 1edb8149..43fc8d32 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -89,7 +89,7 @@ def init_option(self): def alalyze_variable(self): sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() - from __all_virtual_sys_variable order by 2, 4, 5''' + from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' db_variable_info = self.obconn.execute_sql(sql) db_variable_dict = dict() for row in db_variable_info: diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index b30ebfe9..bec7463e 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -83,7 +83,7 @@ def get_version(self): def get_cluster_name(self): cluster_name = "" try: - sql = '''select value from __all_virtual_tenant_parameter_stat t2 where name = 'cluster' ''' + sql = '''select value from oceanbase.__all_virtual_tenant_parameter_stat t2 where name = 'cluster' ''' cluster_info = self.obconn.execute_sql(sql) cluster_name = cluster_info[0][0] except Exception as e: @@ -103,10 +103,10 @@ def get_parameters_info(self): EDIT_LEVEL, now(), '','' from GV$OB_PARAMETERS order by 5,2,3,4,7''' else: sql = '''select version(), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, - EDIT_LEVEL, now(), '','' from __all_virtual_tenant_parameter_info + EDIT_LEVEL, now(), '','' from oceanbase.__all_virtual_tenant_parameter_info union select version(), svr_ip,svr_port,zone,scope,'None' tenant_id,name,value,section, - EDIT_LEVEL, now(), '','' from __all_virtual_sys_parameter_stat where scope='CLUSTER' + EDIT_LEVEL, now(), '','' from oceanbase.__all_virtual_sys_parameter_stat where scope='CLUSTER' ''' parameter_info = self.obconn.execute_sql(sql) self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index b67a1cab..55c790ba 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -72,7 +72,7 @@ def init_option(self): def get_cluster_name(self): cluster_name = "" try: - sql = '''select value from __all_virtual_tenant_parameter_stat t2 where name = 'cluster' ''' + sql = '''select value from oceanbase.__all_virtual_tenant_parameter_stat t2 where name = 'cluster' ''' cluster_info = self.obconn.execute_sql(sql) cluster_name = cluster_info[0][0] except Exception as e: @@ -83,7 +83,7 @@ def get_cluster_name(self): def get_variables_info(self): cluster_name = self.get_cluster_name() sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() - from __all_virtual_sys_variable order by 2, 4, 5''' + from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' variable_info = self.obconn.execute_sql(sql) self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) with open(self.variable_file_name, 'w', newline='') as file: From 3e39f00ecfd7c89ed33781102a5452ee6f1cf98d Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Mon, 15 Jul 2024 17:29:22 +0800 Subject: [PATCH 05/12] =?UTF-8?q?CSV=E6=B7=BB=E5=8A=A0=E8=A1=A8=E5=A4=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handler/gather/gather_parameters.py | 17 +++++++++++++---- handler/gather/gather_variables.py | 3 +++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index bec7463e..ff9ac5a8 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -66,7 +66,8 @@ def init_option(self): store_dir_option = Util.get_option(options, 'store_dir') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format( + os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) return True @@ -109,9 +110,14 @@ def get_parameters_info(self): EDIT_LEVEL, now(), '','' from oceanbase.__all_virtual_sys_parameter_stat where scope='CLUSTER' ''' parameter_info = self.obconn.execute_sql(sql) - self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, + TimeUtils.timestamp_to_filename_time( + self.gather_timestamp)) + header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', + 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT'] with open(self.parameter_file_name, 'w', newline='') as file: writer = csv.writer(file) + writer.writerow(header) for row in parameter_info: if row[5] is None: tmp_row = [col for col in row] @@ -119,9 +125,12 @@ def get_parameters_info(self): writer.writerow(tmp_row) else: writer.writerow(row) - self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'") + self.stdio.print( + "Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format( + self.parameter_file_name) + Style.RESET_ALL + "'") else: - self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.") + self.stdio.warn( + "Failed to retrieve the database version. Please check if the database connection is normal.") def execute(self): try: diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index 55c790ba..82f090ff 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -86,8 +86,11 @@ def get_variables_info(self): from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' variable_info = self.obconn.execute_sql(sql) self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', + 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME'] with open(self.variable_file_name, 'w', newline='') as file: writer = csv.writer(file) + writer.writerow(header) for row in variable_info: writer.writerow(row) self.stdio.print("Gather variables finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.variable_file_name) + Style.RESET_ALL + "'") From c57f281d005b8490fe4de1935302a66f27c5fdb3 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Mon, 15 Jul 2024 18:28:13 +0800 Subject: [PATCH 06/12] obdiag gather and analyze parameters/variables --- core.py | 8 +-- diag_cmd.py | 73 +++++---------------------- handler/analyzer/analyze_parameter.py | 20 ++++---- handler/analyzer/analyze_variable.py | 7 +-- handler/gather/gather_parameters.py | 17 ++----- handler/gather/gather_variables.py | 3 +- 6 files changed, 36 insertions(+), 92 deletions(-) diff --git a/core.py b/core.py index ec26f1a7..738cd1c1 100644 --- a/core.py +++ b/core.py @@ -294,17 +294,17 @@ def analyze_fuction(self, function_type, opt): self.set_context(function_type, 'analyze', config) handler = AnalyzeFltTraceHandler(self.context) handler.handle() - elif function_type == 'analyze_parameter_non_default': + elif function_type == 'analyze_parameter_default': self.set_context(function_type, 'analyze', config) - handler = AnalyzeParameterHandler(self.context, 'non_default') + handler = AnalyzeParameterHandler(self.context, 'default') handler.handle() elif function_type == 'analyze_parameter_diff': self.set_context_skip_cluster_conn(function_type, 'analyze', config) handler = AnalyzeParameterHandler(self.context, 'diff') handler.handle() - elif function_type == 'analyze_variable': + elif function_type == 'analyze_variable_diff': self.set_context(function_type, 'analyze', config) - handler = AnalyzeVariableHandler(self.context) + handler = AnalyzeVariableHandler(self.context, 'diff') handler.handle() elif function_type == 'analyze_sql': self.set_context(function_type, 'analyze', config) diff --git a/diag_cmd.py b/diag_cmd.py index f2970dac..cc54650a 100644 --- a/diag_cmd.py +++ b/diag_cmd.py @@ -697,75 +697,49 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) -class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): +class ObdiagAnalyzeParameterDefaultCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values') + super(ObdiagAnalyzeParameterDefaultCommand, self).__init__('default', 'Analyze the parameter to identify parameters with non-default values') self.parser.add_option('--file', type='string', help="specify initialization parameter file") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) def init(self, cmd, args): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) + super(ObdiagAnalyzeParameterDefaultCommand, self).init(cmd, args) self.parser.set_usage('%s [options]' % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + return obdiag.analyze_fuction('analyze_parameter_default', self.opts) class ObdiagAnalyzeParameterCommand(MajorCommand): def __init__(self): super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') self.register_command(ObdiagAnalyzeParameterDiffCommand()) - self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) + self.register_command(ObdiagAnalyzeParameterDefaultCommand()) -class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): +class ObdiagAnalyzeVariableDiffCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') + super(ObdiagAnalyzeVariableDiffCommand, self).__init__('diff', 'Analyze and identify variables that have changed compared to the specified variable file') self.parser.add_option('--file', type='string', help="specify initialization parameter file") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) def init(self, cmd, args): - super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) + super(ObdiagAnalyzeVariableDiffCommand, self).init(cmd, args) self.parser.set_usage('%s [options]' % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_variable', self.opts) + return obdiag.analyze_fuction('analyze_variable_diff', self.opts) -class ObdiagAnalyzeParameterDiffCommand(ObdiagOriginCommand): - def __init__(self): - super(ObdiagAnalyzeParameterDiffCommand, self).__init__('diff', 'Analyze the parameter configurations between observers and identify the parameters with different values among the observers') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeParameterDiffCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) - - -class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): +class ObdiagAnalyzeVariableCommand(MajorCommand): def __init__(self): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze oceanbase variables info') + self.register_command(ObdiagAnalyzeVariableDiffCommand()) class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand): @@ -819,29 +793,6 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_sql_review', self.opts) -class ObdiagAnalyzeParameterCommand(MajorCommand): - def __init__(self): - super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') - self.register_command(ObdiagAnalyzeParameterDiffCommand()) - self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) - - -class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): - def __init__(self): - super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_variable', self.opts) - - class ObdiagCheckCommand(ObdiagOriginCommand): def __init__(self): diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 97ca13cc..475998d2 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -28,7 +28,7 @@ class AnalyzeParameterHandler(object): - def __init__(self, context, analyze_type='non_default'): + def __init__(self, context, analyze_type='default'): self.context = context self.stdio = self.context.stdio self.export_report_path = None @@ -64,8 +64,8 @@ def get_version(self): return observer_version def handle(self): - if self.analyze_type == 'non_default': - if not self.init_option_non_default(): + if self.analyze_type == 'default': + if not self.init_option_default(): self.stdio.error('init option failed') return False else: @@ -76,7 +76,7 @@ def handle(self): DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) self.execute() - def init_option_non_default(self): + def init_option_default(self): options = self.context.options store_dir_option = Util.get_option(options, 'store_dir') offline_file_option = Util.get_option(options, 'file') @@ -123,7 +123,7 @@ def init_option_diff(self): self.parameter_file_name = os.path.abspath(offline_file_option) return True - def analyze_parameter_non_default(self): + def analyze_parameter_default(self): observer_version = self.get_version() if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"): if self.parameter_file_name is not None: @@ -144,7 +144,7 @@ def analyze_parameter_non_default(self): report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]]) fp.write(report_default_tb.get_string() + "\n") self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") else: if self.parameter_file_name is None: self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values") @@ -179,9 +179,9 @@ def analyze_parameter_non_default(self): fp.write(report_default_tb.get_string() + "\n") if not is_empty: self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") else: - self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.") + self.stdio.print("Analyze parameter default finished. All parameter values are the same as the default values.") def alalyze_parameter_diff(self): if self.parameter_file_name is None: @@ -258,8 +258,8 @@ def alalyze_parameter_diff(self): def execute(self): try: - if self.analyze_type == 'non_default': - self.analyze_parameter_non_default() + if self.analyze_type == 'default': + self.analyze_parameter_default() elif self.analyze_type == 'diff': self.alalyze_parameter_diff() except Exception as e: diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index 43fc8d32..ac86050d 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -26,11 +26,12 @@ class AnalyzeVariableHandler(object): - def __init__(self, context): + def __init__(self, context, analyze_type='diff'): self.context = context self.stdio = self.context.stdio self.export_report_path = None self.variable_file_name = None + self.analyze_type = analyze_type self.ob_cluster = self.context.cluster_config if self.context.get_variable("gather_timestamp", None): self.analyze_timestamp = self.context.get_variable("gather_timestamp") @@ -87,7 +88,7 @@ def init_option(self): return True - def alalyze_variable(self): + def analyze_variable(self): sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' db_variable_info = self.obconn.execute_sql(sql) @@ -131,6 +132,6 @@ def alalyze_variable(self): def execute(self): try: - self.alalyze_variable() + self.analyze_variable() except Exception as e: self.stdio.error("variable info analyze failed, error message: {0}".format(e)) diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index ff9ac5a8..75bc4286 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -66,8 +66,7 @@ def init_option(self): store_dir_option = Util.get_option(options, 'store_dir') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format( - os.path.abspath(store_dir_option))) + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) return True @@ -110,11 +109,8 @@ def get_parameters_info(self): EDIT_LEVEL, now(), '','' from oceanbase.__all_virtual_sys_parameter_stat where scope='CLUSTER' ''' parameter_info = self.obconn.execute_sql(sql) - self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, - TimeUtils.timestamp_to_filename_time( - self.gather_timestamp)) - header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', - 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT'] + self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT'] with open(self.parameter_file_name, 'w', newline='') as file: writer = csv.writer(file) writer.writerow(header) @@ -125,12 +121,9 @@ def get_parameters_info(self): writer.writerow(tmp_row) else: writer.writerow(row) - self.stdio.print( - "Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format( - self.parameter_file_name) + Style.RESET_ALL + "'") + self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'") else: - self.stdio.warn( - "Failed to retrieve the database version. Please check if the database connection is normal.") + self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.") def execute(self): try: diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index 82f090ff..56f7ef20 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -86,8 +86,7 @@ def get_variables_info(self): from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' variable_info = self.obconn.execute_sql(sql) self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) - header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', - 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME'] + header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME'] with open(self.variable_file_name, 'w', newline='') as file: writer = csv.writer(file) writer.writerow(header) From c4630f66f7637ba71ba5d8c3c6b84d4479bd0591 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Tue, 16 Jul 2024 11:24:07 +0800 Subject: [PATCH 07/12] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=9C=89=E6=95=88=E6=80=A7=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handler/analyzer/analyze_parameter.py | 17 +++++++++++++++++ handler/analyzer/analyze_variable.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 475998d2..19aae52b 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -76,6 +76,22 @@ def handle(self): DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) self.execute() + def check_file_valid(self): + with open(self.parameter_file_name, 'r') as f: + header = f.readline() + flag = 1 + if header: + header = header.strip() + if not header: + flag = 0 + if not header.startswith('VERSION'): + flag = 0 + if not header.endswith('ISDEFAULT'): + flag = 0 + if flag == 0: + self.stdio.error('args --file [{0}] is not a valid parameter file, Please specify it again'.format(os.path.abspath(self.parameter_file_name))) + exit(-1) + def init_option_default(self): options = self.context.options store_dir_option = Util.get_option(options, 'store_dir') @@ -97,6 +113,7 @@ def init_option_default(self): exit(-1) else: self.parameter_file_name = os.path.abspath(offline_file_option) + self.check_file_valid() return True def init_option_diff(self): diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index ac86050d..ce0b2977 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -60,6 +60,22 @@ def handle(self): DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) self.execute() + def check_file_valid(self): + with open(self.variable_file_name, 'r') as f: + header = f.readline() + flag = 1 + if header: + header = header.strip() + if not header: + flag = 0 + if not header.startswith('VERSION'): + flag = 0 + if not header.endswith('RECORD_TIME'): + flag = 0 + if flag == 0: + self.stdio.error('args --file [{0}] is not a valid variable file, Please specify it again'.format(os.path.abspath(self.variable_file_name))) + exit(-1) + def init_option(self): options = self.context.options store_dir_option = Util.get_option(options, 'store_dir') @@ -70,6 +86,7 @@ def init_option(self): exit(-1) else: self.variable_file_name = os.path.abspath(offline_file_option) + self.check_file_valid() else: self.stdio.error("an initialization variable file must be provided to find the parts where variables have changed.") exit(-1) From ccb00912112eb19c6267dfb3cfc364f4e2db1f3c Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Tue, 16 Jul 2024 14:15:02 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=9C=89=E6=95=88=E6=80=A7=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handler/analyzer/analyze_parameter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 19aae52b..855bd9df 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -138,6 +138,7 @@ def init_option_diff(self): exit(-1) else: self.parameter_file_name = os.path.abspath(offline_file_option) + self.check_file_valid() return True def analyze_parameter_default(self): From 88b5647d718c3a88005cea2b15ac5fb966d4fb98 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Tue, 16 Jul 2024 14:45:06 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=E8=A1=A8=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handler/analyzer/analyze_parameter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 855bd9df..b9c9adb4 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -258,7 +258,10 @@ def alalyze_parameter_diff(self): if len(value_list) > 0: report_diff_tb = PrettyTable(["name", "diff"]) report_diff_tb.align["task_report"] = "l" - report_diff_tb.title = 'TENANT_ID:' + tenant + if tenant == 'CLUSTER': + report_diff_tb.title = 'SCOPE:' + tenant + else: + report_diff_tb.title = 'SCOPE:TENANT-' + tenant for value_dict in value_list: value_str_list = [] for value in value_dict['value_list']: From 10280f546689cddb7e4c2d15af28cb844f93632c Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Tue, 16 Jul 2024 15:12:04 +0800 Subject: [PATCH 10/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E9=A1=B9=E5=92=8C=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core.py | 22 ---------------------- handler/analyzer/analyze_parameter.py | 8 ++++---- handler/analyzer/analyze_variable.py | 2 +- handler/gather/gather_parameters.py | 10 +++++----- handler/gather/gather_variables.py | 4 ++-- 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/core.py b/core.py index 738cd1c1..73f8a956 100644 --- a/core.py +++ b/core.py @@ -32,8 +32,6 @@ from err import CheckStatus, SUG_SSH_FAILED from handler.analyzer.analyze_flt_trace import AnalyzeFltTraceHandler from handler.analyzer.analyze_log import AnalyzeLogHandler -from handler.analyzer.analyze_parameter import AnalyzeParameterHandler -from handler.analyzer.analyze_variable import AnalyzeVariableHandler from handler.analyzer.analyze_sql import AnalyzeSQLHandler from handler.analyzer.analyze_sql_review import AnalyzeSQLReviewHandler from handler.analyzer.analyze_parameter import AnalyzeParameterHandler @@ -50,8 +48,6 @@ from handler.gather.gather_plan_monitor import GatherPlanMonitorHandler from handler.gather.gather_scenes import GatherSceneHandler from handler.gather.scenes.list import GatherScenesListHandler -from handler.gather.gather_parameters import GatherParametersHandler -from handler.gather.gather_variables import GatherVariablesHandler from handler.gather.gather_tabledump import GatherTableDumpHandler from handler.gather.gather_parameters import GatherParametersHandler from handler.gather.gather_variables import GatherVariablesHandler @@ -250,12 +246,6 @@ def gather_function(self, function_type, opt): elif function_type == 'gather_variables': handler = GatherVariablesHandler(self.context) return handler.handle() - elif function_type == 'gather_parameters': - handler = GatherParametersHandler(self.context) - return handler.handle() - elif function_type == 'gather_variables': - handler = GatherVariablesHandler(self.context) - return handler.handle() else: self._call_stdio('error', 'Not support gather function: {0}'.format(function_type)) return False @@ -314,18 +304,6 @@ def analyze_fuction(self, function_type, opt): self.set_context(function_type, 'analyze', config) handler = AnalyzeSQLReviewHandler(self.context) handler.handle() - elif function_type == 'analyze_parameter_non_default': - self.set_context(function_type, 'analyze', config) - handler = AnalyzeParameterHandler(self.context, 'non_default') - handler.handle() - elif function_type == 'analyze_parameter_diff': - self.set_context_skip_cluster_conn(function_type, 'analyze', config) - handler = AnalyzeParameterHandler(self.context, 'diff') - handler.handle() - elif function_type == 'analyze_variable': - self.set_context(function_type, 'analyze', config) - handler = AnalyzeVariableHandler(self.context) - handler.handle() else: self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type)) return False diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index b9c9adb4..a31ebd38 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -59,8 +59,8 @@ def get_version(self): try: observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) except Exception as e: - self.stdio.warn("AnalyzeHandler Failed to get observer version:{0}".format(e)) - self.stdio.verbose("AnalyzeHandler.init get observer version: {0}".format(observer_version)) + self.stdio.warn("failed to get observer version:{0}".format(e)) + self.stdio.verbose("get observer version: {0}".format(observer_version)) return observer_version def handle(self): @@ -98,7 +98,7 @@ def init_option_default(self): offline_file_option = Util.get_option(options, 'file') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.export_report_path = os.path.abspath(store_dir_option) else: @@ -122,7 +122,7 @@ def init_option_diff(self): offline_file_option = Util.get_option(options, 'file') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.export_report_path = os.path.abspath(store_dir_option) else: diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index ce0b2977..4058c868 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -93,7 +93,7 @@ def init_option(self): if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.export_report_path = os.path.abspath(store_dir_option) else: diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index 75bc4286..187fb779 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -66,7 +66,7 @@ def init_option(self): store_dir_option = Util.get_option(options, 'store_dir') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) return True @@ -76,8 +76,8 @@ def get_version(self): try: observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) except Exception as e: - self.stdio.warn("GatherHandler Failed to get observer version:{0}".format(e)) - self.stdio.verbose("GatherHandler.init get observer version: {0}".format(observer_version)) + self.stdio.warn("failed to get observer version:{0}".format(e)) + self.stdio.verbose("get observer version: {0}".format(observer_version)) return observer_version def get_cluster_name(self): @@ -87,8 +87,8 @@ def get_cluster_name(self): cluster_info = self.obconn.execute_sql(sql) cluster_name = cluster_info[0][0] except Exception as e: - self.stdio.warn("RCAHandler Failed to get oceanbase cluster name:{0}".format(e)) - self.stdio.verbose("RCAHandler.init get oceanbase cluster name {0}".format(cluster_name)) + self.stdio.warn("failed to get oceanbase cluster name:{0}".format(e)) + self.stdio.verbose("get oceanbase cluster name {0}".format(cluster_name)) return cluster_name def get_parameters_info(self): diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index 56f7ef20..6c49b538 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -76,8 +76,8 @@ def get_cluster_name(self): cluster_info = self.obconn.execute_sql(sql) cluster_name = cluster_info[0][0] except Exception as e: - self.stdio.warn("RCAHandler Failed to get oceanbase cluster name:{0}".format(e)) - self.stdio.verbose("RCAHandler.init get oceanbase cluster name {0}".format(cluster_name)) + self.stdio.warn("failed to get oceanbase cluster name:{0}".format(e)) + self.stdio.verbose("get oceanbase cluster name {0}".format(cluster_name)) return cluster_name def get_variables_info(self): From b36d08a04a9b81be0a2647c17068602658354d71 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Tue, 16 Jul 2024 16:50:27 +0800 Subject: [PATCH 11/12] =?UTF-8?q?=E5=8F=98=E9=87=8F=E5=88=86=E6=9E=90?= =?UTF-8?q?=E4=B8=AD=E9=87=87=E9=9B=86=E6=97=B6=E9=97=B4=E6=98=BE=E7=A4=BA?= =?UTF-8?q?BUG=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handler/analyzer/analyze_parameter.py | 4 ++++ handler/analyzer/analyze_variable.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index a31ebd38..8dfa344c 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -180,6 +180,8 @@ def analyze_parameter_default(self): with open(self.parameter_file_name, 'r', newline='') as file: reader = csv.reader(file) for row in reader: + if row[0] == 'VERSION': + continue key = str(row[1]) + '-' + str(row[2]) + '-' + str(row[3]) + '-' + str(row[4]) + '-' + str(row[5]) + '-' + str(row[6]) value = row[7] file_parameter_dict[key] = value @@ -211,6 +213,8 @@ def alalyze_parameter_diff(self): with open(self.parameter_file_name, 'r', newline='') as file: reader = csv.reader(file) for row in reader: + if row[0] == 'VERSION': + continue parameter_info.append(row) tenants_dict = dict() for row in parameter_info: diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index 4058c868..0db30567 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -118,6 +118,8 @@ def analyze_variable(self): with open(self.variable_file_name, 'r', newline='') as file: reader = csv.reader(file) for row in reader: + if row[0] == 'VERSION': + continue key = str(row[1]) + '-' + str(row[3]) file_variable_dict[key] = str(row[5]) if not last_gather_time: From 97ea5495b24aedcf3dbbedb2feef898241be7206 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Tue, 16 Jul 2024 17:34:30 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E6=89=93=E5=8D=B0bug=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handler/analyzer/analyze_parameter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 8dfa344c..0cd4e58b 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -162,7 +162,7 @@ def analyze_parameter_default(self): report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]]) fp.write(report_default_tb.get_string() + "\n") self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL) else: if self.parameter_file_name is None: self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values")