Skip to content

Commit

Permalink
obdiag实现参数、变量采集和比对功能 (#329)
Browse files Browse the repository at this point in the history 
* obdiag gather and analyze parameters/variables

* obdiag gather and analyze parameters/variables

* obdiag gather and analyze parameters/variables

* obdiag gather and analyze parameters/variables

* CSV添加表头

* obdiag gather and analyze parameters/variables

* 添加文件有效性验证

* 添加文件有效性验证

* 修改参数范围表述

* 修改重复项和描述
  • Loading branch information
@oraclebird
oraclebird authored Jul 16, 2024
1 parent a8718bc commit 9c5031f
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 61 deletions.
24 changes: 12 additions & 12 deletions core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,25 +284,25 @@ def analyze_fuction(self, function_type, opt):
self.set_context(function_type, 'analyze', config)
handler = AnalyzeFltTraceHandler(self.context)
handler.handle()
elif function_type == 'analyze_sql':
self.set_context(function_type, 'analyze', config)
handler = AnalyzeSQLHandler(self.context)
handler.handle()
elif function_type == 'analyze_sql_review':
self.set_context(function_type, 'analyze', config)
handler = AnalyzeSQLReviewHandler(self.context)
handler.handle()
elif function_type == 'analyze_parameter_non_default':
elif function_type == 'analyze_parameter_default':
self.set_context(function_type, 'analyze', config)
handler = AnalyzeParameterHandler(self.context, 'non_default')
handler = AnalyzeParameterHandler(self.context, 'default')
handler.handle()
elif function_type == 'analyze_parameter_diff':
self.set_context_skip_cluster_conn(function_type, 'analyze', config)
handler = AnalyzeParameterHandler(self.context, 'diff')
handler.handle()
elif function_type == 'analyze_variable':
elif function_type == 'analyze_variable_diff':
self.set_context(function_type, 'analyze', config)
handler = AnalyzeVariableHandler(self.context)
handler = AnalyzeVariableHandler(self.context, 'diff')
handler.handle()
elif function_type == 'analyze_sql':
self.set_context(function_type, 'analyze', config)
handler = AnalyzeSQLHandler(self.context)
handler.handle()
elif function_type == 'analyze_sql_review':
self.set_context(function_type, 'analyze', config)
handler = AnalyzeSQLReviewHandler(self.context)
handler.handle()
else:
self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type))
Expand Down
64 changes: 37 additions & 27 deletions diag_cmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,20 +697,49 @@ def _do_command(self, obdiag):
return obdiag.analyze_fuction('analyze_parameter_diff', self.opts)


class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand):
class ObdiagAnalyzeParameterDefaultCommand(ObdiagOriginCommand):
def __init__(self):
super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values')
super(ObdiagAnalyzeParameterDefaultCommand, self).__init__('default', 'Analyze the parameter to identify parameters with non-default values')
self.parser.add_option('--file', type='string', help="specify initialization parameter file")
self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))

def init(self, cmd, args):
super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args)
super(ObdiagAnalyzeParameterDefaultCommand, self).init(cmd, args)
self.parser.set_usage('%s [options]' % self.prev_cmd)
return self

def _do_command(self, obdiag):
return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts)
return obdiag.analyze_fuction('analyze_parameter_default', self.opts)


class ObdiagAnalyzeParameterCommand(MajorCommand):
def __init__(self):
super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info')
self.register_command(ObdiagAnalyzeParameterDiffCommand())
self.register_command(ObdiagAnalyzeParameterDefaultCommand())


class ObdiagAnalyzeVariableDiffCommand(ObdiagOriginCommand):
def __init__(self):
super(ObdiagAnalyzeVariableDiffCommand, self).__init__('diff', 'Analyze and identify variables that have changed compared to the specified variable file')
self.parser.add_option('--file', type='string', help="specify initialization parameter file")
self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))

def init(self, cmd, args):
super(ObdiagAnalyzeVariableDiffCommand, self).init(cmd, args)
self.parser.set_usage('%s [options]' % self.prev_cmd)
return self

def _do_command(self, obdiag):
return obdiag.analyze_fuction('analyze_variable_diff', self.opts)


class ObdiagAnalyzeVariableCommand(MajorCommand):
def __init__(self):
super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze oceanbase variables info')
self.register_command(ObdiagAnalyzeVariableDiffCommand())


class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand):
Expand Down Expand Up @@ -764,29 +793,6 @@ def _do_command(self, obdiag):
return obdiag.analyze_fuction('analyze_sql_review', self.opts)


class ObdiagAnalyzeParameterCommand(MajorCommand):
def __init__(self):
super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info')
self.register_command(ObdiagAnalyzeParameterDiffCommand())
self.register_command(ObdiagAnalyzeParameterNonDefaultCommand())


class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand):
def __init__(self):
super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file')
self.parser.add_option('--file', type='string', help="specify initialization parameter file")
self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))

def init(self, cmd, args):
super(ObdiagAnalyzeVariableCommand, self).init(cmd, args)
self.parser.set_usage('%s [options]' % self.prev_cmd)
return self

def _do_command(self, obdiag):
return obdiag.analyze_fuction('analyze_variable', self.opts)


class ObdiagCheckCommand(ObdiagOriginCommand):

def __init__(self):
Expand Down Expand Up @@ -895,6 +901,8 @@ def __init__(self):
self.register_command(ObdiagGatherObproxyLogCommand())
self.register_command(ObdiagGatherSceneCommand())
self.register_command(ObdiagGatherAshReportCommand())
self.register_command(ObdiagGatherParameterCommand())
self.register_command(ObdiagGatherVariableCommand())
self.register_command(ObdiagGatherTableDumpHandler())
self.register_command(ObdiagGatherParameterCommand())
self.register_command(ObdiagGatherVariableCommand())
Expand All @@ -914,6 +922,8 @@ def __init__(self):
super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info')
self.register_command(ObdiagAnalyzeLogCommand())
self.register_command(ObdiagAnalyzeFltTraceCommand())
self.register_command(ObdiagAnalyzeParameterCommand())
self.register_command(ObdiagAnalyzeVariableCommand())
self.register_command(ObdiagAnalyzeSQLCommand())
self.register_command(ObdiagAnalyzeSQLReviewCommand())
self.register_command(ObdiagAnalyzeParameterCommand())
Expand Down
43 changes: 32 additions & 11 deletions handler/analyzer/analyze_parameter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@


class AnalyzeParameterHandler(object):
def __init__(self, context, analyze_type='non_default'):
def __init__(self, context, analyze_type='default'):
self.context = context
self.stdio = self.context.stdio
self.export_report_path = None
Expand Down Expand Up @@ -64,8 +64,8 @@ def get_version(self):
return observer_version

def handle(self):
if self.analyze_type == 'non_default':
if not self.init_option_non_default():
if self.analyze_type == 'default':
if not self.init_option_default():
self.stdio.error('init option failed')
return False
else:
Expand All @@ -76,7 +76,23 @@ def handle(self):
DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio)
self.execute()

def init_option_non_default(self):
def check_file_valid(self):
with open(self.parameter_file_name, 'r') as f:
header = f.readline()
flag = 1
if header:
header = header.strip()
if not header:
flag = 0
if not header.startswith('VERSION'):
flag = 0
if not header.endswith('ISDEFAULT'):
flag = 0
if flag == 0:
self.stdio.error('args --file [{0}] is not a valid parameter file, Please specify it again'.format(os.path.abspath(self.parameter_file_name)))
exit(-1)

def init_option_default(self):
options = self.context.options
store_dir_option = Util.get_option(options, 'store_dir')
offline_file_option = Util.get_option(options, 'file')
Expand All @@ -97,6 +113,7 @@ def init_option_non_default(self):
exit(-1)
else:
self.parameter_file_name = os.path.abspath(offline_file_option)
self.check_file_valid()
return True

def init_option_diff(self):
Expand All @@ -121,9 +138,10 @@ def init_option_diff(self):
exit(-1)
else:
self.parameter_file_name = os.path.abspath(offline_file_option)
self.check_file_valid()
return True

def analyze_parameter_non_default(self):
def analyze_parameter_default(self):
observer_version = self.get_version()
if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"):
if self.parameter_file_name is not None:
Expand All @@ -144,7 +162,7 @@ def analyze_parameter_non_default(self):
report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]])
fp.write(report_default_tb.get_string() + "\n")
self.stdio.print(report_default_tb.get_string())
self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'")
self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'")
else:
if self.parameter_file_name is None:
self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values")
Expand Down Expand Up @@ -179,9 +197,9 @@ def analyze_parameter_non_default(self):
fp.write(report_default_tb.get_string() + "\n")
if not is_empty:
self.stdio.print(report_default_tb.get_string())
self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
else:
self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.")
self.stdio.print("Analyze parameter default finished. All parameter values are the same as the default values.")

def alalyze_parameter_diff(self):
if self.parameter_file_name is None:
Expand Down Expand Up @@ -240,7 +258,10 @@ def alalyze_parameter_diff(self):
if len(value_list) > 0:
report_diff_tb = PrettyTable(["name", "diff"])
report_diff_tb.align["task_report"] = "l"
report_diff_tb.title = 'TENANT_ID:' + tenant
if tenant == 'CLUSTER':
report_diff_tb.title = 'SCOPE:' + tenant
else:
report_diff_tb.title = 'SCOPE:TENANT-' + tenant
for value_dict in value_list:
value_str_list = []
for value in value_dict['value_list']:
Expand All @@ -258,8 +279,8 @@ def alalyze_parameter_diff(self):

def execute(self):
try:
if self.analyze_type == 'non_default':
self.analyze_parameter_non_default()
if self.analyze_type == 'default':
self.analyze_parameter_default()
elif self.analyze_type == 'diff':
self.alalyze_parameter_diff()
except Exception as e:
Expand Down
28 changes: 23 additions & 5 deletions handler/analyzer/analyze_variable.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@


class AnalyzeVariableHandler(object):
def __init__(self, context):
def __init__(self, context, analyze_type='diff'):
self.context = context
self.stdio = self.context.stdio
self.export_report_path = None
self.variable_file_name = None
self.analyze_type = analyze_type
self.ob_cluster = self.context.cluster_config
if self.context.get_variable("gather_timestamp", None):
self.analyze_timestamp = self.context.get_variable("gather_timestamp")
Expand All @@ -48,8 +49,8 @@ def __init__(self, context):
database="oceanbase",
)
except Exception as e:
self.stdio.error("failed to connect to database: {0}".format(e))
raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
self.stdio.error("Failed to connect to database: {0}".format(e))
raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))

def handle(self):
if not self.init_option():
Expand All @@ -59,6 +60,22 @@ def handle(self):
DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio)
self.execute()

def check_file_valid(self):
with open(self.variable_file_name, 'r') as f:
header = f.readline()
flag = 1
if header:
header = header.strip()
if not header:
flag = 0
if not header.startswith('VERSION'):
flag = 0
if not header.endswith('RECORD_TIME'):
flag = 0
if flag == 0:
self.stdio.error('args --file [{0}] is not a valid variable file, Please specify it again'.format(os.path.abspath(self.variable_file_name)))
exit(-1)

def init_option(self):
options = self.context.options
store_dir_option = Util.get_option(options, 'store_dir')
Expand All @@ -69,6 +86,7 @@ def init_option(self):
exit(-1)
else:
self.variable_file_name = os.path.abspath(offline_file_option)
self.check_file_valid()
else:
self.stdio.error("an initialization variable file must be provided to find the parts where variables have changed.")
exit(-1)
Expand All @@ -87,7 +105,7 @@ def init_option(self):

return True

def alalyze_variable(self):
def analyze_variable(self):
sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now()
from oceanbase.__all_virtual_sys_variable order by 2, 4, 5'''
db_variable_info = self.obconn.execute_sql(sql)
Expand Down Expand Up @@ -131,6 +149,6 @@ def alalyze_variable(self):

def execute(self):
try:
self.alalyze_variable()
self.analyze_variable()
except Exception as e:
self.stdio.error("variable info analyze failed, error message: {0}".format(e))
8 changes: 5 additions & 3 deletions handler/gather/gather_parameters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ def __init__(self, context, gather_pack_dir='./'):
database="oceanbase",
)
except Exception as e:
self.stdio.error("failed to connect to database: {0}".format(e))
raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
self.stdio.error("Failed to connect to database: {0}".format(e))
raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))

def handle(self):
if not self.init_option():
Expand Down Expand Up @@ -110,8 +110,10 @@ def get_parameters_info(self):
'''
parameter_info = self.obconn.execute_sql(sql)
self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp))
header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT']
with open(self.parameter_file_name, 'w', newline='') as file:
writer = csv.writer(file)
writer.writerow(header)
for row in parameter_info:
if row[5] is None:
tmp_row = [col for col in row]
Expand All @@ -121,7 +123,7 @@ def get_parameters_info(self):
writer.writerow(row)
self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'")
else:
self.stdio.warn("failed to retrieve the database version. Please check if the database connection is normal.")
self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.")

def execute(self):
try:
Expand Down
8 changes: 5 additions & 3 deletions handler/gather/gather_variables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ def __init__(self, context, gather_pack_dir='./'):
database="oceanbase",
)
except Exception as e:
self.stdio.error("failed to connect to database: {0}".format(e))
raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
self.stdio.error("Failed to connect to database: {0}".format(e))
raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))

def handle(self):
if not self.init_option():
Expand All @@ -64,7 +64,7 @@ def init_option(self):
store_dir_option = Util.get_option(options, 'store_dir')
if store_dir_option and store_dir_option != "./":
if not os.path.exists(os.path.abspath(store_dir_option)):
self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
os.makedirs(os.path.abspath(store_dir_option))
self.gather_pack_dir = os.path.abspath(store_dir_option)
return True
Expand All @@ -86,8 +86,10 @@ def get_variables_info(self):
from oceanbase.__all_virtual_sys_variable order by 2, 4, 5'''
variable_info = self.obconn.execute_sql(sql)
self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp))
header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME']
with open(self.variable_file_name, 'w', newline='') as file:
writer = csv.writer(file)
writer.writerow(header)
for row in variable_info:
writer.writerow(row)
self.stdio.print("Gather variables finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.variable_file_name) + Style.RESET_ALL + "'")
Expand Down

0 comments on commit 9c5031f

Please sign in to comment.