obdiag实现参数、变量采集和比对功能

core.py

@@ -284,25 +284,25 @@ def analyze_fuction(self, function_type, opt):
                 self.set_context(function_type, 'analyze', config)
                 handler = AnalyzeFltTraceHandler(self.context)
                 handler.handle()
-            elif function_type == 'analyze_sql':
-                self.set_context(function_type, 'analyze', config)
-                handler = AnalyzeSQLHandler(self.context)
-                handler.handle()
-            elif function_type == 'analyze_sql_review':
-                self.set_context(function_type, 'analyze', config)
-                handler = AnalyzeSQLReviewHandler(self.context)
-                handler.handle()
-            elif function_type == 'analyze_parameter_non_default':
+            elif function_type == 'analyze_parameter_default':
                 self.set_context(function_type, 'analyze', config)
-                handler = AnalyzeParameterHandler(self.context, 'non_default')
+                handler = AnalyzeParameterHandler(self.context, 'default')
                 handler.handle()
             elif function_type == 'analyze_parameter_diff':
                 self.set_context_skip_cluster_conn(function_type, 'analyze', config)
                 handler = AnalyzeParameterHandler(self.context, 'diff')
                 handler.handle()
-            elif function_type == 'analyze_variable':
+            elif function_type == 'analyze_variable_diff':
                 self.set_context(function_type, 'analyze', config)
-                handler = AnalyzeVariableHandler(self.context)
+                handler = AnalyzeVariableHandler(self.context, 'diff')
+                handler.handle()
+            elif function_type == 'analyze_sql':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeSQLHandler(self.context)
+                handler.handle()
+            elif function_type == 'analyze_sql_review':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeSQLReviewHandler(self.context)
                 handler.handle()
             else:
                 self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type))

diag_cmd.py

@@ -697,20 +697,49 @@ def _do_command(self, obdiag):
         return obdiag.analyze_fuction('analyze_parameter_diff', self.opts)
 
 
-class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand):
+class ObdiagAnalyzeParameterDefaultCommand(ObdiagOriginCommand):
     def __init__(self):
-        super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values')
+        super(ObdiagAnalyzeParameterDefaultCommand, self).__init__('default', 'Analyze the parameter to identify parameters with non-default values')
         self.parser.add_option('--file', type='string', help="specify initialization parameter file")
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
 
     def init(self, cmd, args):
-        super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args)
+        super(ObdiagAnalyzeParameterDefaultCommand, self).init(cmd, args)
         self.parser.set_usage('%s [options]' % self.prev_cmd)
         return self
 
     def _do_command(self, obdiag):
-        return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts)
+        return obdiag.analyze_fuction('analyze_parameter_default', self.opts)
+
+
+class ObdiagAnalyzeParameterCommand(MajorCommand):
+    def __init__(self):
+        super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info')
+        self.register_command(ObdiagAnalyzeParameterDiffCommand())
+        self.register_command(ObdiagAnalyzeParameterDefaultCommand())
+
+
+class ObdiagAnalyzeVariableDiffCommand(ObdiagOriginCommand):
+    def __init__(self):
+        super(ObdiagAnalyzeVariableDiffCommand, self).__init__('diff', 'Analyze and identify variables that have changed compared to the specified variable file')
+        self.parser.add_option('--file', type='string', help="specify initialization parameter file")
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
+
+    def init(self, cmd, args):
+        super(ObdiagAnalyzeVariableDiffCommand, self).init(cmd, args)
+        self.parser.set_usage('%s [options]' % self.prev_cmd)
+        return self
+
+    def _do_command(self, obdiag):
+        return obdiag.analyze_fuction('analyze_variable_diff', self.opts)
+
+
+class ObdiagAnalyzeVariableCommand(MajorCommand):
+    def __init__(self):
+        super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze oceanbase variables info')
+        self.register_command(ObdiagAnalyzeVariableDiffCommand())
 
 
 class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand):
@@ -764,29 +793,6 @@ def _do_command(self, obdiag):
         return obdiag.analyze_fuction('analyze_sql_review', self.opts)
 
 
-class ObdiagAnalyzeParameterCommand(MajorCommand):
-    def __init__(self):
-        super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info')
-        self.register_command(ObdiagAnalyzeParameterDiffCommand())
-        self.register_command(ObdiagAnalyzeParameterNonDefaultCommand())
-
-
-class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand):
-    def __init__(self):
-        super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file')
-        self.parser.add_option('--file', type='string', help="specify initialization parameter file")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
-        self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
-
-    def init(self, cmd, args):
-        super(ObdiagAnalyzeVariableCommand, self).init(cmd, args)
-        self.parser.set_usage('%s [options]' % self.prev_cmd)
-        return self
-
-    def _do_command(self, obdiag):
-        return obdiag.analyze_fuction('analyze_variable', self.opts)
-
-
 class ObdiagCheckCommand(ObdiagOriginCommand):
 
     def __init__(self):
@@ -895,6 +901,8 @@ def __init__(self):
         self.register_command(ObdiagGatherObproxyLogCommand())
         self.register_command(ObdiagGatherSceneCommand())
         self.register_command(ObdiagGatherAshReportCommand())
+        self.register_command(ObdiagGatherParameterCommand())
+        self.register_command(ObdiagGatherVariableCommand())
         self.register_command(ObdiagGatherTableDumpHandler())
         self.register_command(ObdiagGatherParameterCommand())
         self.register_command(ObdiagGatherVariableCommand())
@@ -914,6 +922,8 @@ def __init__(self):
         super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info')
         self.register_command(ObdiagAnalyzeLogCommand())
         self.register_command(ObdiagAnalyzeFltTraceCommand())
+        self.register_command(ObdiagAnalyzeParameterCommand())
+        self.register_command(ObdiagAnalyzeVariableCommand())
         self.register_command(ObdiagAnalyzeSQLCommand())
         self.register_command(ObdiagAnalyzeSQLReviewCommand())
         self.register_command(ObdiagAnalyzeParameterCommand())

handler/analyzer/analyze_parameter.py

@@ -28,7 +28,7 @@
 
 
 class AnalyzeParameterHandler(object):
-    def __init__(self, context, analyze_type='non_default'):
+    def __init__(self, context, analyze_type='default'):
         self.context = context
         self.stdio = self.context.stdio
         self.export_report_path = None
@@ -64,8 +64,8 @@ def get_version(self):
         return observer_version
 
     def handle(self):
-        if self.analyze_type == 'non_default':
-            if not self.init_option_non_default():
+        if self.analyze_type == 'default':
+            if not self.init_option_default():
                 self.stdio.error('init option failed')
                 return False
         else:
@@ -76,7 +76,23 @@ def handle(self):
         DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio)
         self.execute()
 
-    def init_option_non_default(self):
+    def check_file_valid(self):
+        with open(self.parameter_file_name, 'r') as f:
+            header = f.readline()
+            flag = 1
+            if header:
+                header = header.strip()
+            if not header:
+                flag = 0
+            if not header.startswith('VERSION'):
+                flag = 0
+            if not header.endswith('ISDEFAULT'):
+                flag = 0
+            if flag == 0:
+                self.stdio.error('args --file [{0}] is not a valid parameter file, Please specify it again'.format(os.path.abspath(self.parameter_file_name)))
+                exit(-1)
+
+    def init_option_default(self):
         options = self.context.options
         store_dir_option = Util.get_option(options, 'store_dir')
         offline_file_option = Util.get_option(options, 'file')
@@ -97,6 +113,7 @@ def init_option_non_default(self):
                 exit(-1)
             else:
                 self.parameter_file_name = os.path.abspath(offline_file_option)
+                self.check_file_valid()
         return True
 
     def init_option_diff(self):
@@ -121,9 +138,10 @@ def init_option_diff(self):
                 exit(-1)
             else:
                 self.parameter_file_name = os.path.abspath(offline_file_option)
+                self.check_file_valid()
         return True
 
-    def analyze_parameter_non_default(self):
+    def analyze_parameter_default(self):
         observer_version = self.get_version()
         if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"):
             if self.parameter_file_name is not None:
@@ -144,7 +162,7 @@ def analyze_parameter_non_default(self):
                 report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]])
             fp.write(report_default_tb.get_string() + "\n")
             self.stdio.print(report_default_tb.get_string())
-            self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'")
+            self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'")
         else:
             if self.parameter_file_name is None:
                 self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values")
@@ -179,9 +197,9 @@ def analyze_parameter_non_default(self):
                 fp.write(report_default_tb.get_string() + "\n")
                 if not is_empty:
                     self.stdio.print(report_default_tb.get_string())
-                    self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
+                    self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
                 else:
-                    self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.")
+                    self.stdio.print("Analyze parameter default finished. All parameter values are the same as the default values.")
 
     def alalyze_parameter_diff(self):
         if self.parameter_file_name is None:
@@ -240,7 +258,10 @@ def alalyze_parameter_diff(self):
             if len(value_list) > 0:
                 report_diff_tb = PrettyTable(["name", "diff"])
                 report_diff_tb.align["task_report"] = "l"
-                report_diff_tb.title = 'TENANT_ID:' + tenant
+                if tenant == 'CLUSTER':
+                    report_diff_tb.title = 'SCOPE:' + tenant
+                else:
+                    report_diff_tb.title = 'SCOPE:TENANT-' + tenant
                 for value_dict in value_list:
                     value_str_list = []
                     for value in value_dict['value_list']:
@@ -258,8 +279,8 @@ def alalyze_parameter_diff(self):
 
     def execute(self):
         try:
-            if self.analyze_type == 'non_default':
-                self.analyze_parameter_non_default()
+            if self.analyze_type == 'default':
+                self.analyze_parameter_default()
             elif self.analyze_type == 'diff':
                 self.alalyze_parameter_diff()
         except Exception as e:

handler/analyzer/analyze_variable.py

@@ -26,11 +26,12 @@
 
 
 class AnalyzeVariableHandler(object):
-    def __init__(self, context):
+    def __init__(self, context, analyze_type='diff'):
         self.context = context
         self.stdio = self.context.stdio
         self.export_report_path = None
         self.variable_file_name = None
+        self.analyze_type = analyze_type
         self.ob_cluster = self.context.cluster_config
         if self.context.get_variable("gather_timestamp", None):
             self.analyze_timestamp = self.context.get_variable("gather_timestamp")
@@ -48,8 +49,8 @@ def __init__(self, context):
                 database="oceanbase",
             )
         except Exception as e:
-            self.stdio.error("failed to connect to database: {0}".format(e))
-            raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
+            self.stdio.error("Failed to connect to database: {0}".format(e))
+            raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))
 
     def handle(self):
         if not self.init_option():
@@ -59,6 +60,22 @@ def handle(self):
         DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio)
         self.execute()
 
+    def check_file_valid(self):
+        with open(self.variable_file_name, 'r') as f:
+            header = f.readline()
+            flag = 1
+            if header:
+                header = header.strip()
+            if not header:
+                flag = 0
+            if not header.startswith('VERSION'):
+                flag = 0
+            if not header.endswith('RECORD_TIME'):
+                flag = 0
+            if flag == 0:
+                self.stdio.error('args --file [{0}] is not a valid variable file, Please specify it again'.format(os.path.abspath(self.variable_file_name)))
+                exit(-1)
+
     def init_option(self):
         options = self.context.options
         store_dir_option = Util.get_option(options, 'store_dir')
@@ -69,6 +86,7 @@ def init_option(self):
                 exit(-1)
             else:
                 self.variable_file_name = os.path.abspath(offline_file_option)
+                self.check_file_valid()
         else:
             self.stdio.error("an initialization variable file must be provided to find the parts where variables have changed.")
             exit(-1)
@@ -87,7 +105,7 @@ def init_option(self):
 
         return True
 
-    def alalyze_variable(self):
+    def analyze_variable(self):
         sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() 
         from oceanbase.__all_virtual_sys_variable order by 2, 4, 5'''
         db_variable_info = self.obconn.execute_sql(sql)
@@ -131,6 +149,6 @@ def alalyze_variable(self):
 
     def execute(self):
         try:
-            self.alalyze_variable()
+            self.analyze_variable()
         except Exception as e:
             self.stdio.error("variable info analyze failed, error message: {0}".format(e))

handler/gather/gather_parameters.py

@@ -47,8 +47,8 @@ def __init__(self, context, gather_pack_dir='./'):
                 database="oceanbase",
             )
         except Exception as e:
-            self.stdio.error("failed to connect to database: {0}".format(e))
-            raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
+            self.stdio.error("Failed to connect to database: {0}".format(e))
+            raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))
 
     def handle(self):
         if not self.init_option():
@@ -110,8 +110,10 @@ def get_parameters_info(self):
 '''
             parameter_info = self.obconn.execute_sql(sql)
             self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp))
+            header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT']
             with open(self.parameter_file_name, 'w', newline='') as file:
                 writer = csv.writer(file)
+                writer.writerow(header)
                 for row in parameter_info:
                     if row[5] is None:
                         tmp_row = [col for col in row]
@@ -121,7 +123,7 @@ def get_parameters_info(self):
                         writer.writerow(row)
             self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'")
         else:
-            self.stdio.warn("failed to retrieve the database version. Please check if the database connection is normal.")
+            self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.")
 
     def execute(self):
         try:

handler/gather/gather_variables.py

@@ -46,8 +46,8 @@ def __init__(self, context, gather_pack_dir='./'):
                 database="oceanbase",
             )
         except Exception as e:
-            self.stdio.error("failed to connect to database: {0}".format(e))
-            raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
+            self.stdio.error("Failed to connect to database: {0}".format(e))
+            raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))
 
     def handle(self):
         if not self.init_option():
@@ -64,7 +64,7 @@ def init_option(self):
         store_dir_option = Util.get_option(options, 'store_dir')
         if store_dir_option and store_dir_option != "./":
             if not os.path.exists(os.path.abspath(store_dir_option)):
-                self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
+                self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
                 os.makedirs(os.path.abspath(store_dir_option))
         self.gather_pack_dir = os.path.abspath(store_dir_option)
         return True
@@ -86,8 +86,10 @@ def get_variables_info(self):
  from oceanbase.__all_virtual_sys_variable order by 2, 4, 5'''
         variable_info = self.obconn.execute_sql(sql)
         self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp))
+        header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME']
         with open(self.variable_file_name, 'w', newline='') as file:
             writer = csv.writer(file)
+            writer.writerow(header)
             for row in variable_info:
                 writer.writerow(row)
         self.stdio.print("Gather variables finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.variable_file_name) + Style.RESET_ALL + "'")