From 6138999cc25a442fb212d22682901f2de3111caa Mon Sep 17 00:00:00 2001 From: xuyan wang <35394786+wayyoungboy@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:10:13 +0800 Subject: [PATCH] 2.4.0 silent print (#388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update io: add stderr * update io: add stderr * add --inner_config add print_type * support remote_client_sudo * build test package * fix --inner_config * fix io stream * test * test * test * test * test * 取消测试分支打包 * test * test * test * test * test * add new io buffer * add new io buffer * build test package * silent change * delete init.sh cp gather dir more * update * update * update * update * update * update * update * update * fix some bug * delete build tag * delete silent note * code fix --- common/command.py | 16 ++-- common/config_helper.py | 2 +- common/ssh_client/base.py | 1 - common/ssh_client/remote_client.py | 1 - common/tool.py | 16 ++-- conf/inner_config.yml | 1 + config.py | 1 + core.py | 68 ++++++++------ diag_cmd.py | 45 ++++++++-- handler/analyzer/analyze_flt_trace.py | 18 +++- handler/analyzer/analyze_log.py | 11 ++- handler/analyzer/analyze_parameter.py | 19 ++-- handler/analyzer/analyze_sql.py | 9 +- handler/analyzer/analyze_variable.py | 10 ++- handler/checker/check_handler.py | 3 +- handler/checker/check_list.py | 86 ++++++++++-------- .../checker/tasks/observer/bugs/bug_385.yaml | 14 +++ .../tasks/observer/cluster/task_opt_stat.yaml | 16 ++++ .../tasks/observer/system/parameter.yaml | 2 +- handler/gather/gather_ash_report.py | 8 +- handler/gather/gather_awr.py | 4 +- handler/gather/gather_log.py | 3 +- handler/gather/gather_obadmin.py | 6 +- handler/gather/gather_obproxy_log.py | 7 +- handler/gather/gather_obstack2.py | 6 +- handler/gather/gather_parameters.py | 5 +- handler/gather/gather_perf.py | 6 +- handler/gather/gather_plan_monitor.py | 8 +- handler/gather/gather_scenes.py | 10 ++- handler/gather/gather_sysstat.py | 6 +- handler/gather/gather_tabledump.py | 7 +- handler/gather/gather_variables.py | 5 +- handler/gather/scenes/list.py | 23 +++-- handler/rca/rca_handler.py | 13 ++- handler/rca/rca_list.py | 6 +- handler/rca/scene/lock_conflict_scene.py | 4 +- init.sh | 2 +- result_type.py | 45 ++++++++++ stdio.py | 90 ++++++++++++++----- update/update.py | 29 +++--- 40 files changed, 449 insertions(+), 183 deletions(-) create mode 100644 handler/checker/tasks/observer/bugs/bug_385.yaml create mode 100644 handler/checker/tasks/observer/cluster/task_opt_stat.yaml create mode 100644 result_type.py diff --git a/common/command.py b/common/command.py index 915d0f60..8d838198 100644 --- a/common/command.py +++ b/common/command.py @@ -286,7 +286,8 @@ def get_obproxy_version(context): obproxy_version_info = ssh_client.exec_cmd(cmd) stdio.verbose("get obproxy version, run cmd = [{0}] ".format(cmd)) if obproxy_version_info is not None: - ob_version = re.findall(r'[(]OceanBase.(.+? +?)[)]', obproxy_version_info) + pattern = r"(\d+\.\d+\.\d+\.\d+)" + ob_version = re.findall(pattern, obproxy_version_info) if len(ob_version) > 0: return ob_version[0] else: @@ -295,7 +296,6 @@ def get_obproxy_version(context): stdio.verbose("get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format(cmd, obproxy_version_info)) if "REVISION" not in obproxy_version_info: raise Exception("Please check conf about proxy,{0}".format(obproxy_version_info)) - pattern = r"(\d+\.\d+\.\d+\.\d+)" match = re.search(pattern, obproxy_version_info) if match: obproxy_version_info = match.group(1) @@ -405,12 +405,12 @@ def is_empty_file(ssh_client, file_path, stdio=None): return False -def get_obdiag_display(log_dir, trace_id, stdio=None): - cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id) - stdout = LocalClient(stdio).run(cmd) - print_stdout = str(stdout).replace('\\n', '\n').replace('\\t', '\t') - if len(print_stdout) > 0: - print(print_stdout) +# def get_obdiag_display(log_dir, trace_id, stdio=None): +# cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id) +# stdout = LocalClient(stdio).run(cmd) +# print_stdout = str(stdout).replace('\\n', '\n').replace('\\t', '\t') +# if len(print_stdout) > 0: +# print(print_stdout) def uzip_dir_local(uzip_dir, stdio=None): diff --git a/common/config_helper.py b/common/config_helper.py index 4fcc55ed..a27e5ccd 100644 --- a/common/config_helper.py +++ b/common/config_helper.py @@ -98,7 +98,7 @@ def build_configuration(self): self.save_old_configuration(old_config) # rewrite config ob_cluster_name = self.get_cluster_name() - print("\033[33mPlease enter the following configuration !!!\033[0m") + self.stdio.print("\033[33mPlease enter the following configuration !!!\033[0m") global_ssh_username = self.input_with_default("oceanbase host ssh username", "") global_ssh_password = self.input_password_with_default("oceanbase host ssh password", "") global_ssh_port = self.input_with_default("oceanbase host ssh_port", "22") diff --git a/common/ssh_client/base.py b/common/ssh_client/base.py index 870f73b5..16021e7c 100644 --- a/common/ssh_client/base.py +++ b/common/ssh_client/base.py @@ -62,7 +62,6 @@ def progress_bar(self, transferred, to_be_transferred, suffix=''): sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) if transferred == to_be_transferred: sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) - print() def translate_byte(self, B): if B < 0: diff --git a/common/ssh_client/remote_client.py b/common/ssh_client/remote_client.py index c17c874b..d61c0821 100644 --- a/common/ssh_client/remote_client.py +++ b/common/ssh_client/remote_client.py @@ -113,7 +113,6 @@ def progress_bar(self, transferred, to_be_transferred, suffix=''): sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) if transferred == to_be_transferred: sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) - print() def upload(self, remote_path, local_path): transport = self._ssh_fd.get_transport() diff --git a/common/tool.py b/common/tool.py index 0e9921a2..0ab14007 100644 --- a/common/tool.py +++ b/common/tool.py @@ -1358,26 +1358,26 @@ def convert_to_number(s, stdio=None): return s @staticmethod - def print_scene(scene_dict, stdio=None): + def print_scene(scene_dict, stdio): columns_to_print = ['command', 'info_en', 'info_cn'] keys = columns_to_print table_data = [[value[key] for key in keys] for value in scene_dict.values()] column_widths = [max(len(str(item)) * (StringUtils.is_chinese(item) or 1) for item in column) for column in zip(*table_data)] table_data.insert(0, keys) - Util.print_line(length=sum(column_widths) + 5) + Util.print_line(length=sum(column_widths) + 5, stdio=stdio) for i in range(len(table_data)): - print(Fore.GREEN + " ".join(f"{item:<{width}}" for item, width in zip(table_data[i], column_widths)) + Style.RESET_ALL) + stdio.print(Fore.GREEN + " ".join(f"{item:<{width}}" for item, width in zip(table_data[i], column_widths)) + Style.RESET_ALL) if i == 0: - Util.print_line(length=sum(column_widths) + 5) - Util.print_line(length=sum(column_widths) + 5) + Util.print_line(length=sum(column_widths) + 5, stdio=stdio) + Util.print_line(length=sum(column_widths) + 5, stdio=stdio) @staticmethod def print_line(char='-', length=50, stdio=None): - print(char * length) + stdio.print(char * length) @staticmethod - def print_title(name, stdio=None): - print("\n[{0}]:".format(name)) + def print_title(name, stdio): + stdio.print("\n[{0}]:".format(name)) @staticmethod def gen_password(length=8, chars=string.ascii_letters + string.digits, stdio=None): diff --git a/conf/inner_config.yml b/conf/inner_config.yml index db4aa329..f241ba73 100644 --- a/conf/inner_config.yml +++ b/conf/inner_config.yml @@ -13,6 +13,7 @@ obdiag: mode: obdiag stdout_handler_log_level: INFO error_stream: sys.stdout + silent: false ssh_client: remote_client_sudo: 0 check: diff --git a/config.py b/config.py index cc2fc19f..bd32cbbe 100644 --- a/config.py +++ b/config.py @@ -74,6 +74,7 @@ 'mode': 'obdiag', 'stdout_handler_log_level': 'INFO', 'error_stream': 'sys.stdout', + 'silent': False, }, 'ssh_client': { 'remote_client_sudo': False, diff --git a/core.py b/core.py index 3a7d996c..95cb1206 100644 --- a/core.py +++ b/core.py @@ -51,6 +51,7 @@ from handler.gather.gather_tabledump import GatherTableDumpHandler from handler.gather.gather_parameters import GatherParametersHandler from handler.gather.gather_variables import GatherVariablesHandler +from result_type import ObdiagResult from telemetry.telemetry import telemetry from update.update import UpdateHandler from colorama import Fore, Style @@ -72,10 +73,19 @@ def __init__(self, stdio=None, config_path=os.path.expanduser('~/.obdiag/config. self.set_stdio(stdio) self.context = None self.inner_config_manager = InnerConfigManager(stdio=stdio, inner_config_change_map=inner_config_change_map) + # obdiag.logger.error_stream if self.inner_config_manager.config.get("obdiag") is not None and self.inner_config_manager.config.get("obdiag").get("basic") is not None and self.inner_config_manager.config.get("obdiag").get("basic").get("print_type") is not None: stdio.set_err_stream(self.inner_config_manager.config.get("obdiag").get("logger").get("error_stream")) - + # obdiag.logger.silent + if self.inner_config_manager.config.get("obdiag") is not None and self.inner_config_manager.config.get("obdiag").get("logger") is not None and self.inner_config_manager.config.get("obdiag").get("logger").get("silent") is not None: + stdio.set_silent(self.inner_config_manager.config.get("obdiag").get("logger").get("silent")) self.set_stdio(stdio) + if config_path: + if os.path.exists(os.path.abspath(config_path)): + config_path = config_path + else: + stdio.error('The option you provided with -c: {0} is not exist.'.format(config_path)) + return self.config_manager = ConfigManager(config_path, stdio) if ( self.inner_config_manager.config.get("obdiag") is not None @@ -191,7 +201,7 @@ def gather_function(self, function_type, opt): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.stdio.print("{0} start ...".format(function_type)) self.set_context(function_type, 'gather', config) @@ -230,8 +240,7 @@ def gather_function(self, function_type, opt): handler_log = GatherLogHandler(self.context) handler_log.handle() handler_obproxy = GatherObProxyLogHandler(self.context) - handler_obproxy.handle() - return True + return handler_obproxy.handle() elif function_type == 'gather_sysstat': handler = GatherOsInfoHandler(self.context) return handler.handle() @@ -252,13 +261,13 @@ def gather_function(self, function_type, opt): return handler.handle() else: self._call_stdio('error', 'Not support gather function: {0}'.format(function_type)) - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='Not support gather function: {0}'.format(function_type)) def gather_obproxy_log(self, opt): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.set_context_skip_cluster_conn('gather_obproxy_log', 'gather', config) handler = GatherObProxyLogHandler(self.context) @@ -273,33 +282,34 @@ def analyze_fuction(self, function_type, opt): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.stdio.print("{0} start ...".format(function_type)) if function_type == 'analyze_log': self.set_context(function_type, 'analyze', config) handler = AnalyzeLogHandler(self.context) - handler.handle() + return handler.handle() elif function_type == 'analyze_log_offline': self.set_context_skip_cluster_conn(function_type, 'analyze', config) handler = AnalyzeLogHandler(self.context) - handler.handle() + return handler.handle() elif function_type == 'analyze_flt_trace': self.set_context(function_type, 'analyze', config) handler = AnalyzeFltTraceHandler(self.context) - handler.handle() + return handler.handle() elif function_type == 'analyze_parameter_default': self.set_context(function_type, 'analyze', config) handler = AnalyzeParameterHandler(self.context, 'default') - handler.handle() + return handler.handle() elif function_type == 'analyze_parameter_diff': self.set_context_skip_cluster_conn(function_type, 'analyze', config) handler = AnalyzeParameterHandler(self.context, 'diff') - handler.handle() + return handler.handle() elif function_type == 'analyze_variable_diff': self.set_context(function_type, 'analyze', config) handler = AnalyzeVariableHandler(self.context, 'diff') - handler.handle() + return handler.handle() + # todo not support silent elif function_type == 'analyze_sql': self.set_context(function_type, 'analyze', config) handler = AnalyzeSQLHandler(self.context) @@ -310,26 +320,29 @@ def analyze_fuction(self, function_type, opt): handler.handle() else: self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type)) - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='Not support analyze function: {0}'.format(function_type)) def check(self, opts): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.stdio.print("check start ...") self.set_context('check', 'check', config) obproxy_check_handler = None observer_check_handler = None + result_data = {} if self.context.obproxy_config.get("servers") is not None and len(self.context.obproxy_config.get("servers")) > 0: obproxy_check_handler = CheckHandler(self.context, check_target_type="obproxy") obproxy_check_handler.handle() - obproxy_check_handler.execute() + obproxy_result = obproxy_check_handler.execute() + result_data['obproxy'] = obproxy_result if self.context.cluster_config.get("servers") is not None and len(self.context.cluster_config.get("servers")) > 0: observer_check_handler = CheckHandler(self.context, check_target_type="observer") observer_check_handler.handle() - observer_check_handler.execute() + observer_result = observer_check_handler.execute() + result_data['observer'] = observer_result if obproxy_check_handler is not None: obproxy_report_path = os.path.expanduser(obproxy_check_handler.report.get_report_path()) if os.path.exists(obproxy_report_path): @@ -338,59 +351,62 @@ def check(self, opts): observer_report_path = os.path.expanduser(observer_check_handler.report.get_report_path()) if os.path.exists(observer_report_path): self.stdio.print("Check observer finished. For more details, please run cmd'" + Fore.YELLOW + " cat {0} ".format(observer_check_handler.report.get_report_path()) + Style.RESET_ALL + "'") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=result_data) def check_list(self, opts): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.set_offline_context('check_list', 'check_list') handler = CheckListHandler(self.context) - handler.handle() + return handler.handle() def rca_run(self, opts): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.set_context('rca_run', 'rca_run', config) try: handler = RCAHandler(self.context) handler.handle() - handler.execute() + return handler.execute() except Exception as e: self.stdio.error("rca run Exception: {0}".format(e)) self.stdio.verbose(traceback.format_exc()) + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="rca run Exception: {0}".format(e)) def rca_list(self, opts): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.set_offline_context('rca_list', 'rca_list') handler = RcaScenesListHandler(context=self.context) - handler.handle() + return handler.handle() def update(self, opts): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.stdio.print("update start ...") self.set_offline_context('update', 'update') handler = UpdateHandler(self.context) - handler.execute() + return handler.execute() def config(self, opt): config = self.config_manager if not config: self._call_stdio('error', 'No such custum config') - return False + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config') else: self.set_offline_context('config', 'config') config_helper = ConfigHelper(context=self.context) config_helper.build_configuration() + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"msg": "config success"}) diff --git a/diag_cmd.py b/diag_cmd.py index 1ffa72f0..95d5dc14 100644 --- a/diag_cmd.py +++ b/diag_cmd.py @@ -26,6 +26,7 @@ from uuid import uuid1 as uuid, UUID from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter from core import ObdiagHome +from result_type import ObdiagResult from stdio import IO from common.version import get_obdiag_version from telemetry.telemetry import telemetry @@ -195,6 +196,10 @@ def _mk_usage(self): class ObdiagOriginCommand(BaseCommand): OBDIAG_PATH = OBDIAG_HOME_PATH + def __init__(self, name, summary): + super().__init__(name, summary) + self.trace_id = uuid() + @property def enable_log(self): return True @@ -251,9 +256,9 @@ def do_command(self): os.makedirs(log_directory, exist_ok=True) log_path = os.path.join(log_directory, 'obdiag.log') if self.enable_log: - ROOT_IO.init_trace_logger(log_path, 'obdiag', trace_id) + ROOT_IO.init_trace_logger(log_path, 'obdiag', self.trace_id) ROOT_IO.track_limit += 1 - ROOT_IO.verbose('cmd: %s' % self.cmds) + ROOT_IO.verbose('cmd: %s' % self.prev_cmd) ROOT_IO.verbose('opts: %s' % self.opts) config_path = os.path.expanduser('~/.obdiag/config.yml') custom_config = Util.get_option(self.opts, 'c') @@ -263,11 +268,38 @@ def do_command(self): else: ROOT_IO.error('The option you provided with -c: {0} is a non-existent configuration file path.'.format(custom_config)) return - obdiag = ObdiagHome(stdio=ROOT_IO, config_path=config_path, inner_config_change_map=self.inner_config_change_map) + obdiag = ObdiagHome(stdio=ROOT_IO, config_path=custom_config, inner_config_change_map=self.inner_config_change_map) obdiag.set_options(self.opts) obdiag.set_cmds(self.cmds) ret = self._do_command(obdiag) + exit_code = 0 + # if silent is true ,print ret + if ROOT_IO.silent: + if isinstance(ret, ObdiagResult) is False: + ROOT_IO.error('The return value of the command is not ObdiagResult. Please contact thebase community. The return value is: {0}'.format(ret)) + ret = ObdiagResult(code=ObdiagResult.SERVER_ERROR_CODE, error_data="The return value of the command is not ObdiagResult. Maybe the command not support silent. Please contact thebase community.") + ret.set_trace_id(self.trace_id) + + def args_to_str(args): + args_str = "" + for arg in args: + args_str += arg + " " + return args_str.strip() + + ret.set_command(self.prev_cmd + " " + args_to_str(self.args)) + ROOT_IO.set_silent(False) + ROOT_IO.print(ret.get_result()) + ROOT_IO.set_silent(True) + if self.has_trace: + ROOT_IO.print('Trace ID: %s' % self.trace_id) + ROOT_IO.print('If you want to view detailed obdiag logs, please run: {0} display-trace {1}'.format(obdiag_bin, self.trace_id)) telemetry.put_data() + if ROOT_IO.silent: + if ret.get_code() == ObdiagResult.SUCCESS_CODE: + return True + else: + return False + return True except NotImplementedError: ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd) except SystemExit: @@ -277,10 +309,6 @@ def do_command(self): except: e = sys.exc_info()[1] ROOT_IO.exception('Running Error: %s' % e) - if self.has_trace: - ROOT_IO.print('Trace ID: %s' % trace_id) - ROOT_IO.print('If you want to view detailed obdiag logs, please run: {0} display-trace {1}'.format(obdiag_bin, trace_id)) - return ret or True def _do_command(self, obdiag): raise NotImplementedError @@ -857,8 +885,7 @@ def init(self, cmd, args): def _do_command(self, obdiag): if 'list' in self.args: - obdiag.check_list(self.opts) - return + return obdiag.check_list(self.opts) return obdiag.check(self.opts) diff --git a/handler/analyzer/analyze_flt_trace.py b/handler/analyzer/analyze_flt_trace.py index cac530a1..8aa0cacf 100644 --- a/handler/analyzer/analyze_flt_trace.py +++ b/handler/analyzer/analyze_flt_trace.py @@ -28,6 +28,7 @@ from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil +from result_type import ObdiagResult class AnalyzeFltTraceHandler(object): @@ -86,10 +87,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data='init option failed') if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data='init config failed') local_store_parent_dir = os.path.join(self.gather_pack_dir, "obdiag_analyze_flt_result_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] @@ -119,8 +120,8 @@ def handle_from_node(node): data = future.result() tree.build(data) # output tree - self.__output(local_store_parent_dir, tree, self.output) - return analyze_tuples + result = self.__output(local_store_parent_dir, tree, self.output) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": local_store_parent_dir, "result": result}) def __handle_from_node(self, node, old_files, local_store_parent_dir): resp = {"skip": False, "error": ""} @@ -346,6 +347,15 @@ def __output(self, result_dir, tree, output_terminal=60): self.stdio.verbose('Result saved: {}'.format(os.path.abspath(filename))) last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(filename) self.stdio.print(last_info) + result_info = "" + with open(filename, 'r', encoding='utf-8') as f: + line_nu = 0 + for line in f: + result_info += line + line_nu += 1 + if line_nu > 60: + break + return result_info def parse_file(self, file): self.stdio.verbose('parse file: {}'.format(file[1])) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index 439f7928..1794f480 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -32,6 +32,7 @@ from common.tool import FileUtil from common.tool import TimeUtils import common.ssh_client.local_client as ssh_client_local_client +from result_type import ObdiagResult class AnalyzeLogHandler(BaseShellHandler): @@ -125,10 +126,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") local_store_parent_dir = os.path.join(self.gather_pack_dir, "obdiag_analyze_pack_{0}".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] @@ -160,7 +161,11 @@ def handle_from_node(node): FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), field_names[n] + ": " + str(summary_details_list[m][n]) + extend) last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(local_store_parent_dir, "result_details.txt")) self.stdio.print(last_info) - return analyze_tuples + # get info from local_store_parent_dir+/result_details.txt + analyze_info = "" + with open(os.path.join(local_store_parent_dir, "result_details.txt"), "r", encoding="utf-8") as f: + analyze_info = f.read() + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": analyze_info}) def __handle_from_node(self, node, local_store_parent_dir): resp = {"skip": False, "error": ""} diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 26c6c31a..40e5c012 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -26,6 +26,8 @@ import datetime from colorama import Fore, Style +from result_type import ObdiagResult + class AnalyzeParameterHandler(object): def __init__(self, context, analyze_type='default'): @@ -67,14 +69,14 @@ def handle(self): if self.analyze_type == 'default': if not self.init_option_default(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") else: if not self.init_option_diff(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") self.stdio.verbose("Use {0} as pack dir.".format(self.export_report_path)) DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) - self.execute() + return self.execute() def check_file_valid(self): with open(self.parameter_file_name, 'r') as f: @@ -167,10 +169,11 @@ def analyze_parameter_default(self): fp.write(report_default_tb.get_string() + "\n") self.stdio.print(report_default_tb.get_string()) self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": report_default_tb.get_string(), "file_name": file_name}) else: if self.parameter_file_name is None: self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values") - return + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values") else: sql = '''select substr(version(),8), svr_ip,svr_port,zone,scope,TENANT_ID,name,value,section, EDIT_LEVEL, now(),'','' from GV$OB_PARAMETERS order by 5,2,3,4,7''' @@ -262,6 +265,7 @@ def alalyze_parameter_diff(self): file_name = self.export_report_path + '/parameter_diff_{0}.table'.format(date_format) fp = open(file_name, 'a+', encoding="utf8") is_empty = True + report_diff_tbs = [] for tenant, value_list in diff_parameter_dict.items(): if len(value_list) > 0: report_diff_tb = PrettyTable(["name", "diff"]) @@ -279,17 +283,20 @@ def alalyze_parameter_diff(self): fp.write(report_diff_tb.get_string() + "\n") self.stdio.print(report_diff_tb.get_string()) is_empty = False + report_diff_tbs.append(report_diff_tb.get_string()) fp.close() if not is_empty: self.stdio.print("Analyze parameter diff finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": report_diff_tbs, "store_dir": file_name}) else: self.stdio.print("Analyze parameter diff finished. All parameter settings are consistent among observers") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": "Analyze parameter diff finished. All parameter settings are consistent among observers"}) def execute(self): try: if self.analyze_type == 'default': - self.analyze_parameter_default() + return self.analyze_parameter_default() elif self.analyze_type == 'diff': - self.alalyze_parameter_diff() + return self.alalyze_parameter_diff() except Exception as e: self.stdio.error("parameter info analyze failed, error message: {0}".format(e)) diff --git a/handler/analyzer/analyze_sql.py b/handler/analyzer/analyze_sql.py index e6ab6374..ded4f21b 100644 --- a/handler/analyzer/analyze_sql.py +++ b/handler/analyzer/analyze_sql.py @@ -30,6 +30,7 @@ from handler.analyzer.sql.meta.sys_tenant_meta import SysTenantMeta from handler.gather.gather_scenes import GatherSceneHandler from common.command import get_observer_version +from result_type import ObdiagResult class AnalyzeSQLHandler(object): @@ -208,16 +209,16 @@ def handle(self): self.start_time = time.time() if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_inner_config(): self.stdio.error('init inner config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init inner config failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") if not self.init_ob_version(): self.stdio.error('init ob version failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init ob version failed") self.init_db_connector() self.local_store_path = os.path.join(self.local_stored_parrent_path, "obdiag_analyze_sql_result_{0}_{1}.html".format(TimeUtils.timestamp_to_filename_time(self.from_timestamp), TimeUtils.timestamp_to_filename_time(self.to_timestamp))) self.stdio.print("use {0} as result store path.".format(self.local_store_path)) diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index e3bbc5d9..9199e77a 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -24,6 +24,8 @@ import datetime from colorama import Fore, Style +from result_type import ObdiagResult + class AnalyzeVariableHandler(object): def __init__(self, context, analyze_type='diff'): @@ -55,10 +57,10 @@ def __init__(self, context, analyze_type='diff'): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") self.stdio.verbose("Use {0} as pack dir.".format(self.export_report_path)) DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) - self.execute() + return self.execute() def check_file_valid(self): with open(self.variable_file_name, 'r') as f: @@ -149,11 +151,13 @@ def analyze_variable(self): self.stdio.print(Fore.RED + "Since {0}, the following variables have changed：".format(last_gather_time) + Style.RESET_ALL) self.stdio.print(report_default_tb.get_string()) self.stdio.print("Analyze variables changed finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": report_default_tb.get_string()}) else: self.stdio.print("Analyze variables changed finished. Since {0}, No changes in variables".format(last_gather_time)) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": "Since {0}, No changes in variables".format(last_gather_time)}) def execute(self): try: - self.analyze_variable() + return self.analyze_variable() except Exception as e: self.stdio.error("variable info analyze failed, error message: {0}".format(e)) diff --git a/handler/checker/check_handler.py b/handler/checker/check_handler.py index e74b9ccf..1e899562 100644 --- a/handler/checker/check_handler.py +++ b/handler/checker/check_handler.py @@ -101,7 +101,6 @@ def __init__(self, context, check_target_type="observer"): # add ssher ssher = None try: - # ssher = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) ssher = SshClient(context, node) except Exception as e: self.stdio.warn("StepBase get SshHelper fail on{0} ,Exception: {1}".format(node.get("ip"), e)) @@ -113,7 +112,6 @@ def __init__(self, context, check_target_type="observer"): # add OBConnectorPool try: obConnectorPool = checkOBConnectorPool(context, 3, self.cluster) - except Exception as e: self.stdio.warn("obConnector init error. Error info is {0}".format(e)) finally: @@ -243,6 +241,7 @@ def execute(self): t_report = self.execute_one(task) self.report.add_task_report(t_report) self.report.export_report() + return self.report.report_tobeMap() except CheckrReportException as e: self.stdio.error("Report error :{0}".format(e)) self.stdio.verbose(traceback.format_exc()) diff --git a/handler/checker/check_list.py b/handler/checker/check_list.py index 53ab952c..8a5d6451 100644 --- a/handler/checker/check_list.py +++ b/handler/checker/check_list.py @@ -20,6 +20,7 @@ import yaml from common.tool import Util +from result_type import ObdiagResult class CheckListHandler: @@ -29,41 +30,56 @@ def __init__(self, context): self.work_path = os.path.expanduser(self.context.inner_config["check"]["work_path"] or "~/.obdiag/check") def handle(self): - self.stdio.verbose("list check cases") - entries = os.listdir(self.work_path) - files = [f for f in entries if os.path.isfile(os.path.join(self.work_path, f))] - for file in files: - if "check_package" in file: - cases_map = {"all": {"name": "all", "command": "obdiag check", "info_en": "default check all task without filter", "info_cn": "默认执行除filter组里的所有巡检项"}} - # Obtain which files match and corresponding header files - # Using string segmentation methods - parts = file.split('_') - if len(parts) < 1: - self.stdio.warn("invalid check package name :{0} , Please don't add file, which 'check_package' in the name".format(file)) - continue - target = parts[0] - file = "{0}/{1}".format(self.work_path, file) - package_file_data = None - # read yaml file - with open(file, 'r') as f: - package_file_data = yaml.safe_load(f) - if not package_file_data or len(package_file_data) == 0: - self.stdio.warn("No data check package data :{0} ".format(file)) + try: + self.stdio.verbose("list check cases") + entries = os.listdir(self.work_path) + files = [f for f in entries if os.path.isfile(os.path.join(self.work_path, f))] + result_map = {} + for file in files: + if "check_package" in file: + cases_map = {"all": {"name": "all", "command": "obdiag check", "info_en": "default check all task without filter", "info_cn": "默认执行除filter组里的所有巡检项"}} + # Obtain which files match and corresponding header files + # Using string segmentation methods + parts = file.split('_') + if len(parts) < 1: + self.stdio.warn("invalid check package name :{0} , Please don't add file, which 'check_package' in the name".format(file)) continue - for package_data in package_file_data: - if package_data == "filter": + target = parts[0] + file = "{0}/{1}".format(self.work_path, file) + package_file_data = None + # read yaml file + with open(file, 'r') as f: + package_file_data = yaml.safe_load(f) + result_map[target] = {} + result_map[target]["commands"] = [] + if not package_file_data or len(package_file_data) == 0: + self.stdio.warn("No data check package data :{0} ".format(file)) continue - package_target = target - if target == "observer": - package_target = "cases" - else: - package_target = "{0}_cases".format(target) + for package_data in package_file_data: + if package_data == "filter": + continue + package_target = target + if target == "observer": + package_target = "cases" + else: + package_target = "{0}_cases".format(target) - cases_map[package_data] = { - "name": package_data, - "command": "obdiag check --{0}={1}".format(package_target, package_data), - "info_en": package_file_data[package_data].get("info_en") or "", - "info_cn": package_file_data[package_data].get("info_cn") or "", - } - Util.print_title("check cases about {0}".format(target)) - Util.print_scene(cases_map) + cases_map[package_data] = { + "name": package_data, + "command": "obdiag check --{0}={1}".format(package_target, package_data), + "info_en": package_file_data[package_data].get("info_en") or "", + "info_cn": package_file_data[package_data].get("info_cn") or "", + } + result_map[target]["commands"].append( + { + "name": package_data, + "command": "obdiag check --{0}={1}".format(package_target, package_data), + "info_en": package_file_data[package_data].get("info_en") or "", + "info_cn": package_file_data[package_data].get("info_cn") or "", + } + ) + Util.print_title("check cases about {0}".format(target), stdio=self.stdio) + Util.print_scene(cases_map, stdio=self.stdio) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=result_map) + except Exception as e: + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data=str(e)) diff --git a/handler/checker/tasks/observer/bugs/bug_385.yaml b/handler/checker/tasks/observer/bugs/bug_385.yaml new file mode 100644 index 00000000..255a5ece --- /dev/null +++ b/handler/checker/tasks/observer/bugs/bug_385.yaml @@ -0,0 +1,14 @@ +info: "OB version [4.2.1.0,4.2.1.3] If tenants have multiple root users. Please consider upgrading the OceanBase version or removing the redundant users. github issue #385" +task: + - version: "[4.2.1.0,4.2.1.3]" + steps: + - type: sql + sql: "SELECT GROUP_CONCAT(TENANT_ID) AS TENANT_ID +FROM oceanbase.CDB_OB_USERS +WHERE USER_NAME = 'root' +GROUP BY TENANT_ID +HAVING COUNT(*) > 1;" + result: + set_value: TENANT_ID + verify: '[ -z "$TENANT_ID" ]' + err_msg: "tenant: #{$TENANT_ID}. These tenants have multiple root users. Please consider upgrading the OceanBase version or removing the redundant users. Please get bug's on https://github.com/oceanbase/obdiag/issues/385" diff --git a/handler/checker/tasks/observer/cluster/task_opt_stat.yaml b/handler/checker/tasks/observer/cluster/task_opt_stat.yaml new file mode 100644 index 00000000..7d926e82 --- /dev/null +++ b/handler/checker/tasks/observer/cluster/task_opt_stat.yaml @@ -0,0 +1,16 @@ +info: 'Check whether data_dir and log_dir_disk are on the same disk.' +task: + - version: "[4.2.0.0,*]" + steps: + - type: sql + sql: 'SELECT GROUP_CONCAT(DISTINCT TENANT_ID) +FROM oceanbase.__all_tenant t +WHERE NOT EXISTS(SELECT 1 + FROM oceanbase.__all_virtual_task_opt_stat_gather_history h + WHERE TYPE = 1 + AND start_time > date_sub(now(), interval 1 day) + AND h.tenant_id = t.tenant_id);' + result: + set_value: failed_scheduler_tenant_id + verify: '[ -n "${failed_scheduler_tenant_id}" ]' + err_msg: "failed_scheduler_tenant_id is exists. Please check the tenant_ids: #{failed_scheduler_tenant_id}" \ No newline at end of file diff --git a/handler/checker/tasks/observer/system/parameter.yaml b/handler/checker/tasks/observer/system/parameter.yaml index 38fc7fd7..3be8519f 100644 --- a/handler/checker/tasks/observer/system/parameter.yaml +++ b/handler/checker/tasks/observer/system/parameter.yaml @@ -176,7 +176,7 @@ task: set_value: parameter report_type: warning verify: "[ 6573688 -le ${parameter} ]" - err_msg: 'fs.file-max: #{parameter}. recommended: #{parameter} is ≥ 6573688.' + err_msg: 'fs.file-max: #{parameter}. recommended: is ≥ 6573688.' - type: get_system_parameter parameter: fs.pipe-user-pages-soft result: diff --git a/handler/gather/gather_ash_report.py b/handler/gather/gather_ash_report.py index fc1e4eb1..6cd91510 100644 --- a/handler/gather/gather_ash_report.py +++ b/handler/gather/gather_ash_report.py @@ -22,6 +22,7 @@ from common.ob_connector import OBConnector from common.obdiag_exception import OBDIAGFormatException, OBDIAGException from common.tool import DirectoryUtil, TimeUtils, Util, StringUtils +from result_type import ObdiagResult from stdio import SafeStdio from colorama import Fore, Style @@ -60,13 +61,13 @@ def __init__(self, context, gather_pack_dir='./'): def handle(self): if not self.version_check(): self.stdio.error('version check failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="version check failed") if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") self.__init_report_path() self.execute() - self.__print_result() + return self.__print_result() def version_check(self): observer_version = "" @@ -189,3 +190,4 @@ def init_option(self): def __print_result(self): self.stdio.print(Fore.YELLOW + "\nGather ash_report results stored in this directory: {0}".format(self.report_path) + Style.RESET_ALL) self.stdio.print("") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": self.report_path}) diff --git a/handler/gather/gather_awr.py b/handler/gather/gather_awr.py index bec5b9e6..ae6b60f5 100644 --- a/handler/gather/gather_awr.py +++ b/handler/gather/gather_awr.py @@ -27,6 +27,7 @@ from common.tool import Util from common.tool import TimeUtils from common.ocp import ocp_task, ocp_api +from result_type import ObdiagResult class GatherAwrHandler(object): @@ -100,7 +101,8 @@ def handle_awr_from_ocp(ocp_url, cluster_name): # 将汇总结果持久化记录到文件中 FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) - return gather_tuples, gather_pack_path_dict + # return gather_tuples, gather_pack_path_dict + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __download_report(self, store_path, name, report_id): """ diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 80f6cb81..5246ed16 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -29,6 +29,7 @@ from common.tool import DirectoryUtil from common.tool import FileUtil from common.tool import NetUtils +from result_type import ObdiagResult class GatherLogHandler(BaseShellHandler): @@ -168,7 +169,7 @@ def handle_from_node(node): # Persist the summary results to a file FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) - return True + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __handle_from_node(self, pack_dir_this_command, node): resp = {"skip": False, "error": "", "zip_password": "", "gather_pack_path": ""} diff --git a/handler/gather/gather_obadmin.py b/handler/gather/gather_obadmin.py index a7c3da04..93c6ece2 100644 --- a/handler/gather/gather_obadmin.py +++ b/handler/gather/gather_obadmin.py @@ -31,6 +31,7 @@ from common.tool import DirectoryUtil from common.tool import FileUtil from common.tool import NetUtils +from result_type import ObdiagResult class GatherObAdminHandler(BaseShellHandler): @@ -116,10 +117,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") pack_dir_this_command = os.path.join(self.local_stored_path, "obdiag_gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] @@ -152,6 +153,7 @@ def handle_from_node(node): FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __handle_from_node(self, local_stored_path, node): resp = {"skip": False, "error": "", "gather_pack_path": ""} diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index ee32c8b7..1550505d 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -31,6 +31,7 @@ from common.tool import FileUtil from common.tool import NetUtils from common.tool import TimeUtils +from result_type import ObdiagResult class GatherObProxyLogHandler(BaseShellHandler): @@ -136,10 +137,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") if self.is_scene: pack_dir_this_command = self.gather_pack_dir else: @@ -171,7 +172,7 @@ def handle_from_node(node): self.pack_dir_this_command = pack_dir_this_command FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) - return True + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __handle_from_node(self, node, pack_dir_this_command): resp = {"skip": False, "error": "", "zip_password": "", "gather_pack_path": ""} diff --git a/handler/gather/gather_obstack2.py b/handler/gather/gather_obstack2.py index 9b0e8084..4a922f6b 100644 --- a/handler/gather/gather_obstack2.py +++ b/handler/gather/gather_obstack2.py @@ -32,6 +32,7 @@ from common.tool import FileUtil from common.tool import NetUtils from common.tool import StringUtils +from result_type import ObdiagResult class GatherObstack2Handler(BaseShellHandler): @@ -77,10 +78,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") if self.is_scene: pack_dir_this_command = self.local_stored_path else: @@ -104,6 +105,7 @@ def handle_from_node(node): # Persist the summary results to a file FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __handle_from_node(self, local_stored_path, node): resp = {"skip": False, "error": "", "gather_pack_path": ""} diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index 187fb779..359ff423 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -23,6 +23,8 @@ import csv from colorama import Fore, Style +from result_type import ObdiagResult + class GatherParametersHandler(object): def __init__(self, context, gather_pack_dir='./'): @@ -53,13 +55,14 @@ def __init__(self, context, gather_pack_dir='./'): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") # example of the format of pack dir for this command: (gather_pack_dir)/gather_pack_20190610123344 pack_dir_this_command = os.path.join(self.gather_pack_dir, "gather_parameters") self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) self.gather_pack_dir = pack_dir_this_command self.execute() + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"gather_pack_dir": pack_dir_this_command}) def init_option(self): options = self.context.options diff --git a/handler/gather/gather_perf.py b/handler/gather/gather_perf.py index db792d3d..61d71450 100644 --- a/handler/gather/gather_perf.py +++ b/handler/gather/gather_perf.py @@ -30,6 +30,7 @@ from common.tool import FileUtil from common.tool import NetUtils from common.tool import TimeUtils +from result_type import ObdiagResult class GatherPerfHandler(BaseShellHandler): @@ -79,10 +80,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") if self.is_scene: pack_dir_this_command = self.local_stored_path else: @@ -113,6 +114,7 @@ def handle_from_node(node): # Persist the summary results to a file FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __handle_from_node(self, node, local_stored_path): resp = {"skip": False, "error": "", "gather_pack_path": ""} diff --git a/handler/gather/gather_plan_monitor.py b/handler/gather/gather_plan_monitor.py index 6d248bcf..7ab0ba73 100644 --- a/handler/gather/gather_plan_monitor.py +++ b/handler/gather/gather_plan_monitor.py @@ -33,6 +33,7 @@ from common.tool import FileUtil from common.tool import TimeUtils from handler.gather.gather_tabledump import GatherTableDumpHandler +from result_type import ObdiagResult class GatherPlanMonitorHandler(object): @@ -95,10 +96,10 @@ def __init_db_connector(self): def handle(self): if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if self.is_scene: pack_dir_this_command = self.local_stored_path else: @@ -219,7 +220,8 @@ def handle_plan_monitor_from_ob(cluster_name): self.stdio.print(summary_tuples) # 将汇总结果持久化记录到文件中 FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) - return gather_tuples, gather_pack_path_dict + # return gather_tuples, gather_pack_path_dict + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __init_db_conn(self, env): try: diff --git a/handler/gather/gather_scenes.py b/handler/gather/gather_scenes.py index d54e2f57..55a17d91 100644 --- a/handler/gather/gather_scenes.py +++ b/handler/gather/gather_scenes.py @@ -18,6 +18,7 @@ import os import re +from result_type import ObdiagResult from stdio import SafeStdio import datetime from handler.gather.scenes.base import SceneBase @@ -65,19 +66,19 @@ def init_config(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed") self.__init_variables() self.__init_report_path() self.__init_task_names() self.execute() if self.is_inner: result = self.__get_sql_result() - return result + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": self.report_path}) else: - self.__print_result() + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": self.report_path}) def execute(self): try: @@ -231,3 +232,4 @@ def __get_sql_result(self): def __print_result(self): self.stdio.print(Fore.YELLOW + "\nGather scene results stored in this directory: {0}\n".format(self.report_path) + Style.RESET_ALL) + return self.report_path diff --git a/handler/gather/gather_sysstat.py b/handler/gather/gather_sysstat.py index a77dff57..f6aea7c2 100644 --- a/handler/gather/gather_sysstat.py +++ b/handler/gather/gather_sysstat.py @@ -29,6 +29,7 @@ from common.tool import FileUtil from common.tool import NetUtils from common.tool import TimeUtils +from result_type import ObdiagResult class GatherOsInfoHandler(BaseShellHandler): @@ -80,10 +81,10 @@ def init_option(self): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, "init option failed") if not self.init_config(): self.stdio.error('init config failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, "init config failed") if self.is_scene: pack_dir_this_command = self.local_stored_path @@ -114,6 +115,7 @@ def handle_from_node(node): self.stdio.print(summary_tuples) # Persist the summary results to a file FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def __handle_from_node(self, node, local_stored_path): resp = {"skip": False, "error": "", "gather_pack_path": ""} diff --git a/handler/gather/gather_tabledump.py b/handler/gather/gather_tabledump.py index afecfd24..88ac6078 100644 --- a/handler/gather/gather_tabledump.py +++ b/handler/gather/gather_tabledump.py @@ -18,6 +18,8 @@ import os import time + +from result_type import ObdiagResult from stdio import SafeStdio from common.ob_connector import OBConnector from common.tool import StringUtils @@ -99,10 +101,12 @@ def handle(self): self.start_time = time.time() if not self.init(): self.stdio.error('init failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init failed") excute_status = self.execute() if not self.is_innner and excute_status: self.__print_result() + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": self.store_dir}) + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="execute failed") def execute(self): try: @@ -253,3 +257,4 @@ def __print_result(self): self.stdio.print("\nAnalyze SQL Summary:") self.stdio.print(table) self.stdio.print("\n") + return diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index 6c49b538..970e5ad2 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -22,6 +22,8 @@ import csv from colorama import Fore, Style +from result_type import ObdiagResult + class GatherVariablesHandler(object): def __init__(self, context, gather_pack_dir='./'): @@ -52,12 +54,13 @@ def __init__(self, context, gather_pack_dir='./'): def handle(self): if not self.init_option(): self.stdio.error('init option failed') - return False + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init option failed") pack_dir_this_command = os.path.join(self.gather_pack_dir, "gather_variables") self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) self.gather_pack_dir = pack_dir_this_command self.execute() + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": pack_dir_this_command}) def init_option(self): options = self.context.options diff --git a/handler/gather/scenes/list.py b/handler/gather/scenes/list.py index d4a970cf..099e13a8 100644 --- a/handler/gather/scenes/list.py +++ b/handler/gather/scenes/list.py @@ -17,6 +17,8 @@ """ import os + +from result_type import ObdiagResult from stdio import SafeStdio from common.tool import YamlUtils from handler.gather.scenes.register import hardcode_scene_list @@ -44,8 +46,9 @@ def handle(self): self.stdio.verbose("len of observer_tasks: {0}; len of observer_tasks: {1}; len of observer_tasks: {2};".format(len(self.observer_tasks), len(self.obproxy_tasks), len(self.other_tasks))) if (len(self.observer_tasks) + len(self.obproxy_tasks) + len(self.other_tasks)) == 0: self.stdio.error("Failed to find any tasks") + return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data="Failed to find any tasks") else: - self.print_scene_data() + return self.print_scene_data() def get_all_yaml_tasks(self): try: @@ -116,18 +119,24 @@ def print_scene_data(self): sorted_observer_tasks_dict = {} sorted_obproxy_tasks_dict = {} sorted_other_tasks_dict = {} + result_data = {} if self.other_tasks: sorted_other_tasks = sorted(self.other_tasks.items(), key=lambda x: x[0]) sorted_other_tasks_dict = {k: v for k, v in sorted_other_tasks} - Util.print_title("Other Problem Gather Scenes") - Util.print_scene(sorted_other_tasks_dict) + Util.print_title("Other Problem Gather Scenes", stdio=self.stdio) + Util.print_scene(sorted_other_tasks_dict, stdio=self.stdio) + result_data["sorted_other_tasks"] = sorted_other_tasks_dict if self.obproxy_tasks: sorted_obproxy_tasks = sorted(self.obproxy_tasks.items(), key=lambda x: x[0]) sorted_obproxy_tasks_dict = {k: v for k, v in sorted_obproxy_tasks} - Util.print_title("Obproxy Problem Gather Scenes") - Util.print_scene(sorted_obproxy_tasks_dict) + Util.print_title("Obproxy Problem Gather Scenes", stdio=self.stdio) + Util.print_scene(sorted_obproxy_tasks_dict, stdio=self.stdio) + result_data["sorted_obproxy_tasks"] = sorted_obproxy_tasks_dict + if self.observer_tasks: sorted_observer_tasks = sorted(self.observer_tasks.items(), key=lambda x: x[0]) sorted_observer_tasks_dict = {k: v for k, v in sorted_observer_tasks} - Util.print_title("Observer Problem Gather Scenes") - Util.print_scene(sorted_observer_tasks_dict) + Util.print_title("Observer Problem Gather Scenes", stdio=self.stdio) + Util.print_scene(sorted_observer_tasks_dict, stdio=self.stdio) + result_data["sorted_observer_tasks"] = sorted_observer_tasks_dict + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=result_data) diff --git a/handler/rca/rca_handler.py b/handler/rca/rca_handler.py index 4699c37d..1fe3a14e 100644 --- a/handler/rca/rca_handler.py +++ b/handler/rca/rca_handler.py @@ -33,6 +33,8 @@ from common.tool import StringUtils from colorama import Fore, Style +from result_type import ObdiagResult + class RCAHandler: def __init__(self, context): @@ -173,7 +175,7 @@ def execute(self): self.rca_scene.execute() except RCANotNeedExecuteException as e: self.stdio.warn("rca_scene.execute not need execute: {0}".format(e)) - pass + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, data="rca_scene.execute not need execute: {0}") except Exception as e: raise Exception("rca_scene.execute err: {0}".format(e)) try: @@ -181,6 +183,7 @@ def execute(self): except Exception as e: raise Exception("rca_scene.export_result err: {0}".format(e)) self.stdio.print("rca finished. For more details, the result on '" + Fore.YELLOW + self.get_result_path() + Style.RESET_ALL + "' \nYou can get the suggest by '" + Fore.YELLOW + "cat " + self.get_result_path() + "/record" + Style.RESET_ALL + "'") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": self.get_result_path(), "record": self.rca_scene.Result.records_data()}) class RcaScene: @@ -277,6 +280,14 @@ def export(self): f.write(record.export_suggest()) f.write("\n") + def records_data(self): + records_data = [] + for record in self.records: + if record.records is None or len(record.records) == 0: + continue + records_data.append({"record": record.records, "suggest": record.suggest}) + return records_data + class RCA_ResultRecord: def __init__(self, stdio=None): diff --git a/handler/rca/rca_list.py b/handler/rca/rca_list.py index 71b94be4..bd6c3914 100644 --- a/handler/rca/rca_list.py +++ b/handler/rca/rca_list.py @@ -19,6 +19,7 @@ from common.constant import const from common.tool import DynamicLoading from common.tool import Util +from result_type import ObdiagResult class RcaScenesListHandler: @@ -62,10 +63,11 @@ def handle(self): try: self.stdio.verbose("list rca scenes") scene_info_list, scene_itme_list = self.get_all_scenes() - Util.print_scene(scene_info_list) + Util.print_scene(scene_info_list, stdio=self.stdio) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=scene_info_list) except Exception as e: self.stdio.error("RcaScenesListHandler Exception: {0}".format(e)) - raise e + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="RcaScenesListHandler Exception:".format(e)) def __find_rca_files(self): files = [] diff --git a/handler/rca/scene/lock_conflict_scene.py b/handler/rca/scene/lock_conflict_scene.py index b5a1b5d0..f05095fe 100644 --- a/handler/rca/scene/lock_conflict_scene.py +++ b/handler/rca/scene/lock_conflict_scene.py @@ -68,7 +68,7 @@ def __execute_4_2(self): trans_record.add_record("get holding_lock trans_id:{0}".format(trans_id)) holding_lock_session_id = trans_id self.stdio.verbose("get holding lock SESSION_ID by trans_id:{0}".format(trans_id)) - cursor_by_trans_id = self.ob_connector.execute_sql_return_cursor_dictionary('select * from oceanbase.GV$OB_TRANSACTION_PARTICIPANTS where TX_ID="{0}";'.format(holding_lock_session_id)) + cursor_by_trans_id = self.ob_connector.execute_sql_return_cursor_dictionary('select * from oceanbase.GV$OB_TRANSACTION_PARTICIPANTS where TX_ID="{0}" and SESSION_ID<>0;'.format(holding_lock_session_id)) holding_lock_session_id_datas = cursor_by_trans_id.fetchall() holding_lock_session_id = "not get" self.stdio.verbose("get sql_info by holding_lock_session_id:{0}".format(holding_lock_session_id_datas)) @@ -82,7 +82,7 @@ def __execute_4_2(self): wait_lock_trans_id = OB_LOCKS_data["TRANS_ID"] trans_record.add_record("wait_lock_trans_id is {0}".format(wait_lock_trans_id)) - cursor_by_trans_id = self.ob_connector.execute_sql_return_cursor_dictionary('select * from oceanbase.GV$OB_TRANSACTION_PARTICIPANTS where TX_ID="{0}";'.format(wait_lock_trans_id)) + cursor_by_trans_id = self.ob_connector.execute_sql_return_cursor_dictionary('select * from oceanbase.GV$OB_TRANSACTION_PARTICIPANTS where TX_ID="{0}" and SESSION_ID<>0;'.format(wait_lock_trans_id)) wait_lock_session_datas = cursor_by_trans_id.fetchall() self.stdio.verbose("get sql_info by holding_lock_session_id:{0}".format(holding_lock_session_id)) diff --git a/init.sh b/init.sh index 2c007a5d..bb4f6eec 100755 --- a/init.sh +++ b/init.sh @@ -33,7 +33,6 @@ if [ -d "${WORK_DIR}/rca" ]; then cp -rf ${WORK_DIR}/rca ${OBDIAG_HOME}/ fi - ALIAS_OBDIAG_EXIST=$(grep "alias obdiag='sh" ~/.bashrc | head -n 1) if [[ "${ALIAS_OBDIAG_EXIST}" != "" ]]; then echo "need update obdiag alias" @@ -46,3 +45,4 @@ if [ -d "${OBDIAG_HOME}/check_package.yaml" ]; then echo "${OBDIAG_HOME}/*check_package.yaml and ${OBDIAG_HOME}/tasks has been discarded. If you have made any changes to these files on your own, please transfer the relevant data to *check_package.yaml in ${OBDIAG_HOME}/check/" fi echo "Init obdiag finished" +cd - diff --git a/result_type.py b/result_type.py new file mode 100644 index 00000000..71f30b01 --- /dev/null +++ b/result_type.py @@ -0,0 +1,45 @@ +import json + + +class ObdiagResult: + # ObdiagResult is the result of obdiag. + # It contains the code and result of obdiag. + + # SERVER_ERROR_CODE(5xx) is the code of server error. + SERVER_ERROR_CODE = 500 + # INPUT_ERROR_CODE(4xx) is the code of input error. + INPUT_ERROR_CODE = 400 + # SUCCESS_CODE(200) is the code of success. + SUCCESS_CODE = 200 + + def __init__(self, code, data=None, error_data=None): + self.command = None + self.trace_id = None + self.data = data + self.error_data = error_data + if code is None: + raise TypeError("ObdiagResult code is None. Please contact the Oceanbase community. ") + self.code = code + if data is not None: + if isinstance(data, dict): + self.data = data + else: + raise TypeError("ObdiagResult data is not dict. Please contact the Oceanbase community. ") + if error_data is not None: + if isinstance(error_data, str): + self.error_data = error_data + else: + raise TypeError("ObdiagResult error_data is not str. Please contact the Oceanbase community. ") + + def set_trace_id(self, trace_id): + self.trace_id = "{0}".format(trace_id) + + def set_command(self, command): + self.command = command + + def get_result(self): + result = {"code": self.code, "data": self.data, "error_data": self.error_data, "trace_id": self.trace_id, "command": self.command} + return json.dumps(result, ensure_ascii=False) + + def get_code(self): + return self.code diff --git a/stdio.py b/stdio.py index bdb10105..5a192ea9 100644 --- a/stdio.py +++ b/stdio.py @@ -84,6 +84,13 @@ def flush(self): return True +class SetBufferIO(BufferIO): + + def write(self, s): + if s not in self._buffer: + return super(SetBufferIO, self).write(s) + + class SysStdin(object): NONBLOCK = False @@ -147,6 +154,7 @@ def _readline(cls): try: for line in sys.stdin: return line + return '' except IOError: return '' finally: @@ -359,7 +367,8 @@ class IO(object): WARNING_PREV = FormtatText.warning('[WARN]') ERROR_PREV = FormtatText.error('[ERROR]') - def __init__(self, level, msg_lv=MsgLevel.DEBUG, use_cache=False, track_limit=0, root_io=None, input_stream=SysStdin, output_stream=sys.stdout, error_stream=sys.stdout): + def __init__(self, level, msg_lv=MsgLevel.DEBUG, use_cache=False, track_limit=0, root_io=None, input_stream=SysStdin, output_stream=sys.stdout, error_stream=sys.stdout, silent=False): + self.silent = silent self.level = level self.msg_lv = msg_lv self.default_confirm = False @@ -378,12 +387,17 @@ def __init__(self, level, msg_lv=MsgLevel.DEBUG, use_cache=False, track_limit=0, self._cur_out_obj = None self._cur_err_obj = None self._before_critical = None + self._exit_msg = "" self._output_is_tty = False self._input_is_tty = False + self._exit_buffer = SetBufferIO() self.set_input_stream(input_stream) self.set_output_stream(output_stream) self.set_err_stream(error_stream) + def set_silent(self, silent=False): + self.silent = bool(silent) + def isatty(self): if self._root_io: return self._root_io.isatty() @@ -488,8 +502,20 @@ def before_close(self): except: pass + @property + def exit_msg(self): + return self._exit_msg + + @exit_msg.setter + def exit_msg(self, msg): + self._exit_msg = msg + def _close(self): self.before_close() + self._flush_cache() + if self.exit_msg: + self.print(self.exit_msg) + self.exit_msg = "" self._flush_log() def __del__(self): @@ -533,6 +559,11 @@ def get_cur_out_obj(self): return self._root_io.get_cur_out_obj() return self._cur_out_obj + def get_exit_buffer(self): + if self._root_io: + return self._root_io.get_exit_buffer() + return self._exit_buffer + def _start_buffer_io(self): if self._root_io: return False @@ -606,6 +637,8 @@ def _stop_sync_obj(self, sync_clz, stop_type, *arg, **kwargs): return ret def start_loading(self, text, *arg, **kwargs): + if self.silent: + return True if self.sync_obj: return False self.sync_obj = self._start_sync_obj(IOHalo, lambda x: x.stop_loading('fail'), *arg, **kwargs) @@ -614,6 +647,8 @@ def start_loading(self, text, *arg, **kwargs): return self.sync_obj.start(text) def stop_loading(self, stop_type, *arg, **kwargs): + if self.silent: + return True if not isinstance(self.sync_obj, IOHalo): return False if getattr(self.sync_obj, stop_type, False): @@ -677,15 +712,18 @@ def print_list(self, ary, field_names=None, exp=lambda x: x if isinstance(x, (li def read(self, msg='', blocked=False): if msg: - self._print(MsgLevel.INFO, msg) - return self.get_input_stream().read(blocked) + if self.syncing: + self.verbose(msg, end='') + else: + self._print(MsgLevel.INFO, msg, end='') + return self.get_input_stream().readline(not self.syncing and blocked) def confirm(self, msg): - msg = '%s [y/n]: ' % msg - self.print(msg, end='') if self.default_confirm: - self.verbose("default confirm: True") + self.verbose("%s and then auto confirm yes" % msg) return True + msg = '%s [y/n]: ' % msg + self.print(msg, end='') if self.isatty() and not self.syncing: while True: try: @@ -697,6 +735,7 @@ def confirm(self, msg): except Exception as e: if not e: return False + self.print(msg, end='') else: self.verbose("isatty: %s, syncing: %s, auto confirm: False" % (self.isatty(), self.syncing)) return False @@ -714,13 +753,26 @@ def _print(self, msg_lv, msg, *args, **kwargs): del kwargs['prev_msg'] else: print_msg = msg - if msg_lv == MsgLevel.ERROR: - kwargs['file'] = self.get_cur_err_obj() + if kwargs.get('_on_exit'): + kwargs['file'] = self.get_exit_buffer() + del kwargs['_on_exit'] + else: + if msg_lv == MsgLevel.ERROR: + kwargs['file'] = self.get_cur_err_obj() + else: + kwargs['file'] = self.get_cur_out_obj() + if '_disable_log' in kwargs: + enaable_log = not kwargs['_disable_log'] + del kwargs['_disable_log'] + else: + enaable_log = True + # if self.silent is True, Not print to stream + if self.silent: + pass else: - kwargs['file'] = self.get_cur_out_obj() - kwargs['file'] and print(self._format(print_msg, *args), **kwargs) + kwargs['file'] and print(self._format(print_msg, *args), **kwargs) del kwargs['file'] - self.log(msg_lv, msg, *args, **kwargs) + enaable_log and self.log(msg_lv, msg, *args, **kwargs) def log(self, levelno, msg, *args, **kwargs): self._cache_log(levelno, msg, *args, **kwargs) @@ -745,6 +797,12 @@ def _log(self, levelno, msg, *args, **kwargs): if self.trace_logger: self.trace_logger.log(levelno, msg, *args, **kwargs) + def _flush_cache(self): + if not self._root_io: + text = self._exit_buffer.read() + if text: + self.print(text, _disable_log=True) + def print(self, msg, *args, **kwargs): self._print(MsgLevel.INFO, msg, *args, **kwargs) @@ -770,16 +828,6 @@ def verbose(self, msg, *args, **kwargs): return self._print(MsgLevel.VERBOSE, '%s %s' % (self._verbose_prefix, msg), *args, **kwargs) - def print_result_json(self, result): - - if not result: - return - if isinstance(result, dict): - result = json.dumps(result, indent=4) - self.print(result) - - pass - if sys.version_info.major == 2: def exception(self, msg='', *args, **kwargs): diff --git a/update/update.py b/update/update.py index 125a230b..c9b73ae6 100644 --- a/update/update.py +++ b/update/update.py @@ -26,6 +26,8 @@ from common.version import OBDIAG_VERSION import yaml +from result_type import ObdiagResult + # for update obdiag files without obdiag class UpdateHandler: @@ -55,17 +57,16 @@ def execute(self): local_update_file_name = os.path.expanduser('~/.obdiag/data.tar') local_update_log_file_name = os.path.expanduser('~/.obdiag/data_version.yaml') if file_path and file_path != "": - self.handle_update_offline(file_path) - return + return self.handle_update_offline(file_path) if NetUtils.network_connectivity(remote_server) is False: self.stdio.warn("[update] network connectivity failed. Please check your network connection.") - return + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="network connectivity failed. Please check your network connection.") NetUtils.download_file(remote_version_file_name, os.path.expanduser(local_version_file_name)) with open(local_version_file_name, 'r') as file: remote_data = yaml.safe_load(file) if remote_data.get("obdiag_version") is None: self.stdio.warn("obdiag_version is None. Do not perform the upgrade process.") - return + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="obdiag_version is None. Do not perform the upgrade process.") else: self.remote_obdiag_version = remote_data["obdiag_version"].strip() if StringUtils.compare_versions_greater(self.remote_obdiag_version, self.local_obdiag_version): @@ -74,10 +75,15 @@ def execute(self): "remote_obdiag_version>local_obdiag_version. Unable to update dependency files, please upgrade " "obdiag. Do not perform the upgrade process.".format(self.remote_obdiag_version, self.local_obdiag_version) ) - return + return ObdiagResult( + ObdiagResult.SERVER_ERROR_CODE, + error_data="remote_obdiag_version is {0}. local_obdiag_version is {1}. " + "remote_obdiag_version>local_obdiag_version. Unable to update dependency files, please upgrade " + "obdiag. Do not perform the upgrade process.".format(self.remote_obdiag_version, self.local_obdiag_version), + ) if remote_data.get("remote_tar_sha") is None: self.stdio.warn("remote_tar_sha is None. Do not perform the upgrade process.") - return + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="remote_tar_sha is None. Do not perform the upgrade process.") else: self.remote_tar_sha = remote_data["remote_tar_sha"] # need update? @@ -88,7 +94,7 @@ def execute(self): local_data = yaml.safe_load(file) if local_data.get("remote_tar_sha") is not None and local_data.get("remote_tar_sha") == self.remote_tar_sha: self.stdio.warn("[update] remote_tar_sha as local_tar_sha. No need to update.") - return + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"msg": "remote_tar_sha as local_tar_sha. No need to update."}) # get data_update_time if local_data.get("data_update_time") is not None and time.time() - local_data["data_update_time"] < 3600 * 24 * 7: self.stdio.warn("[update] data_update_time No need to update.") @@ -123,9 +129,10 @@ def execute(self): with open(os.path.expanduser("~/.obdiag/data_version.yaml"), 'w') as f: yaml.dump({"data_update_time": int(time.time()), "remote_tar_sha": self.remote_tar_sha}, f) self.stdio.print("[update] Successfully updated. The original data is stored in the *. d folder.") - return + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"msg": "Successfully updated. The original data is stored in the *. d folder."}) except Exception as e: self.stdio.warn('[update] Failed to update. Error message: {0}'.format(e)) + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="Failed to update. Error message: {0}".format(e)) def handle_update_offline(self, file): file = os.path.expanduser(file) @@ -133,10 +140,10 @@ def handle_update_offline(self, file): self.local_update_file_sha = FileUtil.calculate_sha256(file) if os.path.exists(file) is False: self.stdio.error('{0} does not exist.'.format(file)) - return + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="{0} does not exist.".format(file)) if not file.endswith('.tar'): self.stdio.error('{0} is not a tar file.'.format(file)) - return + return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="{0} is not a tar file.".format(file)) ## check_old_files if os.path.exists(os.path.expanduser("~/.obdiag/check.d")): shutil.rmtree(os.path.expanduser("~/.obdiag/check.d")) @@ -147,7 +154,6 @@ def handle_update_offline(self, file): shutil.rmtree(os.path.expanduser("~/.obdiag/gather.d")) if os.path.exists(os.path.expanduser("~/.obdiag/gather")): os.rename(os.path.expanduser("~/.obdiag/gather"), os.path.expanduser("~/.obdiag/gather.d")) - ## rca if os.path.exists(os.path.expanduser("~/.obdiag/rca.d")): shutil.rmtree(os.path.expanduser("~/.obdiag/rca.d")) @@ -159,3 +165,4 @@ def handle_update_offline(self, file): with open(os.path.expanduser("~/.obdiag/data_version.yaml"), 'w') as f: yaml.dump({"data_update_time": int(time.time()), "remote_tar_sha": self.remote_tar_sha}, f) self.stdio.print("[update] Successfully updated. The original data is stored in the *. d folder.") + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"msg": "Successfully updated. The original data is stored in the *. d folder."})