From e99056275b153f88d230fdb3d9cd4b0b2e55ee29 Mon Sep 17 00:00:00 2001 From: Teingi Date: Mon, 27 May 2024 11:05:50 +0800 Subject: [PATCH] format code --- cmd.py | 941 ++- common/__init__.py | 2 +- common/command.py | 287 +- common/config_helper.py | 157 +- common/constant.py | 61 +- common/log.py | 4 +- common/ob_connector.py | 26 +- common/ob_log_level.py | 2 +- common/obdiag_exception.py | 45 +- common/ocp/__init__.py | 2 +- common/ocp/ocp_api.py | 1 - common/ocp/ocp_task.py | 18 +- common/scene.py | 79 +- common/ssh.py | 685 +- common/tool.py | 679 +- common/types.py | 165 +- common/version.py | 12 +- config.py | 222 +- context.py | 17 +- core.py | 226 +- dependencies/check_dependencies.py | 5 +- err.py | 20 +- handler/__init__.py | 2 +- handler/analyzer/__init__.py | 2 +- handler/analyzer/analyze_flt_trace.py | 193 +- handler/analyzer/analyze_log.py | 501 +- handler/analyzer/log_parser/__init__.py | 2 +- handler/analyzer/log_parser/log_entry.py | 106 +- handler/analyzer/log_parser/log_enum.py | 16 +- handler/analyzer/log_parser/tree.py | 196 +- handler/checker/__init__.py | 1 - handler/checker/check_exception.py | 15 +- handler/checker/check_handler.py | 180 +- handler/checker/check_list.py | 38 +- handler/checker/check_report.py | 74 +- handler/checker/check_task.py | 48 +- handler/checker/result/__init__.py | 1 - handler/checker/result/result.py | 39 +- handler/checker/result/verify.py | 86 +- handler/checker/step/__init__.py | 1 - handler/checker/step/data_size.py | 28 +- handler/checker/step/get_system_parameter.py | 52 +- handler/checker/step/sql.py | 46 +- handler/checker/step/ssh.py | 42 +- handler/checker/step/stepbase.py | 154 +- handler/gather/__init__.py | 2 +- handler/gather/gather_ash_report.py | 181 +- handler/gather/gather_awr.py | 229 +- handler/gather/gather_log.py | 474 +- handler/gather/gather_obadmin.py | 352 +- handler/gather/gather_obproxy_log.py | 383 +- handler/gather/gather_obstack2.py | 329 +- handler/gather/gather_perf.py | 234 +- handler/gather/gather_plan_monitor.py | 1129 ++- handler/gather/gather_scenes.py | 158 +- handler/gather/gather_sysstat.py | 320 +- handler/gather/scenes/__init__.py | 2 +- handler/gather/scenes/base.py | 82 +- handler/gather/scenes/cpu_high.py | 44 +- handler/gather/scenes/list.py | 44 +- handler/gather/scenes/px_collect_log.py | 115 +- handler/gather/scenes/register.py | 33 +- handler/gather/scenes/sql_problem.py | 43 +- handler/gather/step/__init__.py | 2 +- handler/gather/step/base.py | 123 +- handler/gather/step/sql.py | 52 +- handler/gather/step/ssh.py | 40 +- handler/meta/__init__.py | 2 +- handler/meta/check_meta.py | 8 +- handler/meta/html_meta.py | 16 +- handler/meta/ob_error.py | 6816 +++++++++++++++--- handler/meta/sql_meta.py | 120 +- handler/rca/__init__.py | 1 - handler/rca/plugins/__init__.py | 1 - handler/rca/plugins/gather.py | 72 +- handler/rca/rca_exception.py | 17 +- handler/rca/rca_handler.py | 207 +- handler/rca/rca_list.py | 35 +- handler/rca/scene/ddl_disk_full_scene.py | 268 +- handler/rca/scene/disconnection_scene.py | 123 +- handler/rca/scene/lock_conflict_scene.py | 189 +- handler/rca/scene/log_error_scene.py | 242 +- handler/rca/scene/major_hold_scene.py | 532 +- main.py | 6 +- stdio.py | 346 +- telemetry/__init__.py | 2 +- telemetry/telemetry.py | 81 +- test/analyzer/test_tree.py | 257 +- update/__init__.py | 2 +- update/update.py | 156 +- 90 files changed, 14750 insertions(+), 4599 deletions(-) diff --git a/cmd.py b/cmd.py index 2a225da8..c730d53c 100644 --- a/cmd.py +++ b/cmd.py @@ -30,7 +30,7 @@ from telemetry.telemetry import telemetry ROOT_IO = IO(1) -OBDIAG_HOME_PATH = os.path.join(os.getenv('HOME'), 'oceanbase-diagnostic-tool') +OBDIAG_HOME_PATH = os.path.join(os.getenv("HOME"), "oceanbase-diagnostic-tool") class OptionHelpFormatter(IndentedHelpFormatter): @@ -48,12 +48,13 @@ def format_option(self, option): result.append(opts) if option.help: help_text = self.expand_default(option) - help_lines = help_text.split('\n') + help_lines = help_text.split("\n") if len(help_lines) == 1: help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) - result.extend(["%*s%s\n" % (self.help_position, "", line) - for line in help_lines[1:]]) + result.extend( + ["%*s%s\n" % (self.help_position, "", line) for line in help_lines[1:]] + ) elif opts[-1] != "\n": result.append("\n") return "".join(result) @@ -62,23 +63,33 @@ def format_option(self, option): class AllowUndefinedOptionParser(OptionParser): IS_TTY = sys.stdin.isatty() - def __init__(self, - usage=None, - option_list=None, - option_class=Option, - version=None, - conflict_handler="resolve", - description=None, - formatter=None, - add_help_option=True, - prog=None, - epilog=None, - allow_undefine=True, - undefine_warn=True - ): + def __init__( + self, + usage=None, + option_list=None, + option_class=Option, + version=None, + conflict_handler="resolve", + description=None, + formatter=None, + add_help_option=True, + prog=None, + epilog=None, + allow_undefine=True, + undefine_warn=True, + ): OptionParser.__init__( - self, usage, option_list, option_class, version, conflict_handler, - description, formatter, add_help_option, prog, epilog + self, + usage, + option_list, + option_class, + version, + conflict_handler, + description, + formatter, + add_help_option, + prog, + epilog, ) self.allow_undefine = allow_undefine self.undefine_warn = undefine_warn @@ -87,7 +98,7 @@ def warn(self, msg, file=None): if self.IS_TTY: print("%s %s" % (IO.WARNING_PREV, msg)) else: - print('warn: %s' % msg) + print("warn: %s" % msg) def _process_long_opt(self, rargs, values): try: @@ -96,8 +107,12 @@ def _process_long_opt(self, rargs, values): except BadOptionError as e: if self.allow_undefine: key = e.opt_str - value = value[len(key) + 1:] - setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) + value = value[len(key) + 1 :] + setattr( + values, + key.strip("-").replace("-", "_"), + value if value != "" else True, + ) self.undefine_warn and self.warn(e) else: raise e @@ -109,8 +124,12 @@ def _process_short_opts(self, rargs, values): except BadOptionError as e: if self.allow_undefine: key = e.opt_str - value = value[len(key) + 1:] - setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) + value = value[len(key) + 1 :] + setattr( + values, + key.strip("-").replace("-", "_"), + value if value != "" else True, + ) self.undefine_warn and self.warn(e) else: raise e @@ -124,16 +143,28 @@ def __init__(self, name, summary): self.args = [] self.cmds = [] self.opts = {} - self.prev_cmd = '' + self.prev_cmd = "" self.is_init = False self.hidden = False self.has_trace = True self.parser = AllowUndefinedOptionParser(add_help_option=True) - self.parser.add_option('-h', '--help', action='callback', callback=self._show_help, help='Show help and exit.') - self.parser.add_option('-v', '--verbose', action='callback', callback=self._set_verbose, help='Activate verbose output.') + self.parser.add_option( + "-h", + "--help", + action="callback", + callback=self._show_help, + help="Show help and exit.", + ) + self.parser.add_option( + "-v", + "--verbose", + action="callback", + callback=self._set_verbose, + help="Activate verbose output.", + ) def _set_verbose(self, *args, **kwargs): - ROOT_IO.set_verbose_level(0xfffffff) + ROOT_IO.set_verbose_level(0xFFFFFFF) def init(self, cmd, args): if self.is_init is False: @@ -170,7 +201,7 @@ def enable_log(self): return True def is_valid_time_format(self, time_string): - time_pattern = r'^\d{2}:\d{2}:\d{2}$' + time_pattern = r"^\d{2}:\d{2}:\d{2}$" return bool(re.match(time_pattern, time_string)) def preprocess_argv(self, argv): @@ -182,15 +213,15 @@ def preprocess_argv(self, argv): from_index = None to_index = None for i, arg in enumerate(argv): - if arg == '--from': + if arg == "--from": from_index = i + 1 - elif arg == '--to': + elif arg == "--to": to_index = i + 1 if from_index is not None and i == from_index: next_arg = argv[i + 1] if i + 1 < len(argv) else None if next_arg and self.is_valid_time_format(next_arg): - processed_argv.append(argv[i] + ' ' + next_arg) + processed_argv.append(argv[i] + " " + next_arg) from_index = None i += 1 else: @@ -198,7 +229,7 @@ def preprocess_argv(self, argv): elif to_index is not None and i == to_index: next_arg = argv[i + 1] if i + 1 < len(argv) else None if next_arg and self.is_valid_time_format(next_arg): - processed_argv.append(argv[i] + ' ' + next_arg) + processed_argv.append(argv[i] + " " + next_arg) to_index = None i += 1 else: @@ -219,14 +250,14 @@ def do_command(self): log_directory = os.path.join(os.path.expanduser("~"), ".obdiag", "log") if not os.path.exists(log_directory): os.makedirs(log_directory, exist_ok=True) - log_path = os.path.join(log_directory, 'obdiag.log') + log_path = os.path.join(log_directory, "obdiag.log") if self.enable_log: - ROOT_IO.init_trace_logger(log_path, 'obdiag', trace_id) + ROOT_IO.init_trace_logger(log_path, "obdiag", trace_id) ROOT_IO.track_limit += 1 - ROOT_IO.verbose('cmd: %s' % self.cmds) - ROOT_IO.verbose('opts: %s' % self.opts) - config_path = os.path.expanduser('~/.obdiag/config.yml') - custom_config = Util.get_option(self.opts, 'c') + ROOT_IO.verbose("cmd: %s" % self.cmds) + ROOT_IO.verbose("opts: %s" % self.opts) + config_path = os.path.expanduser("~/.obdiag/config.yml") + custom_config = Util.get_option(self.opts, "c") if custom_config: config_path = custom_config obdiag = ObdiagHome(stdio=ROOT_IO, config_path=config_path) @@ -235,17 +266,20 @@ def do_command(self): ret = self._do_command(obdiag) telemetry.put_data() except NotImplementedError: - ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd) + ROOT_IO.exception("command '%s' is not implemented" % self.prev_cmd) except SystemExit: pass except KeyboardInterrupt: - ROOT_IO.exception('Keyboard Interrupt') + ROOT_IO.exception("Keyboard Interrupt") except: e = sys.exc_info()[1] - ROOT_IO.exception('Running Error: %s' % e) + ROOT_IO.exception("Running Error: %s" % e) if self.has_trace: - ROOT_IO.print('Trace ID: %s' % trace_id) - ROOT_IO.print('If you want to view detailed obdiag logs, please run: obdiag display-trace %s' % trace_id) + ROOT_IO.print("Trace ID: %s" % trace_id) + ROOT_IO.print( + "If you want to view detailed obdiag logs, please run: obdiag display-trace %s" + % trace_id + ) return ret def _do_command(self, obdiag): @@ -253,18 +287,22 @@ def _do_command(self, obdiag): def get_white_ip_list(self): if self.opts.white: - return self.opts.white.split(',') - ROOT_IO.warn("Security Risk: the whitelist is empty and anyone can request this program!") + return self.opts.white.split(",") + ROOT_IO.warn( + "Security Risk: the whitelist is empty and anyone can request this program!" + ) if ROOT_IO.confirm("Do you want to continue?"): return [] wthite_ip_list = ROOT_IO.read("Please enter the whitelist, eq: '192.168.1.1'") - raise wthite_ip_list.split(',') + raise wthite_ip_list.split(",") class DisplayTraceCommand(ObdiagOriginCommand): def __init__(self): - super(DisplayTraceCommand, self).__init__('display-trace', 'display trace_id log.') + super(DisplayTraceCommand, self).__init__( + "display-trace", "display trace_id log." + ) self.has_trace = False @property @@ -273,23 +311,27 @@ def enable_log(self): def _do_command(self, obdiag): from common.ssh import LocalClient + if not self.cmds: return self._show_help() - log_dir = os.path.expanduser('~/.obdiag/log') + log_dir = os.path.expanduser("~/.obdiag/log") trace_id = self.cmds[0] - ROOT_IO.verbose('Get log by trace_id') + ROOT_IO.verbose("Get log by trace_id") try: if UUID(trace_id).version != 1: - ROOT_IO.critical('%s is not trace id' % trace_id) + ROOT_IO.critical("%s is not trace id" % trace_id) return False except: - ROOT_IO.print('%s is not trace id' % trace_id) + ROOT_IO.print("%s is not trace id" % trace_id) return False - cmd = 'cd {} && grep -h "\[{}\]" $(ls -tr {}*) | sed "s/\[{}\] //g" '.format(log_dir, trace_id, log_dir, trace_id) + cmd = 'cd {} && grep -h "\[{}\]" $(ls -tr {}*) | sed "s/\[{}\] //g" '.format( + log_dir, trace_id, log_dir, trace_id + ) data = LocalClient.execute_command(cmd) ROOT_IO.print(data.stdout) return True + class MajorCommand(BaseCommand): def __init__(self, name, summary): @@ -298,21 +340,27 @@ def __init__(self, name, summary): def _mk_usage(self): if self.commands: - usage = ['%s [options]\n\nAvailable commands:\n' % self.prev_cmd] - commands = [x for x in self.commands.values() if not (hasattr(x, 'hidden') and x.hidden)] + usage = ["%s [options]\n\nAvailable commands:\n" % self.prev_cmd] + commands = [ + x + for x in self.commands.values() + if not (hasattr(x, "hidden") and x.hidden) + ] commands.sort(key=lambda x: x.name) for command in commands: if command.hidden is False: usage.append("%-12s %s\n" % (command.name, command.summary)) - self.parser.set_usage('\n'.join(usage)) + self.parser.set_usage("\n".join(usage)) return super(MajorCommand, self)._mk_usage() def do_command(self): if not self.is_init: - ROOT_IO.error('%s command not init' % self.prev_cmd) - raise SystemExit('command not init') + ROOT_IO.error("%s command not init" % self.prev_cmd) + raise SystemExit("command not init") if len(self.args) < 1: - ROOT_IO.print('You need to give some commands.\n\nTry `obdiag --help` for more information.') + ROOT_IO.print( + "You need to give some commands.\n\nTry `obdiag --help` for more information." + ) self._show_help() return False base, args = self.args[0], self.args[1:] @@ -320,11 +368,11 @@ def do_command(self): self.parse_command() self._show_help() return False - cmd = '%s %s' % (self.prev_cmd, base) + cmd = "%s %s" % (self.prev_cmd, base) ROOT_IO.track_limit += 1 if "main.py" in cmd: - telemetry.work_tag=False - telemetry.push_cmd_info("cmd: {0}. args:{1}".format(cmd,args)) + telemetry.work_tag = False + telemetry.push_cmd_info("cmd: {0}. args:{1}".format(cmd, args)) return self.commands[base].init(cmd, args).do_command() def register_command(self, command): @@ -338,191 +386,442 @@ def init(self, cmd, args): return self def __init__(self): - super(ObdiagGatherAllCommand, self).__init__('all', 'Gather oceanbase diagnostic info') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - + super(ObdiagGatherAllCommand, self).__init__( + "all", "Gather oceanbase diagnostic info" + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--scope", + type="string", + help="log type constrains, choices=[observer, election, rootservice, all]", + default="all", + ) + self.parser.add_option( + "--grep", action="append", type="string", help="specify keywords constrain" + ) + self.parser.add_option( + "--encrypt", + type="string", + help="Whether the returned results need to be encrypted, choices=[true, false]", + default="false", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) + def init(self, cmd, args): super(ObdiagGatherAllCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_all', self.opts) + return obdiag.gather_function("gather_all", self.opts) class ObdiagGatherLogCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherLogCommand, self).__init__('log', 'Gather oceanbase logs from oceanbase machines') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherLogCommand, self).__init__( + "log", "Gather oceanbase logs from oceanbase machines" + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--scope", + type="string", + help="log type constrains, choices=[observer, election, rootservice, all]", + default="all", + ) + self.parser.add_option( + "--grep", action="append", type="string", help="specify keywords constrain" + ) + self.parser.add_option( + "--encrypt", + type="string", + help="Whether the returned results need to be encrypted, choices=[true, false]", + default="false", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherLogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_log', self.opts) + return obdiag.gather_function("gather_log", self.opts) + class ObdiagGatherSysStatCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherSysStatCommand, self).__init__('sysstat', 'Gather Host information') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherSysStatCommand, self).__init__( + "sysstat", "Gather Host information" + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherSysStatCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_sysstat', self.opts) + return obdiag.gather_function("gather_sysstat", self.opts) class ObdiagGatherStackCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherStackCommand, self).__init__('stack', 'Gather stack') - - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherStackCommand, self).__init__("stack", "Gather stack") + + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherStackCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_obstack', self.opts) + return obdiag.gather_function("gather_obstack", self.opts) class ObdiagGatherPerfCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherPerfCommand, self).__init__('perf', 'Gather perf') - - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('--scope', type='string', help="perf type constrains, choices=[sample, flame, pstack, all]", default='all') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherPerfCommand, self).__init__("perf", "Gather perf") + + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "--scope", + type="string", + help="perf type constrains, choices=[sample, flame, pstack, all]", + default="all", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherPerfCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_perf', self.opts) + return obdiag.gather_function("gather_perf", self.opts) class ObdiagGatherSlogCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherSlogCommand, self).__init__('slog', 'Gather slog') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherSlogCommand, self).__init__("slog", "Gather slog") + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--encrypt", + type="string", + help="Whether the returned results need to be encrypted, choices=[true, false]", + default="false", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherSlogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_slog', self.opts) + return obdiag.gather_function("gather_slog", self.opts) class ObdiagGatherClogCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherClogCommand, self).__init__('clog', 'Gather clog') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherClogCommand, self).__init__("clog", "Gather clog") + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--encrypt", + type="string", + help="Whether the returned results need to be encrypted, choices=[true, false]", + default="false", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherClogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_clog', self.opts) + return obdiag.gather_function("gather_clog", self.opts) + class ObdiagGatherAwrCommand(ObdiagOriginCommand): - + def __init__(self): - super(ObdiagGatherAwrCommand, self).__init__('awr', 'Gather ParalleSQL information') - self.parser.add_option('--cluster_name', type='string', help='cluster_name from ocp') - self.parser.add_option('--cluster_id', type='string', help='cluster_id from ocp') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherAwrCommand, self).__init__( + "awr", "Gather ParalleSQL information" + ) + self.parser.add_option( + "--cluster_name", type="string", help="cluster_name from ocp" + ) + self.parser.add_option( + "--cluster_id", type="string", help="cluster_id from ocp" + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherAwrCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_awr', self.opts) + return obdiag.gather_function("gather_awr", self.opts) class ObdiagGatherPlanMonitorCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information') - self.parser.add_option('--trace_id', type='string', help='sql trace id') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('--env', type='string', help='''env, eg: "{db_connect='-h127.0.0.1 -P2881 -utest@test -p****** -Dtest'}"''') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherPlanMonitorCommand, self).__init__( + "plan_monitor", "Gather ParalleSQL information" + ) + self.parser.add_option("--trace_id", type="string", help="sql trace id") + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "--env", + type="string", + help='''env, eg: "{db_connect='-h127.0.0.1 -P2881 -utest@test -p****** -Dtest'}"''', + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherPlanMonitorCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_plan_monitor', self.opts) + return obdiag.gather_function("gather_plan_monitor", self.opts) class ObdiagGatherObproxyLogCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherObproxyLogCommand, self).__init__('obproxy_log', 'Gather obproxy log from obproxy machines') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherObproxyLogCommand, self).__init__( + "obproxy_log", "Gather obproxy log from obproxy machines" + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--scope", + type="string", + help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", + default="all", + ) + self.parser.add_option( + "--grep", action="append", type="string", help="specify keywords constrain" + ) + self.parser.add_option( + "--encrypt", + type="string", + help="Whether the returned results need to be encrypted, choices=[true, false]", + default="false", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherObproxyLogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): @@ -530,10 +829,10 @@ def _do_command(self, obdiag): class ObdiagGatherSceneListCommand(ObdiagOriginCommand): - + def __init__(self): - super(ObdiagGatherSceneListCommand, self).__init__('list', 'gather scene list') - + super(ObdiagGatherSceneListCommand, self).__init__("list", "gather scene list") + def init(self, cmd, args): super(ObdiagGatherSceneListCommand, self).init(cmd, args) return self @@ -543,121 +842,251 @@ def _do_command(self, obdiag): class ObdiagGatherSceneRunCommand(ObdiagOriginCommand): - + def __init__(self): - super(ObdiagGatherSceneRunCommand, self).__init__('run', 'gather scene run') - self.parser.add_option('--scene', type='string', help="Specify the scene to be gather") - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') - self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"') - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagGatherSceneRunCommand, self).__init__("run", "gather scene run") + self.parser.add_option( + "--scene", type="string", help="Specify the scene to be gather" + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) + self.parser.add_option( + "--env", type="string", help='env, eg: "{env1=xxx, env2=xxx}"' + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherSceneRunCommand, self).init(cmd, args) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_scenes_run', self.opts) + return obdiag.gather_function("gather_scenes_run", self.opts) class ObdiagGatherAshReportCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagGatherAshReportCommand, self).__init__('ash', 'Gather ash report') - self.parser.add_option('--trace_id', type='string', - help="The TRACE.ID of the SQL to be sampled, if left blank or filled with NULL, indicates that TRACE.ID is not restricted.") - self.parser.add_option('--sql_id', type='string', - help="The SQL.ID, if left blank or filled with NULL, indicates that SQL.ID is not restricted.") + super(ObdiagGatherAshReportCommand, self).__init__("ash", "Gather ash report") + self.parser.add_option( + "--trace_id", + type="string", + help="The TRACE.ID of the SQL to be sampled, if left blank or filled with NULL, indicates that TRACE.ID is not restricted.", + ) + self.parser.add_option( + "--sql_id", + type="string", + help="The SQL.ID, if left blank or filled with NULL, indicates that SQL.ID is not restricted.", + ) # WAIT_CLASS - self.parser.add_option('--wait_class', type='string', - help='Event types to be sampled.') - self.parser.add_option('--report_type', type='string', - help='Report type, currently only supports text type.', default='TEXT') - self.parser.add_option('--from', type='string', - help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', - help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--store_dir', type='string', - help='the dir to store gather result, current dir by default.', default='./') - - self.parser.add_option('-c', type='string', help='obdiag custom config', - default=os.path.expanduser('~/.obdiag/config.yml')) + self.parser.add_option( + "--wait_class", type="string", help="Event types to be sampled." + ) + self.parser.add_option( + "--report_type", + type="string", + help="Report type, currently only supports text type.", + default="TEXT", + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagGatherAshReportCommand, self).init(cmd, args) return self def _do_command(self, obdiag): - return obdiag.gather_function('gather_ash_report', self.opts) + return obdiag.gather_function("gather_ash_report", self.opts) class ObdiagAnalyzeLogCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeLogCommand, self).__init__('log', 'Analyze oceanbase log from online observer machines or offline oceanbase log files') - self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') - self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--log_level', type='string', help="oceanbase logs greater than or equal to this level will be analyze, choices=[DEBUG, TRACE, INFO, WDIAG, WARN, EDIAG, ERROR]") - self.parser.add_option('--files', action="append", type='string', help="specify files") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - self.parser.add_option('--since', type='string',help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') + super(ObdiagAnalyzeLogCommand, self).__init__( + "log", + "Analyze oceanbase log from online observer machines or offline oceanbase log files", + ) + self.parser.add_option( + "--from", + type="string", + help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--to", + type="string", + help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'", + ) + self.parser.add_option( + "--scope", + type="string", + help="log type constrains, choices=[observer, election, rootservice, all]", + default="all", + ) + self.parser.add_option( + "--grep", action="append", type="string", help="specify keywords constrain" + ) + self.parser.add_option( + "--log_level", + type="string", + help="oceanbase logs greater than or equal to this level will be analyze, choices=[DEBUG, TRACE, INFO, WDIAG, WARN, EDIAG, ERROR]", + ) + self.parser.add_option( + "--files", action="append", type="string", help="specify files" + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) + self.parser.add_option( + "--since", + type="string", + help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", + default="30m", + ) def init(self, cmd, args): super(ObdiagAnalyzeLogCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - offline_args_sign = '--files' + offline_args_sign = "--files" if self.args and (offline_args_sign in self.args): - return obdiag.analyze_fuction('analyze_log_offline', self.opts) + return obdiag.analyze_fuction("analyze_log_offline", self.opts) else: - return obdiag.analyze_fuction('analyze_log', self.opts) + return obdiag.analyze_fuction("analyze_log", self.opts) class ObdiagAnalyzeFltTraceCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeFltTraceCommand, self).__init__('flt_trace', 'Analyze oceanbase trace.log from online observer machines or offline oceanbase trace.log files') - self.parser.add_option('--flt_trace_id', type='string', help="flt trace id, . format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") - self.parser.add_option('--files', action="append", help="specify files") - self.parser.add_option('--top', type='string', help="top leaf span", default=5) - self.parser.add_option('--recursion', type='string', help="Maximum number of recursion", default=8) - self.parser.add_option('--output', type='string', help="Print the result to the maximum output line on the screen", default=60) - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagAnalyzeFltTraceCommand, self).__init__( + "flt_trace", + "Analyze oceanbase trace.log from online observer machines or offline oceanbase trace.log files", + ) + self.parser.add_option( + "--flt_trace_id", + type="string", + help="flt trace id, . format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + ) + self.parser.add_option("--files", action="append", help="specify files") + self.parser.add_option("--top", type="string", help="top leaf span", default=5) + self.parser.add_option( + "--recursion", type="string", help="Maximum number of recursion", default=8 + ) + self.parser.add_option( + "--output", + type="string", + help="Print the result to the maximum output line on the screen", + default=60, + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store gather result, current dir by default.", + default="./", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagAnalyzeFltTraceCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_flt_trace', self.opts) + return obdiag.analyze_fuction("analyze_flt_trace", self.opts) class ObdiagCheckCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagCheckCommand, self).__init__('check', 'check oceanbase cluster') - self.parser.add_option('--cases', type='string', help="check observer's cases on package_file") - self.parser.add_option('--obproxy_cases', type='string', help="check obproxy's cases on package_file") - self.parser.add_option('--store_dir', type='string', help='the dir to store check result, current dir by default.', default='./check_report/') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - + super(ObdiagCheckCommand, self).__init__("check", "check oceanbase cluster") + self.parser.add_option( + "--cases", type="string", help="check observer's cases on package_file" + ) + self.parser.add_option( + "--obproxy_cases", + type="string", + help="check obproxy's cases on package_file", + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store check result, current dir by default.", + default="./check_report/", + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagCheckCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): - if 'list' in self.args: + if "list" in self.args: obdiag.check_list(self.opts) return return obdiag.check(self.opts) @@ -666,15 +1095,29 @@ def _do_command(self, obdiag): class ObdiagRCARunCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagRCARunCommand, self).__init__('run', 'root cause analysis') - self.parser.add_option('--scene', type='string', help="rca scene name. The argument is required.") - self.parser.add_option('--store_dir', type='string', help='the dir to store rca result, current dir by default.', default='./rca/') - self.parser.add_option('--input_parameters', type='string', help='input parameters of scene') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + super(ObdiagRCARunCommand, self).__init__("run", "root cause analysis") + self.parser.add_option( + "--scene", type="string", help="rca scene name. The argument is required." + ) + self.parser.add_option( + "--store_dir", + type="string", + help="the dir to store rca result, current dir by default.", + default="./rca/", + ) + self.parser.add_option( + "--input_parameters", type="string", help="input parameters of scene" + ) + self.parser.add_option( + "-c", + type="string", + help="obdiag custom config", + default=os.path.expanduser("~/.obdiag/config.yml"), + ) def init(self, cmd, args): super(ObdiagRCARunCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): @@ -684,11 +1127,11 @@ def _do_command(self, obdiag): class ObdiagRCAListCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagRCAListCommand, self).__init__('list', 'show list of rca list') + super(ObdiagRCAListCommand, self).__init__("list", "show list of rca list") def init(self, cmd, args): super(ObdiagRCAListCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): @@ -696,33 +1139,41 @@ def _do_command(self, obdiag): class ObdiagConfigCommand(ObdiagOriginCommand): - + def __init__(self): - super(ObdiagConfigCommand, self).__init__('config', 'Quick build config') - self.parser.add_option('-h', type='string', help="database host") - self.parser.add_option('-u', type='string', help='sys_user', default='root@sys') - self.parser.add_option('-p', type='string', help="password", default='') - self.parser.add_option('-P', type='string', help="port") + super(ObdiagConfigCommand, self).__init__("config", "Quick build config") + self.parser.add_option("-h", type="string", help="database host") + self.parser.add_option("-u", type="string", help="sys_user", default="root@sys") + self.parser.add_option("-p", type="string", help="password", default="") + self.parser.add_option("-P", type="string", help="port") def init(self, cmd, args): super(ObdiagConfigCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): return obdiag.config(self.opts) + class ObdiagUpdateCommand(ObdiagOriginCommand): - - def __init__(self): - super(ObdiagUpdateCommand, self).__init__('update', 'Update cheat files') - self.parser.add_option('--file', type='string', help="obdiag update cheat file path. Please note that you need to ensure the reliability of the files on your own.") - self.parser.add_option('--force', action='store_true', help='You can force online upgrades by adding --force in the command',) + def __init__(self): + super(ObdiagUpdateCommand, self).__init__("update", "Update cheat files") + self.parser.add_option( + "--file", + type="string", + help="obdiag update cheat file path. Please note that you need to ensure the reliability of the files on your own.", + ) + self.parser.add_option( + "--force", + action="store_true", + help="You can force online upgrades by adding --force in the command", + ) def init(self, cmd, args): super(ObdiagUpdateCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) + self.parser.set_usage("%s [options]" % self.prev_cmd) return self def _do_command(self, obdiag): @@ -732,7 +1183,9 @@ def _do_command(self, obdiag): class ObdiagGatherCommand(MajorCommand): def __init__(self): - super(ObdiagGatherCommand, self).__init__('gather', 'Gather oceanbase diagnostic info') + super(ObdiagGatherCommand, self).__init__( + "gather", "Gather oceanbase diagnostic info" + ) self.register_command(ObdiagGatherAllCommand()) self.register_command(ObdiagGatherLogCommand()) self.register_command(ObdiagGatherSysStatCommand()) @@ -748,9 +1201,11 @@ def __init__(self): class ObdiagGatherSceneCommand(MajorCommand): - + def __init__(self): - super(ObdiagGatherSceneCommand, self).__init__('scene', 'Gather scene diagnostic info') + super(ObdiagGatherSceneCommand, self).__init__( + "scene", "Gather scene diagnostic info" + ) self.register_command(ObdiagGatherSceneListCommand()) self.register_command(ObdiagGatherSceneRunCommand()) @@ -758,7 +1213,9 @@ def __init__(self): class ObdiagAnalyzeCommand(MajorCommand): def __init__(self): - super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info') + super(ObdiagAnalyzeCommand, self).__init__( + "analyze", "Analyze oceanbase diagnostic info" + ) self.register_command(ObdiagAnalyzeLogCommand()) self.register_command(ObdiagAnalyzeFltTraceCommand()) @@ -766,15 +1223,15 @@ def __init__(self): class ObdiagRCACommand(MajorCommand): def __init__(self): - super(ObdiagRCACommand, self).__init__('rca', 'root cause analysis') + super(ObdiagRCACommand, self).__init__("rca", "root cause analysis") self.register_command(ObdiagRCARunCommand()) self.register_command(ObdiagRCAListCommand()) class MainCommand(MajorCommand): - + def __init__(self): - super(MainCommand, self).__init__('obdiag', '') + super(MainCommand, self).__init__("obdiag", "") self.register_command(DisplayTraceCommand()) self.register_command(ObdiagGatherCommand()) self.register_command(ObdiagAnalyzeCommand()) diff --git a/common/__init__.py b/common/__init__.py index 2b33595e..4f2405d9 100644 --- a/common/__init__.py +++ b/common/__init__.py @@ -14,4 +14,4 @@ @time: 2022/6/20 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/common/command.py b/common/command.py index ae1f8c46..351b2b5f 100644 --- a/common/command.py +++ b/common/command.py @@ -30,10 +30,14 @@ def __init__(self, stdio=None): def run(self, cmd): try: self.stdio.verbose("[local host] run cmd = [{0}] on localhost".format(cmd)) - out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) + out = subprocess.Popen( + cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True + ) stdout, stderr = out.communicate() if stderr: - self.stdio.error("run cmd = [{0}] on localhost, stderr=[{1}]".format(cmd, stderr)) + self.stdio.error( + "run cmd = [{0}] on localhost, stderr=[{1}]".format(cmd, stderr) + ) return stdout except: self.stdio.error("run cmd = [{0}] on localhost".format(cmd)) @@ -41,7 +45,9 @@ def run(self, cmd): def run_get_stderr(self, cmd): try: self.stdio.verbose("run cmd = [{0}] on localhost".format(cmd)) - out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) + out = subprocess.Popen( + cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True + ) stdout, stderr = out.communicate() return stderr except: @@ -54,28 +60,56 @@ def __init__(self, stdio=None): def run(self, ssh_helper, cmd): try: - self.stdio.verbose("[remote host {0}] excute cmd = [{1}]".format(ssh_helper.get_name(), cmd)) + self.stdio.verbose( + "[remote host {0}] excute cmd = [{1}]".format( + ssh_helper.get_name(), cmd + ) + ) stdout = ssh_helper.ssh_exec_cmd(cmd) - self.stdio.verbose("[remote host {0}] excute cmd = [{1}] complete, stdout=[{2}]".format(ssh_helper.get_name(), cmd, stdout)) + self.stdio.verbose( + "[remote host {0}] excute cmd = [{1}] complete, stdout=[{2}]".format( + ssh_helper.get_name(), cmd, stdout + ) + ) return stdout except Exception as e: - self.stdio.error("[remote host {0}] excute cmd = [{1}] except: [{2}]".format(ssh_helper.get_name(), cmd, e)) + self.stdio.error( + "[remote host {0}] excute cmd = [{1}] except: [{2}]".format( + ssh_helper.get_name(), cmd, e + ) + ) def run_get_stderr(self, ssh_helper, cmd): try: - self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd)) + self.stdio.verbose( + "[remote host {0}] run cmd = [{1}] start ...".format( + ssh_helper.get_name(), cmd + ) + ) std = ssh_helper.ssh_exec_cmd_get_stderr(cmd) return std except Exception as e: - self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e)) + self.stdio.error( + "[remote host {0}] run ssh cmd = [{1}] except: {2}".format( + ssh_helper.get_name(), cmd, e + ) + ) def run_ignore_err(self, ssh_helper, cmd): try: - self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd)) + self.stdio.verbose( + "[remote host {0}] run cmd = [{1}] start ...".format( + ssh_helper.get_name(), cmd + ) + ) std = ssh_helper.ssh_exec_cmd_ignore_err(cmd) return std except SSHException as e: - self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e)) + self.stdio.error( + "[remote host {0}] run ssh cmd = [{1}] except: {2}".format( + ssh_helper.get_name(), cmd, e + ) + ) def download_file(is_ssh, ssh_helper, remote_path, local_path, stdio=None): @@ -86,7 +120,11 @@ def download_file(is_ssh, ssh_helper, remote_path, local_path, stdio=None): """ try: if is_ssh: - stdio.verbose("Please wait a moment, download file [{0}] from server {1} to [{2}]".format(remote_path, ssh_helper.get_name(),local_path)) + stdio.verbose( + "Please wait a moment, download file [{0}] from server {1} to [{2}]".format( + remote_path, ssh_helper.get_name(), local_path + ) + ) ssh_helper.download(remote_path, local_path) else: cmd = "cp -r {0} {1}".format(remote_path, local_path) @@ -102,8 +140,11 @@ def upload_file(is_ssh, ssh_helper, local_path, remote_path, stdio=None): :param args: is_ssh, ssh helper, local file path, remote file path :return: local path """ - stdio.verbose("Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format( - ssh_helper.get_name(), local_path, remote_path)) + stdio.verbose( + "Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format( + ssh_helper.get_name(), local_path, remote_path + ) + ) try: if is_ssh: ssh_helper.upload(local_path, remote_path) @@ -126,13 +167,14 @@ def rm_rf_file(is_ssh, ssh_helper, dir, stdio=None): else: LocalClient(stdio).run(cmd) + def delete_file_in_folder(is_ssh, ssh_helper, file_path, stdio): """ delete file :param args: is_ssh, ssh helper, file_name :return: """ - if (file_path is None) or (not 'gather_pack' in file_path): + if (file_path is None) or (not "gather_pack" in file_path): raise Exception("Please check file path, {0}".format(file_path)) cmd = "rm -rf {file_path}/*".format(file_path=file_path) if is_ssh: @@ -160,7 +202,7 @@ def is_empty_dir(is_ssh, ssh_helper, dir, stdio=None): def get_file_start_time(is_ssh, ssh_helper, file_name, dir, stdio=None): """ - get log file start time + get log file start time :param args: is_ssh, ssh helper, gather log full path :return: true or false """ @@ -172,41 +214,68 @@ def get_file_start_time(is_ssh, ssh_helper, file_name, dir, stdio=None): return TimeUtils.extract_time_from_log_file_text(str(first_line_text)) -def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_dir, log_files, stdio=None): +def get_logfile_name_list( + is_ssh, ssh_helper, from_time_str, to_time_str, log_dir, log_files, stdio=None +): """ - get log name list + get log name list :param args: is_ssh, ssh helper, from time, to time, log dir, log file list :return: true or false """ - stdio.verbose("get log file name list, from time {0}, to time {1}, log dir {2}, log files {3}".format(from_time_str, to_time_str, log_dir, log_files)) + stdio.verbose( + "get log file name list, from time {0}, to time {1}, log dir {2}, log files {3}".format( + from_time_str, to_time_str, log_dir, log_files + ) + ) log_name_list = [] last_file_dict = {"prefix_file_name": "", "file_name": "", "file_end_time": ""} - for file_name in log_files.split('\n'): + for file_name in log_files.split("\n"): if file_name == "": stdio.verbose("existing file name is empty") continue if not file_name.endswith("log") and not file_name.endswith("wf"): file_start_time_str = "" prefix_name = file_name[:-14] if len(file_name) > 24 else "" - file_end_time_str = TimeUtils.filename_time_to_datetime(TimeUtils.extract_filename_time_from_log_name(file_name, stdio), stdio) - if last_file_dict["prefix_file_name"] != "" and last_file_dict["prefix_file_name"] == prefix_name: + file_end_time_str = TimeUtils.filename_time_to_datetime( + TimeUtils.extract_filename_time_from_log_name(file_name, stdio), stdio + ) + if ( + last_file_dict["prefix_file_name"] != "" + and last_file_dict["prefix_file_name"] == prefix_name + ): file_start_time_str = last_file_dict["file_end_time"] - elif last_file_dict["prefix_file_name"] != "" and last_file_dict["prefix_file_name"] != prefix_name: + elif ( + last_file_dict["prefix_file_name"] != "" + and last_file_dict["prefix_file_name"] != prefix_name + ): file_start_time_str = "" file_end_time_str = "" elif last_file_dict["prefix_file_name"] == "": - file_start_time_str = get_file_start_time(is_ssh, ssh_helper, file_name, log_dir, stdio) + file_start_time_str = get_file_start_time( + is_ssh, ssh_helper, file_name, log_dir, stdio + ) # When two time intervals overlap, need to add the file - if (file_end_time_str != "") and (file_start_time_str != "") and (file_start_time_str <= to_time_str) and ( - file_end_time_str >= from_time_str): + if ( + (file_end_time_str != "") + and (file_start_time_str != "") + and (file_start_time_str <= to_time_str) + and (file_end_time_str >= from_time_str) + ): log_name_list.append(file_name) - last_file_dict = {"prefix_file_name": prefix_name, "file_name": file_name, - "file_end_time": file_end_time_str} + last_file_dict = { + "prefix_file_name": prefix_name, + "file_name": file_name, + "file_end_time": file_end_time_str, + } elif file_name.endswith("log") or file_name.endswith("wf"): # Get the first and last lines of text of the file. Here, use a command - get_first_line_cmd = "head -n 1 {0}/{1} && tail -n 1 {0}/{1}".format(log_dir, file_name) + get_first_line_cmd = "head -n 1 {0}/{1} && tail -n 1 {0}/{1}".format( + log_dir, file_name + ) if is_ssh: - first_and_last_line_text = SshClient(stdio).run(ssh_helper, get_first_line_cmd) + first_and_last_line_text = SshClient(stdio).run( + ssh_helper, get_first_line_cmd + ) else: first_and_last_line_text = LocalClient(stdio).run(get_first_line_cmd) @@ -217,18 +286,39 @@ def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_di last_line_text = first_and_last_line_text_list[-1] # Time to parse the first and last lines of text - file_start_time_str = TimeUtils.extract_time_from_log_file_text(first_line_text, stdio) - file_end_time = TimeUtils.extract_time_from_log_file_text(last_line_text, stdio) - stdio.verbose("The log file {0} starts at {1} ends at {2}".format(file_name, file_start_time_str,file_end_time)) - stdio.verbose("to_time_str {0} from_time_str {1}".format(to_time_str, from_time_str)) - if (file_start_time_str <= to_time_str) and (file_end_time >= from_time_str): + file_start_time_str = TimeUtils.extract_time_from_log_file_text( + first_line_text, stdio + ) + file_end_time = TimeUtils.extract_time_from_log_file_text( + last_line_text, stdio + ) + stdio.verbose( + "The log file {0} starts at {1} ends at {2}".format( + file_name, file_start_time_str, file_end_time + ) + ) + stdio.verbose( + "to_time_str {0} from_time_str {1}".format( + to_time_str, from_time_str + ) + ) + if (file_start_time_str <= to_time_str) and ( + file_end_time >= from_time_str + ): log_name_list.append(file_name) if len(log_name_list) > 0: - stdio.verbose("Find the qualified log file {0} on Server [{1}], " - "wait for the next step".format(log_name_list, "localhost" if not is_ssh else ssh_helper.get_name())) + stdio.verbose( + "Find the qualified log file {0} on Server [{1}], " + "wait for the next step".format( + log_name_list, "localhost" if not is_ssh else ssh_helper.get_name() + ) + ) else: - stdio.warn("No found the qualified log file on Server [{0}]".format( - "localhost" if not is_ssh else ssh_helper.get_name())) + stdio.warn( + "No found the qualified log file on Server [{0}]".format( + "localhost" if not is_ssh else ssh_helper.get_name() + ) + ) return log_name_list @@ -265,8 +355,8 @@ def zip_dir(is_ssh, ssh_helper, father_dir, zip_dir, stdio=None): :return: """ cmd = "cd {father_dir} && zip {zip_dir}.zip -rm {zip_dir}".format( - father_dir=father_dir, - zip_dir=zip_dir) + father_dir=father_dir, zip_dir=zip_dir + ) stdio.verbose("Please wait a moment ...") if is_ssh: SshClient(stdio).run(ssh_helper, cmd) @@ -281,15 +371,15 @@ def zip_encrypt_dir(is_ssh, ssh_helper, zip_password, father_dir, zip_dir, stdio :return: """ cmd = "cd {father_dir} && zip --password {zip_password} {zip_dir}.zip -rm {zip_dir}".format( - zip_password=zip_password, - father_dir=father_dir, - zip_dir=zip_dir) + zip_password=zip_password, father_dir=father_dir, zip_dir=zip_dir + ) stdio.verbose("Please wait a moment ...") if is_ssh: SshClient(stdio).run(ssh_helper, cmd) else: LocalClient(stdio).run(cmd) + def is_support_arch(is_ssh, ssh_helper, stdio=None): """ Determine if it is a supported operating system @@ -319,29 +409,36 @@ def get_observer_version(is_ssh, ssh_helper, ob_install_dir, stdio): :return: """ ob_version = "" - cmd = "{ob_install_dir}/bin/observer --version".format(ob_install_dir=ob_install_dir) + cmd = "{ob_install_dir}/bin/observer --version".format( + ob_install_dir=ob_install_dir + ) if is_ssh: ob_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) else: ob_version_info = LocalClient(stdio).run_get_stderr(cmd) stdio.verbose("get observer version, run cmd = [{0}] ".format(cmd)) if ob_version_info is not None: - ob_version = re.findall(r'[(]OceanBase.(.+?)[)]', ob_version_info) + ob_version = re.findall(r"[(]OceanBase.(.+?)[)]", ob_version_info) if len(ob_version) > 0: - result = re.sub(r'[a-zA-Z]', '', ob_version[0]) + result = re.sub(r"[a-zA-Z]", "", ob_version[0]) return result.strip() else: cmd = "export LD_LIBRARY_PATH={ob_install_dir}/lib && {ob_install_dir}/bin/observer --version".format( - ob_install_dir=ob_install_dir) + ob_install_dir=ob_install_dir + ) if is_ssh: ob_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) else: ob_version_info = LocalClient(stdio).run_get_stderr(cmd) - stdio.verbose("get observer version with LD_LIBRARY_PATH,cmd:{0}".format(cmd)) + stdio.verbose( + "get observer version with LD_LIBRARY_PATH,cmd:{0}".format(cmd) + ) if "REVISION" not in ob_version_info: - raise Exception("Please check conf about observer,{0}".format(ob_version_info)) - ob_version = re.findall(r'[(]OceanBase.*\s(.+?)[)]', ob_version_info) - result = re.sub(r'[a-zA-Z]', '', ob_version[0]) + raise Exception( + "Please check conf about observer,{0}".format(ob_version_info) + ) + ob_version = re.findall(r"[(]OceanBase.*\s(.+?)[)]", ob_version_info) + result = re.sub(r"[a-zA-Z]", "", ob_version[0]) return result.strip() @@ -352,26 +449,35 @@ def get_obproxy_version(is_ssh, ssh_helper, obproxy_install_dir, stdio): :return: """ obproxy_version = "" - cmd = "{obproxy_install_dir}/bin/obproxy --version".format(obproxy_install_dir=obproxy_install_dir) + cmd = "{obproxy_install_dir}/bin/obproxy --version".format( + obproxy_install_dir=obproxy_install_dir + ) if is_ssh: obproxy_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) else: obproxy_version_info = LocalClient(stdio).run_get_stderr(cmd) stdio.verbose("get obproxy version, run cmd = [{0}] ".format(cmd)) if obproxy_version_info is not None: - ob_version = re.findall(r'[(]OceanBase.(.+? +?)[)]', obproxy_version_info) + ob_version = re.findall(r"[(]OceanBase.(.+? +?)[)]", obproxy_version_info) if len(ob_version) > 0: return ob_version[0] else: cmd = "export LD_LIBRARY_PATH={obproxy_install_dir}/lib && {obproxy_install_dir}/bin/obproxy --version".format( - obproxy_install_dir=obproxy_install_dir) + obproxy_install_dir=obproxy_install_dir + ) if is_ssh: obproxy_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) else: obproxy_version_info = LocalClient(stdio).run_get_stderr(cmd) - stdio.verbose("get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format(cmd,obproxy_version_info)) + stdio.verbose( + "get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format( + cmd, obproxy_version_info + ) + ) if "REVISION" not in obproxy_version_info: - raise Exception("Please check conf about proxy,{0}".format(obproxy_version_info)) + raise Exception( + "Please check conf about proxy,{0}".format(obproxy_version_info) + ) pattern = r"(\d+\.\d+\.\d+\.\d+)" match = re.search(pattern, obproxy_version_info) if match: @@ -383,28 +489,38 @@ def get_obproxy_version(is_ssh, ssh_helper, obproxy_install_dir, stdio): obproxy_version_info = match.group(1) obproxy_version_info = obproxy_version_info.split()[0] return obproxy_version_info + + # Only applicable to the community version + def get_observer_version_by_sql(ob_cluster, stdio=None): stdio.verbose("start get_observer_version_by_sql . input: {0}".format(ob_cluster)) try: - ob_connector = OBConnector(ip=ob_cluster.get("db_host"), - port=ob_cluster.get("db_port"), - username=ob_cluster.get("tenant_sys").get("user"), - password=ob_cluster.get("tenant_sys").get("password"), - stdio=stdio, - timeout=100) + ob_connector = OBConnector( + ip=ob_cluster.get("db_host"), + port=ob_cluster.get("db_port"), + username=ob_cluster.get("tenant_sys").get("user"), + password=ob_cluster.get("tenant_sys").get("password"), + stdio=stdio, + timeout=100, + ) ob_version_info = ob_connector.execute_sql("select version();") except Exception as e: - raise Exception("get_observer_version_by_sql Exception. Maybe cluster'info is error: " + e.__str__()) + raise Exception( + "get_observer_version_by_sql Exception. Maybe cluster'info is error: " + + e.__str__() + ) ob_version = ob_version_info[0] - stdio.verbose("get_observer_version_by_sql ob_version_info is {0}".format(ob_version)) - version = re.findall(r'OceanBase(_)?(.CE)?-v(.+)', ob_version[0]) + stdio.verbose( + "get_observer_version_by_sql ob_version_info is {0}".format(ob_version) + ) + version = re.findall(r"OceanBase(_)?(.CE)?-v(.+)", ob_version[0]) if len(version) > 0: return version[0][2] else: - version = re.findall(r'(.+)', ob_version[0]) + version = re.findall(r"(.+)", ob_version[0]) return version[0] @@ -415,13 +531,17 @@ def get_observer_pid(is_ssh, ssh_helper, ob_install_dir, stdio=None): :return: """ try: - cmd = "cat {ob_install_dir}/run/observer.pid".format(ob_install_dir=ob_install_dir) + cmd = "cat {ob_install_dir}/run/observer.pid".format( + ob_install_dir=ob_install_dir + ) if is_ssh: pids = SshClient(stdio).run(ssh_helper, cmd) else: pids = LocalClient(stdio).run(cmd) pid_list = pids.split() - stdio.verbose("get observer pid, run cmd = [{0}], result:{1} ".format(cmd, pid_list)) + stdio.verbose( + "get observer pid, run cmd = [{0}], result:{1} ".format(cmd, pid_list) + ) except: stdio.verbose("get observer pid failed") return [] @@ -447,7 +567,9 @@ def delete_empty_file(is_ssh, ssh_helper, file_path, stdio=None): :param args: is_ssh, ssh helper, file_name :return: """ - cmd = "find {file_path} -name '*' -type f -size 0c | xargs -n 1 rm -f".format(file_path=file_path) + cmd = "find {file_path} -name '*' -type f -size 0c | xargs -n 1 rm -f".format( + file_path=file_path + ) if is_ssh: SshClient(stdio).run(ssh_helper, cmd) else: @@ -512,9 +634,11 @@ def is_empty_file(is_ssh, ssh_helper, file_path, stdio=None): def get_obdiag_display(log_dir, trace_id, stdio=None): - cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id) + cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format( + trace_id, log_dir, trace_id + ) stdout = LocalClient(stdio).run(cmd) - print_stdout = str(stdout).replace('\\n', '\n').replace('\\t', '\t') + print_stdout = str(stdout).replace("\\n", "\n").replace("\\t", "\t") if len(print_stdout) > 0: print(print_stdout) @@ -528,6 +652,7 @@ def uzip_dir_local(uzip_dir, stdio=None): cmd = f"cd {uzip_dir} && unzip *.zip && rm -rf *.zip" LocalClient(stdio).run(cmd) + def analyze_log_get_sqc_addr(uzip_dir, stdio): """ analyze files @@ -535,31 +660,35 @@ def analyze_log_get_sqc_addr(uzip_dir, stdio): :return: ip_port """ cmd = "cd {uzip_dir} && cd ob_log* && grep {key_words} * | grep -oP '{key_words}=\"\\K[^\"]+' | sort | uniq".format( - uzip_dir=uzip_dir, - key_words = "px_obdiag_sqc_addr") + uzip_dir=uzip_dir, key_words="px_obdiag_sqc_addr" + ) stdout = LocalClient(stdio).run(cmd) - sqc_addrs = stdout.decode().strip().split('\n') + sqc_addrs = stdout.decode().strip().split("\n") if len(sqc_addrs) > 0: - if not re.match(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', sqc_addrs[0]): + if not re.match(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", sqc_addrs[0]): return None else: return sqc_addrs[0] else: return None + def find_home_path_by_port(is_ssh, ssh_helper, internal_port_str, stdio): cmd = "ps aux | grep observer | grep 'P {internal_port_str}' | grep -oP '/[^\s]*/bin/observer' ".format( - internal_port_str = internal_port_str) + internal_port_str=internal_port_str + ) if is_ssh: stdout = SshClient(stdio).run(ssh_helper, cmd) else: stdout = LocalClient(stdio).run(cmd) - - str_list = stdout.strip().split('\n') + + str_list = stdout.strip().split("\n") home_path = "" for original_str in str_list: original_str = str(original_str) - if original_str.endswith("/bin/observer") and not original_str.startswith('/[^\s]*'): + if original_str.endswith("/bin/observer") and not original_str.startswith( + "/[^\s]*" + ): home_path = original_str.rstrip("/bin/observer") break stdio.verbose("home_path:{0}".format(home_path)) diff --git a/common/config_helper.py b/common/config_helper.py index dacce797..d841d954 100644 --- a/common/config_helper.py +++ b/common/config_helper.py @@ -35,28 +35,38 @@ def __init__(self, context): self.context = context self.stdio = context.stdio options = self.context.options - self.sys_tenant_user = Util.get_option(options, 'u') - self.sys_tenant_password = Util.get_option(options, 'p') - self.db_host = Util.get_option(options, 'h') - self.db_port = Util.get_option(options, 'P') - self.config_path = os.path.expanduser('~/.obdiag/config.yml') + self.sys_tenant_user = Util.get_option(options, "u") + self.sys_tenant_password = Util.get_option(options, "p") + self.db_host = Util.get_option(options, "h") + self.db_port = Util.get_option(options, "P") + self.config_path = os.path.expanduser("~/.obdiag/config.yml") self.inner_config = self.context.inner_config - self.ob_cluster = {"db_host": self.db_host, "db_port": self.db_port, "tenant_sys": {"password": self.sys_tenant_password, "user": self.sys_tenant_user, }} + self.ob_cluster = { + "db_host": self.db_host, + "db_port": self.db_port, + "tenant_sys": { + "password": self.sys_tenant_password, + "user": self.sys_tenant_user, + }, + } def get_cluster_name(self): ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) obConnetcor = OBConnector( - ip=self.db_host, - port=self.db_port, + ip=self.db_host, + port=self.db_port, username=self.sys_tenant_user, - password=self.sys_tenant_password, + password=self.sys_tenant_password, stdio=self.stdio, - timeout=100) + timeout=100, + ) if ob_version.startswith("3") or ob_version.startswith("2"): sql = "select cluster_name from oceanbase.v$ob_cluster" res = obConnetcor.execute_sql(sql) if len(res) == 0: - self.stdio.error("Failed to get cluster name, please check whether the cluster config correct!!!") + self.stdio.error( + "Failed to get cluster name, please check whether the cluster config correct!!!" + ) else: return res[0][0] else: @@ -64,19 +74,29 @@ def get_cluster_name(self): def get_host_info_list_by_cluster(self): ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) - obConnetcor = OBConnector(ip=self.db_host, - port=self.db_port, - username=self.sys_tenant_user, - password=self.sys_tenant_password, - stdio=self.stdio, - timeout=100) - sql = "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.DBA_OB_SERVERS" - if ob_version.startswith("3") or ob_version.startswith("2") or ob_version.startswith("1"): + obConnetcor = OBConnector( + ip=self.db_host, + port=self.db_port, + username=self.sys_tenant_user, + password=self.sys_tenant_password, + stdio=self.stdio, + timeout=100, + ) + sql = ( + "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.DBA_OB_SERVERS" + ) + if ( + ob_version.startswith("3") + or ob_version.startswith("2") + or ob_version.startswith("1") + ): sql = "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.__all_server" res = obConnetcor.execute_sql(sql) if len(res) == 0: - raise Exception("Failed to get the node from cluster config, " - "please check whether the cluster config correct!!!") + raise Exception( + "Failed to get the node from cluster config, " + "please check whether the cluster config correct!!!" + ) host_info_list = [] for row in res: host_info = OrderedDict() @@ -86,7 +106,9 @@ def get_host_info_list_by_cluster(self): return host_info_list def build_configuration(self): - self.stdio.verbose("Getting all the node information of the cluster, please wait a moment ...") + self.stdio.verbose( + "Getting all the node information of the cluster, please wait a moment ..." + ) all_host_info_list = self.get_host_info_list_by_cluster() self.stdio.verbose("get node list %s", all_host_info_list) all_host_ip_list = [] @@ -104,13 +126,24 @@ def build_configuration(self): ob_cluster_name = self.get_cluster_name() print("\033[33mPlease enter the following configuration !!!\033[0m") global_ssh_username = self.input_with_default("oceanbase host ssh username", "") - global_ssh_password = self.input_password_with_default("oceanbase host ssh password", "") + global_ssh_password = self.input_password_with_default( + "oceanbase host ssh password", "" + ) global_ssh_port = self.input_with_default("oceanbase host ssh_port", "22") - global_home_path = self.input_with_default("oceanbase install home_path", const.OB_INSTALL_DIR_DEFAULT) + global_home_path = self.input_with_default( + "oceanbase install home_path", const.OB_INSTALL_DIR_DEFAULT + ) default_data_dir = os.path.join(global_home_path, "store") - global_data_dir = self.input_with_default("oceanbase data_dir", default_data_dir) - global_redo_dir = self.input_with_default("oceanbase redo_dir", default_data_dir) - tenant_sys_config = {"user": self.sys_tenant_user, "password": self.sys_tenant_password} + global_data_dir = self.input_with_default( + "oceanbase data_dir", default_data_dir + ) + global_redo_dir = self.input_with_default( + "oceanbase redo_dir", default_data_dir + ) + tenant_sys_config = { + "user": self.sys_tenant_user, + "password": self.sys_tenant_password, + } global_config = { "ssh_username": global_ssh_username, "ssh_password": global_ssh_password, @@ -118,7 +151,7 @@ def build_configuration(self): "ssh_key_file": "", "home_path": global_home_path, "data_dir": global_data_dir, - "redo_dir": global_redo_dir + "redo_dir": global_redo_dir, } new_config = { "obcluster": { @@ -126,27 +159,40 @@ def build_configuration(self): "db_host": self.db_host, "db_port": self.db_port, "tenant_sys": tenant_sys_config, - "servers": { - "nodes": nodes_config, - "global": global_config - }}} + "servers": {"nodes": nodes_config, "global": global_config}, + } + } YamlUtils.write_yaml_data(new_config, self.config_path) - need_config_obproxy = self.input_choice_default("need config obproxy [y/N]", "N") + need_config_obproxy = self.input_choice_default( + "need config obproxy [y/N]", "N" + ) if need_config_obproxy: self.build_obproxy_configuration(self.config_path) - self.stdio.verbose("Node information has been rewritten to the configuration file {0}, and you can enjoy the journey !".format(self.config_path)) + self.stdio.verbose( + "Node information has been rewritten to the configuration file {0}, and you can enjoy the journey !".format( + self.config_path + ) + ) def build_obproxy_configuration(self, path): - obproxy_servers = self.input_with_default("obproxy server eg:'192.168.1.1;192.168.1.2;192.168.1.3'", "") + obproxy_servers = self.input_with_default( + "obproxy server eg:'192.168.1.1;192.168.1.2;192.168.1.3'", "" + ) obproxy_server_list = StringUtils.split_ip(obproxy_servers) if len(obproxy_server_list) > 0: nodes_config = [] for server in obproxy_server_list: nodes_config.append({"ip": server}) - global_ssh_username = self.input_with_default("obproxy host ssh username", "") - global_ssh_password = self.input_password_with_default("obproxy host ssh password", "") + global_ssh_username = self.input_with_default( + "obproxy host ssh username", "" + ) + global_ssh_password = self.input_password_with_default( + "obproxy host ssh password", "" + ) global_ssh_port = self.input_with_default("obproxy host ssh port", "22") - global_home_path = self.input_with_default("obproxy install home_path", const.OBPROXY_INSTALL_DIR_DEFAULT) + global_home_path = self.input_with_default( + "obproxy install home_path", const.OBPROXY_INSTALL_DIR_DEFAULT + ) global_config = { "ssh_username": global_ssh_username, "ssh_password": global_ssh_password, @@ -157,10 +203,9 @@ def build_obproxy_configuration(self, path): new_config = { "obproxy": { "obproxy_cluster_name": "obproxy", - "servers": { - "nodes": nodes_config, - "global": global_config - }}} + "servers": {"nodes": nodes_config, "global": global_config}, + } + } YamlUtils.write_yaml_data_append(new_config, path) def get_old_configuration(self, path): @@ -171,29 +216,41 @@ def get_old_configuration(self, path): pass def save_old_configuration(self, config): - backup_config_dir = os.path.expanduser(self.inner_config["obdiag"]["basic"]["config_backup_dir"]) - filename = "config_backup_{0}.yml".format(TimeUtils.timestamp_to_filename_time(int(round(time.time() * 1000000)))) + backup_config_dir = os.path.expanduser( + self.inner_config["obdiag"]["basic"]["config_backup_dir"] + ) + filename = "config_backup_{0}.yml".format( + TimeUtils.timestamp_to_filename_time(int(round(time.time() * 1000000))) + ) backup_config_path = os.path.join(backup_config_dir, filename) DirectoryUtil.mkdir(path=backup_config_dir) YamlUtils.write_yaml_data(config, backup_config_path) def input_with_default(self, prompt, default): - value = input("\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default)).strip() - if value == '' or value.lower() == "y" or value.lower() == "yes": + value = input( + "\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default) + ).strip() + if value == "" or value.lower() == "y" or value.lower() == "yes": return default else: return value def input_password_with_default(self, prompt, default): - value = pwinput.pwinput(prompt="\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default), - mask='*') - if value == '' or value.lower() == "y" or value.lower() == "yes": + value = pwinput.pwinput( + prompt="\033[32mEnter your {0} (default:'{1}'): \033[0m".format( + prompt, default + ), + mask="*", + ) + if value == "" or value.lower() == "y" or value.lower() == "yes": return default else: return value def input_choice_default(self, prompt, default): - value = input("\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default)).strip() + value = input( + "\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default) + ).strip() if value.lower() == "y" or value.lower() == "yes": return True else: diff --git a/common/constant.py b/common/constant.py index 4f7b9755..1141690a 100644 --- a/common/constant.py +++ b/common/constant.py @@ -42,7 +42,7 @@ def __setattr__(self, name, value): const.GATHER_LOG_TEMPORARY_DIR_DEFAULT = "/tmp" -const.OBSTACK2_DEFAULT_INSTALL_PATH = '/tmp/obstack_x86_64_7' +const.OBSTACK2_DEFAULT_INSTALL_PATH = "/tmp/obstack_x86_64_7" const.OBSTACK2_LOCAL_STORED_PATH = "./dependencies/bin/obstack_x86_64_7" # 限制收集任务的并发线程数量 10 @@ -54,7 +54,7 @@ def __setattr__(self, name, value): const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN = "4.0.0" -const.DEFAULT_CONFIG_PATH = os.path.join(os.path.expanduser('~'), ".obdiag/config.yml") +const.DEFAULT_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".obdiag/config.yml") const.FLT_TRACE_TREE_MAX_RECURSION = 5 const.FLT_TRACE_TREE_TOP_LEAF = 5 @@ -62,40 +62,35 @@ def __setattr__(self, name, value): const.FLT_TRACE_OUTPUT = 50 const.OBDIAG_BASE_DEFAULT_CONFIG = { - "obdiag": { - "basic": { - "config_path": "~/.obdiag/config.yml", - "config_backup_dir": "~/.obdiag/backup_conf", - "file_number_limit": 20, - "file_size_limit": "2G" - }, - "logger": { - "file_handler_log_level": "DEBUG", - "log_dir": "~/.obdiag/log", - "log_filename": "obdiag.log", - "log_level": "INFO", - "mode": "obdiag", - "stdout_handler_log_level": "INFO" + "obdiag": { + "basic": { + "config_path": "~/.obdiag/config.yml", + "config_backup_dir": "~/.obdiag/backup_conf", + "file_number_limit": 20, + "file_size_limit": "2G", + }, + "logger": { + "file_handler_log_level": "DEBUG", + "log_dir": "~/.obdiag/log", + "log_filename": "obdiag.log", + "log_level": "INFO", + "mode": "obdiag", + "stdout_handler_log_level": "INFO", + }, } - } } const.OBDIAG_CHECK_DEFAULT_CONFIG = { - "check": { - "ignore_version": "false", - "report": { - "report_path": "./check_report/", - "export_type": "table" - }, - "package_file": "~/.obdiag/check/check_package.yaml", - "tasks_base_path": "~/.obdiag/check/tasks/" - } + "check": { + "ignore_version": "false", + "report": {"report_path": "./check_report/", "export_type": "table"}, + "package_file": "~/.obdiag/check/check_package.yaml", + "tasks_base_path": "~/.obdiag/check/tasks/", + } } const.OBDIAG_GATHER_DEFAULT_CONFIG = { - "gather": { - "cases_base_path": "~/.obdiag/gather/tasks" - } + "gather": {"cases_base_path": "~/.obdiag/gather/tasks"} } const.OBDIAG_RCA_DEFAULT_CONFIG = { @@ -107,7 +102,7 @@ def __setattr__(self, name, value): const.TELEMETRY_CONTENT_REPORTER = "obdiag" const.TELEMETRY_URL = "openwebapi.oceanbase.com" const.TELEMETRY_PATH = "/api/web/oceanbase/report" -const.UPDATE_REMOTE_SERVER = 'https://obbusiness-private.oss-cn-shanghai.aliyuncs.com' -const.UPDATE_REMOTE_VERSION_FILE_NAME = 'https://obbusiness-private.oss-cn-shanghai.aliyuncs.com/download-center/opensource/obdiag/version.yaml' -const.UPDATE_REMOTE_UPDATE_FILE_NAME = 'https://obbusiness-private.oss-cn-shanghai.aliyuncs.com/download-center/opensource/obdiag/data.tar' -const.RCA_WORK_PATH= '~/.obdiag/rca' \ No newline at end of file +const.UPDATE_REMOTE_SERVER = "https://obbusiness-private.oss-cn-shanghai.aliyuncs.com" +const.UPDATE_REMOTE_VERSION_FILE_NAME = "https://obbusiness-private.oss-cn-shanghai.aliyuncs.com/download-center/opensource/obdiag/version.yaml" +const.UPDATE_REMOTE_UPDATE_FILE_NAME = "https://obbusiness-private.oss-cn-shanghai.aliyuncs.com/download-center/opensource/obdiag/data.tar" +const.RCA_WORK_PATH = "~/.obdiag/rca" diff --git a/common/log.py b/common/log.py index 4a98b80e..80ad0ca8 100644 --- a/common/log.py +++ b/common/log.py @@ -26,5 +26,5 @@ def __init__(self, name, level=logging.DEBUG): self.buffer = [] self.buffer_size = 0 - def _log(self, level, msg, args, end='\n', **kwargs): - return super(Logger, self)._log(level, msg, args, **kwargs) \ No newline at end of file + def _log(self, level, msg, args, end="\n", **kwargs): + return super(Logger, self)._log(level, msg, args, **kwargs) diff --git a/common/ob_connector.py b/common/ob_connector.py index 948644a5..95b87c71 100644 --- a/common/ob_connector.py +++ b/common/ob_connector.py @@ -20,7 +20,16 @@ class OBConnector(object): - def __init__(self, ip, port, username, password=None, database=None, stdio=None, timeout=30,): + def __init__( + self, + ip, + port, + username, + password=None, + database=None, + stdio=None, + timeout=30, + ): self.ip = str(ip) self.port = int(port) self.username = str(username) @@ -49,20 +58,25 @@ def _connect_db(self): ) self.stdio.verbose("connect databse ...") except mysql.Error as e: - self.stdio.error("connect OB: {0}:{1} with user {2} failed, error:{3}".format(self.ip, self.port, self.username, e)) + self.stdio.error( + "connect OB: {0}:{1} with user {2} failed, error:{3}".format( + self.ip, self.port, self.username, e + ) + ) return try: - ob_trx_timeout=self.timeout*1000000 + ob_trx_timeout = self.timeout * 1000000 self.execute_sql("SET SESSION ob_trx_timeout={0};".format(ob_trx_timeout)) except Exception as e: self.stdio.warn("set ob_trx_timeout failed, error:{0}".format(e)) try: - ob_query_timeout=self.timeout*1000000 - self.execute_sql("SET SESSION ob_query_timeout={0};".format(ob_query_timeout)) + ob_query_timeout = self.timeout * 1000000 + self.execute_sql( + "SET SESSION ob_query_timeout={0};".format(ob_query_timeout) + ) except Exception as e: self.stdio.warn("set ob_query_timeout failed, error:{0}".format(e)) - def execute_sql(self, sql): if self.conn is None: self._connect_db() diff --git a/common/ob_log_level.py b/common/ob_log_level.py index f5c07c6a..844f91ae 100644 --- a/common/ob_log_level.py +++ b/common/ob_log_level.py @@ -16,6 +16,7 @@ @desc: """ + class OBLogLevel(object): CRITICAL = 50 FATAL = 50 @@ -49,4 +50,3 @@ def get_log_level(self, level_str): return self.DEBUG else: return self.NOTSET - \ No newline at end of file diff --git a/common/obdiag_exception.py b/common/obdiag_exception.py index 3779f2cc..5dd0d433 100644 --- a/common/obdiag_exception.py +++ b/common/obdiag_exception.py @@ -27,7 +27,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -38,7 +41,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -49,7 +55,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -60,7 +69,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -71,7 +83,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -82,7 +97,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -93,7 +111,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -104,7 +125,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -115,7 +139,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) diff --git a/common/ocp/__init__.py b/common/ocp/__init__.py index 2b33595e..4f2405d9 100644 --- a/common/ocp/__init__.py +++ b/common/ocp/__init__.py @@ -14,4 +14,4 @@ @time: 2022/6/20 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/common/ocp/ocp_api.py b/common/ocp/ocp_api.py index 6c63c8f2..e6a5fc16 100644 --- a/common/ocp/ocp_api.py +++ b/common/ocp/ocp_api.py @@ -35,4 +35,3 @@ # task task = "/api/v2/tasks/instances" - diff --git a/common/ocp/ocp_task.py b/common/ocp/ocp_task.py index 96c33ef6..856b8b5e 100644 --- a/common/ocp/ocp_task.py +++ b/common/ocp/ocp_task.py @@ -32,9 +32,7 @@ class Task: STATUS_TASK_PENDING = "PENDING" STATUS_TASK_SUCCESS = "SUCCESSFUL" STATUS_TASK_FAILED = "FAILED" - TASK_RETRY_PERMIT = { - "Task retry": 3 - } + TASK_RETRY_PERMIT = {"Task retry": 3} def __init__(self, url, auth, task_id): self.url = url @@ -77,7 +75,9 @@ def wait_done(self, interval=20): try: self.get() except Exception as e: - logger.warning("task(%s:%s) get statsu failed, error:%s" % (self.id, self.name, e)) + logger.warning( + "task(%s:%s) get statsu failed, error:%s" % (self.id, self.name, e) + ) break if self.status == Task.STATUS_TASK_SUCCESS: logger.info("task(%s:%s) success" % (self.id, self.name)) @@ -88,12 +88,16 @@ def wait_done(self, interval=20): else: self.retry() self._retry_times += 1 - logger.warning('task %s failed,but allowed retry %s times, now retry %s time' % ( - self.name, self._retry_permit_time, self._retry_times)) + logger.warning( + "task %s failed,but allowed retry %s times, now retry %s time" + % (self.name, self._retry_permit_time, self._retry_times) + ) time.sleep(interval) continue else: logger.info( - "task(%s:%s) not finished yet, now status %s, waiting" % (self.id, self.name, self.status)) + "task(%s:%s) not finished yet, now status %s, waiting" + % (self.id, self.name, self.status) + ) time.sleep(interval) return self.status diff --git a/common/scene.py b/common/scene.py index c9c4016c..6369f6d2 100644 --- a/common/scene.py +++ b/common/scene.py @@ -17,7 +17,11 @@ """ from common.ssh import SshHelper from common.tool import StringUtils -from common.command import get_observer_version, get_obproxy_version, get_observer_version_by_sql +from common.command import ( + get_observer_version, + get_obproxy_version, + get_observer_version_by_sql, +) def filter_by_version(scene, cluster, stdio=None): @@ -31,10 +35,22 @@ def filter_by_version(scene, cluster, stdio=None): if "version" in now_steps: steps_versions = now_steps["version"] if not isinstance(steps_versions, str): - stdio.exception("filter_by_version steps_version Exception : {0}".format("the type of version is not string")) - raise Exception("filter_by_version steps_version Exception : {0}".format("the type of version is not string")) + stdio.exception( + "filter_by_version steps_version Exception : {0}".format( + "the type of version is not string" + ) + ) + raise Exception( + "filter_by_version steps_version Exception : {0}".format( + "the type of version is not string" + ) + ) version_real = cluster["version"] - stdio.verbose("version_int is {0} steps_versions is {1}".format(version_real, steps_versions)) + stdio.verbose( + "version_int is {0} steps_versions is {1}".format( + version_real, steps_versions + ) + ) steps_versions = steps_versions.replace(" ", "") steps_versions = steps_versions[1:-1] @@ -46,7 +62,9 @@ def filter_by_version(scene, cluster, stdio=None): minVersion = "-1" if maxVersion == "*": maxVersion = "999" - if StringUtils.compare_versions_greater(version_real, minVersion) and StringUtils.compare_versions_greater(maxVersion, version_real): + if StringUtils.compare_versions_greater( + version_real, minVersion + ) and StringUtils.compare_versions_greater(maxVersion, version_real): break else: stdio.verbose("not version in now_steps") @@ -60,18 +78,31 @@ def filter_by_version(scene, cluster, stdio=None): stdio.exception("filter_by_version Exception : {0}".format(e)) raise Exception("filter_by_version Exception : {0}".format(e)) -def get_version(nodes, type,cluster, stdio=None): + +def get_version(nodes, type, cluster, stdio=None): try: if len(nodes) < 1: raise Exception("input nodes is empty, please check your config") node = nodes[0] - ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + ssh = SshHelper( + True, + node.get("ip"), + node.get("ssh_username"), + node.get("ssh_password"), + node.get("ssh_port"), + node.get("ssh_key_file"), + node, + ) version = "" if type == "observer": try: - version = get_observer_version_by_sql(cluster,stdio) + version = get_observer_version_by_sql(cluster, stdio) except Exception as e: - stdio.warn("get observer version by sql fail, use node ssher to get. Exception:{0}".format(e)) + stdio.warn( + "get observer version by sql fail, use node ssher to get. Exception:{0}".format( + e + ) + ) version = get_observer_version(True, ssh, nodes[0]["home_path"], stdio) elif type == "obproxy": version = get_obproxy_version(True, ssh, nodes[0]["home_path"], stdio) @@ -80,24 +111,42 @@ def get_version(nodes, type,cluster, stdio=None): stdio.exception("can't get version, Exception: {0}".format(e)) raise Exception("can't get version, Exception: {0}".format(e)) + def get_obproxy_and_ob_version(obproxy_nodes, nodes, type, stdio=None): try: if type == "observer" or type == "other": if len(nodes) < 1: raise Exception("input nodes is empty, please check your config") node = nodes[0] - ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + ssh = SshHelper( + True, + node.get("ip"), + node.get("ssh_username"), + node.get("ssh_password"), + node.get("ssh_port"), + node.get("ssh_key_file"), + node, + ) version = get_observer_version(True, ssh, nodes[0]["home_path"], stdio) elif type == "obproxy": if len(nodes) < 1: - raise Exception("input obproxy nodes is empty, please check your config") + raise Exception( + "input obproxy nodes is empty, please check your config" + ) node = obproxy_nodes[0] - ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + ssh = SshHelper( + True, + node.get("ip"), + node.get("ssh_username"), + node.get("ssh_password"), + node.get("ssh_port"), + node.get("ssh_key_file"), + node, + ) version = get_obproxy_version(True, ssh, nodes[0]["home_path"], stdio) else: - raise Exception( - "type is {0} . No func to get the version".format(type)) + raise Exception("type is {0} . No func to get the version".format(type)) return version except Exception as e: stdio.exception("can't get version, Exception: {0}".format(e)) - raise Exception("can't get version, Exception: {0}".format(e)) \ No newline at end of file + raise Exception("can't get version, Exception: {0}".format(e)) diff --git a/common/ssh.py b/common/ssh.py index f58b8994..cb4683f0 100644 --- a/common/ssh.py +++ b/common/ssh.py @@ -42,6 +42,7 @@ from stdio import SafeStdio from err import EC_SSH_CONNECT from subprocess32 import Popen, PIPE + warnings.filterwarnings("ignore") @@ -50,7 +51,15 @@ class SshConfig(object): - def __init__(self, host, username='root', password=None, key_filename=None, port=22, timeout=30): + def __init__( + self, + host, + username="root", + password=None, + key_filename=None, + port=22, + timeout=30, + ): self.host = host self.username = username self.password = password if password is None else str(password) @@ -59,7 +68,7 @@ def __init__(self, host, username='root', password=None, key_filename=None, port self.timeout = int(timeout) def __str__(self): - return '%s@%s' % (self.username ,self.host) + return "%s@%s" % (self.username, self.host) class SshReturn(object): @@ -71,10 +80,10 @@ def __init__(self, code, stdout, stderr): def __bool__(self): return self.code == 0 - + def __nonzero__(self): return self.__bool__() - + class FeatureSshReturn(SshReturn, SafeStdio): @@ -91,21 +100,21 @@ def _get_return(self): try: p = self.popen output, error = p.communicate(timeout=self.timeout) - self._stdout = output.decode(errors='replace') - self._stderr = error.decode(errors='replace') + self._stdout = output.decode(errors="replace") + self._stderr = error.decode(errors="replace") self._code = p.returncode - verbose_msg = 'exited code %s' % self._code + verbose_msg = "exited code %s" % self._code if self._code: - verbose_msg += ', error output:\n%s' % self._stderr + verbose_msg += ", error output:\n%s" % self._stderr self.stdio.verbose(verbose_msg) except Exception as e: - self._stdout = '' + self._stdout = "" self._stderr = str(e) self._code = 255 - verbose_msg = 'exited code 255, error output:\n%s' % self._stderr + verbose_msg = "exited code 255, error output:\n%s" % self._stderr self.stdio.verbose(verbose_msg) - self.stdio.exception('') - + self.stdio.exception("") + @property def code(self): self._get_return() @@ -132,7 +141,7 @@ def __init__(self, client, command, timeout=None, stdio=None): if self.stdio: self.stdio = self.stdio.sub_io() self.finsh = False - super(FutureSshReturn, self).__init__(127, '', '') + super(FutureSshReturn, self).__init__(127, "", "") def set_return(self, ssh_return): self.code = ssh_return.code @@ -158,7 +167,9 @@ def size(self): @staticmethod def execute(future): client = SshClient(future.client.config, future.stdio) - future.set_return(client.execute_command(future.command, timeout=future.timeout)) + future.set_return( + client.execute_command(future.command, timeout=future.timeout) + ) return future def submit(self): @@ -175,7 +186,7 @@ def submit(self): class LocalClient(SafeStdio): - + @staticmethod def init_env(env=None): if env is None: @@ -186,69 +197,87 @@ def init_env(env=None): @staticmethod def execute_command_background(command, env=None, timeout=None, stdio=None): - stdio.verbose('local background execute: %s ' % command, end='') + stdio.verbose("local background execute: %s " % command, end="") try: - p = Popen(command, env=LocalClient.init_env(env), shell=True, stdout=PIPE, stderr=PIPE) + p = Popen( + command, + env=LocalClient.init_env(env), + shell=True, + stdout=PIPE, + stderr=PIPE, + ) return FeatureSshReturn(p, timeout, stdio) except Exception as e: - output = '' + output = "" error = str(e) code = 255 - verbose_msg = 'exited code 255, error output:\n%s' % error + verbose_msg = "exited code 255, error output:\n%s" % error stdio.verbose(verbose_msg) - stdio.exception('') + stdio.exception("") return SshReturn(code, output, error) - @staticmethod def execute_command(command, env=None, timeout=None, stdio=None): - stdio.verbose('local execute: %s ' % command, end='') + stdio.verbose("local execute: %s " % command, end="") try: - p = Popen(command, env=LocalClient.init_env(env), shell=True, stdout=PIPE, stderr=PIPE) + p = Popen( + command, + env=LocalClient.init_env(env), + shell=True, + stdout=PIPE, + stderr=PIPE, + ) output, error = p.communicate(timeout=timeout) code = p.returncode - output = output.decode(errors='replace') - error = error.decode(errors='replace') - verbose_msg = 'exited code %s' % code + output = output.decode(errors="replace") + error = error.decode(errors="replace") + verbose_msg = "exited code %s" % code if code: - verbose_msg += ', error output:\n%s' % error + verbose_msg += ", error output:\n%s" % error stdio.verbose(verbose_msg) except Exception as e: - output = '' + output = "" error = str(e) code = 255 - verbose_msg = 'exited code 255, error output:\n%s' % error + verbose_msg = "exited code 255, error output:\n%s" % error stdio.verbose(verbose_msg) - stdio.exception('') + stdio.exception("") return SshReturn(code, output, error) @staticmethod def put_file(local_path, remote_path, stdio=None): - if LocalClient.execute_command('mkdir -p %s && cp -f %s %s' % (os.path.dirname(remote_path), local_path, remote_path), stdio=stdio): + if LocalClient.execute_command( + "mkdir -p %s && cp -f %s %s" + % (os.path.dirname(remote_path), local_path, remote_path), + stdio=stdio, + ): return True return False @staticmethod def put_dir(local_dir, remote_dir, stdio=None): if os.path.isdir(local_dir): - local_dir = os.path.join(local_dir, '*') + local_dir = os.path.join(local_dir, "*") if os.path.exists(os.path.dirname(local_dir)) and not glob(local_dir): stdio.verbose("%s is empty" % local_dir) return True - if LocalClient.execute_command('mkdir -p %s && cp -frL %s %s' % (remote_dir, local_dir, remote_dir), stdio=stdio): + if LocalClient.execute_command( + "mkdir -p %s && cp -frL %s %s" % (remote_dir, local_dir, remote_dir), + stdio=stdio, + ): return True return False @staticmethod - def write_file(content, file_path, mode='w', stdio=None): - stdio.verbose('write {} to {}'.format(content, file_path)) + def write_file(content, file_path, mode="w", stdio=None): + stdio.verbose("write {} to {}".format(content, file_path)) try: with FileUtil.open(file_path, mode, stdio=stdio) as f: f.write(content) f.flush() return True except: - stdio.exception('') + stdio.exception("") return False @staticmethod @@ -260,23 +289,37 @@ def get_dir(local_path, remote_path, stdio=None): return LocalClient.put_dir(remote_path, local_path, stdio=stdio) @staticmethod - def run_command(command, env=None, timeout=None, print_stderr=True, elimit=0, olimit=0, stdio=None): - stdio.verbose('local execute: %s ' % command) + def run_command( + command, + env=None, + timeout=None, + print_stderr=True, + elimit=0, + olimit=0, + stdio=None, + ): + stdio.verbose("local execute: %s " % command) stdout = "" process = None try: with Timeout(timeout): - process = Popen(command, env=LocalClient.init_env(env), shell=True, stdout=PIPE, stderr=PIPE) + process = Popen( + command, + env=LocalClient.init_env(env), + shell=True, + stdout=PIPE, + stderr=PIPE, + ) while process.poll() is None: lines = process.stdout.readline() line = lines.strip() if line: - stdio.print(line.decode("utf8", 'ignore')) - stderr = process.stderr.read().decode("utf8", 'ignore') + stdio.print(line.decode("utf8", "ignore")) + stderr = process.stderr.read().decode("utf8", "ignore") code = process.returncode - verbose_msg = 'exit code {}'.format(code) + verbose_msg = "exit code {}".format(code) if code != 0 and stderr: - verbose_msg += ', error output:\n' + verbose_msg += ", error output:\n" stdio.verbose(verbose_msg) if print_stderr: stdio.print(stderr) @@ -287,17 +330,18 @@ def run_command(command, env=None, timeout=None, print_stderr=True, elimit=0, ol except Exception as e: if process: process.terminate() - stdout = '' + stdout = "" stderr = str(e) code = 255 - verbose_msg = 'exited code 255, error output:\n%s' % stderr + verbose_msg = "exited code 255, error output:\n%s" % stderr stdio.verbose(verbose_msg) - stdio.exception('') + stdio.exception("") finally: if process: process.terminate() return SshReturn(code, stdout, stderr) + class RemoteTransporter(enum.Enum): CLIENT = 0 RSYNC = 1 @@ -311,8 +355,8 @@ def __gt__(self, other): class SshClient(SafeStdio): - DEFAULT_PATH = '/sbin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:' - LOCAL_HOST = ['127.0.0.1', 'localhost', '127.1', '127.0.1'] + DEFAULT_PATH = "/sbin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:" + LOCAL_HOST = ["127.0.0.1", "localhost", "127.1", "127.0.1"] DISABLED_ALGORITHMS = dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]) def __init__(self, config, stdio=None): @@ -321,7 +365,7 @@ def __init__(self, config, stdio=None): self.sftp = None self.is_connected = False self.ssh_client = SSHClient() - self.env_str = '' + self.env_str = "" self._remote_transporter = None self.task_queue = None self.result_queue = None @@ -329,7 +373,7 @@ def __init__(self, config, stdio=None): if self._is_local: self.env = {} else: - self.env = {'PATH': self.DEFAULT_PATH} + self.env = {"PATH": self.DEFAULT_PATH} self._update_env() self._disabled_rsa_algorithms = None @@ -343,18 +387,24 @@ def _update_env(self): env = [] for key in self.env: if self.env[key]: - env.append('export %s=%s$%s;' % (key, self.env[key], key)) - self.env_str = ''.join(env) + env.append("export %s=%s$%s;" % (key, self.env[key], key)) + self.env_str = "".join(env) def add_env(self, key, value, rewrite=False, stdio=None): if key not in self.env or not self.env[key] or rewrite: - stdio.verbose('%s@%s set env %s to \'%s\'' % (self.config.username, self.config.host, key, value)) + stdio.verbose( + "%s@%s set env %s to '%s'" + % (self.config.username, self.config.host, key, value) + ) if self._is_local: self._add_env_for_local(key, value, rewrite) else: self.env[key] = value else: - stdio.verbose('%s@%s append \'%s\' to %s' % (self.config.username, self.config.host, value, key)) + stdio.verbose( + "%s@%s append '%s' to %s" + % (self.config.username, self.config.host, value, key) + ) if self._is_local: self._add_env_for_local(key, value, rewrite) else: @@ -366,20 +416,22 @@ def _add_env_for_local(self, key, value, rewrite=False): self.env[key] = value else: if key not in self.env: - self.env[key] = COMMAND_ENV.get(key, '') + self.env[key] = COMMAND_ENV.get(key, "") self.env[key] += value def get_env(self, key, stdio=None): return self.env[key] if key in self.env else None - def del_env(self, key, stdio=None): + def del_env(self, key, stdio=None): if key in self.env: - stdio.verbose('%s@%s delete env %s' % (self.config.username, self.config.host, key)) + stdio.verbose( + "%s@%s delete env %s" % (self.config.username, self.config.host, key) + ) del self.env[key] self._update_env() def __str__(self): - return '%s@%s:%d' % (self.config.username, self.config.host, self.config.port) + return "%s@%s:%d" % (self.config.username, self.config.host, self.config.port) def is_localhost(self, stdio=None): return self.config.host in self.LOCAL_HOST @@ -390,7 +442,15 @@ def _login(self, stdio=None, exit=True): err = None try: self.ssh_client.set_missing_host_key_policy(AutoAddPolicy()) - stdio.verbose('host: %s, port: %s, user: %s, password: %s' % (self.config.host, self.config.port, self.config.username, self.config.password)) + stdio.verbose( + "host: %s, port: %s, user: %s, password: %s" + % ( + self.config.host, + self.config.port, + self.config.username, + self.config.password, + ) + ) self.ssh_client.connect( self.config.host, port=self.config.port, @@ -398,18 +458,26 @@ def _login(self, stdio=None, exit=True): password=self.config.password, key_filename=self.config.key_filename, timeout=self.config.timeout, - disabled_algorithms=self._disabled_rsa_algorithms + disabled_algorithms=self._disabled_rsa_algorithms, ) self.is_connected = True except AuthenticationException: - stdio.exception('') - err = EC_SSH_CONNECT.format(user=self.config.username, ip=self.config.host, message='username or password error') + stdio.exception("") + err = EC_SSH_CONNECT.format( + user=self.config.username, + ip=self.config.host, + message="username or password error", + ) except NoValidConnectionsError: - stdio.exception('') - err = EC_SSH_CONNECT.format(user=self.config.username, ip=self.config.host, message='time out') + stdio.exception("") + err = EC_SSH_CONNECT.format( + user=self.config.username, ip=self.config.host, message="time out" + ) except BaseException as e: - stdio.exception('') - err = EC_SSH_CONNECT.format(user=self.config.username, ip=self.config.host, message=e) + stdio.exception("") + err = EC_SSH_CONNECT.format( + user=self.config.username, ip=self.config.host, message=e + ) if err: if exit: stdio.critical(err) @@ -452,33 +520,38 @@ def __del__(self): def _execute_command(self, command, timeout=None, retry=3, stdio=None): if not self._login(stdio): - return SshReturn(255, '', 'connect failed') + return SshReturn(255, "", "connect failed") try: - stdin, stdout, stderr = self.ssh_client.exec_command(command, timeout=timeout) - output = stdout.read().decode(errors='replace') - error = stderr.read().decode(errors='replace') + stdin, stdout, stderr = self.ssh_client.exec_command( + command, timeout=timeout + ) + output = stdout.read().decode(errors="replace") + error = stderr.read().decode(errors="replace") if output: - idx = output.rindex('\n') + idx = output.rindex("\n") code = int(output[idx:]) stdout = output[:idx] - verbose_msg = 'exited code %s' % code + verbose_msg = "exited code %s" % code else: - code, stdout = 1, '' + code, stdout = 1, "" if code: - verbose_msg = 'exited code %s, error output:\n%s' % (code, error) + verbose_msg = "exited code %s, error output:\n%s" % (code, error) stdio.verbose(verbose_msg) except SSHException as e: if retry: self.close() - return self._execute_command(command, retry-1, stdio) + return self._execute_command(command, retry - 1, stdio) else: - stdio.exception('') - stdio.critical('%s@%s connect failed: %s' % (self.config.username, self.config.host, e)) + stdio.exception("") + stdio.critical( + "%s@%s connect failed: %s" + % (self.config.username, self.config.host, e) + ) raise e except Exception as e: - stdio.exception('') + stdio.exception("") code = 255 - stdout = '' + stdout = "" error = str(e) return SshReturn(code, stdout, error) @@ -489,11 +562,16 @@ def execute_command(self, command, timeout=None, stdio=None): timeout = None if self._is_local: - return LocalClient.execute_command(command, self.env if self.env else None, timeout, stdio=stdio) + return LocalClient.execute_command( + command, self.env if self.env else None, timeout, stdio=stdio + ) - verbose_msg = '%s execute: %s ' % (self.config, command) - stdio.verbose(verbose_msg, end='') - command = '(%s %s);echo -e "\n$?\c"' % (self.env_str, command.strip(';').lstrip('\n')) + verbose_msg = "%s execute: %s " % (self.config, command) + stdio.verbose(verbose_msg, end="") + command = '(%s %s);echo -e "\n$?\c"' % ( + self.env_str, + command.strip(";").lstrip("\n"), + ) return self._execute_command(command, retry=3, timeout=timeout, stdio=stdio) @property @@ -503,16 +581,20 @@ def remote_transporter(self): _transporter = RemoteTransporter.CLIENT if not self._is_local and self._remote_transporter is None: if not self.config.password and not self.disable_rsync: - ret = LocalClient.execute_command('rsync -h', stdio=self.stdio) and self.execute_command('rsync -h', stdio=self.stdio) + ret = LocalClient.execute_command( + "rsync -h", stdio=self.stdio + ) and self.execute_command("rsync -h", stdio=self.stdio) if ret: _transporter = RemoteTransporter.RSYNC self._remote_transporter = _transporter - self.stdio.verbose("current remote_transporter {}".format(self._remote_transporter)) + self.stdio.verbose( + "current remote_transporter {}".format(self._remote_transporter) + ) return self._remote_transporter def put_file(self, local_path, remote_path, stdio=None): if not os.path.isfile(local_path): - stdio.error('path: %s is not file' % local_path) + stdio.error("path: %s is not file" % local_path) return False if self._is_local: return LocalClient.put_file(local_path, remote_path, stdio=stdio) @@ -520,20 +602,20 @@ def put_file(self, local_path, remote_path, stdio=None): return False return self._put_file(local_path, remote_path, stdio=stdio) - def write_file(self, content, file_path, mode='w', stdio=None): + def write_file(self, content, file_path, mode="w", stdio=None): if self._is_local: return LocalClient.write_file(content, file_path, mode, stdio) return self._write_file(content, file_path, mode, stdio) - def _write_file(self, content, file_path, mode='w', stdio=None): - stdio.verbose('write {} to {}: {}'.format(content, self, file_path)) + def _write_file(self, content, file_path, mode="w", stdio=None): + stdio.verbose("write {} to {}: {}".format(content, self, file_path)) try: with tempfile.NamedTemporaryFile(mode=mode) as f: f.write(content) f.flush() return self.put_file(f.name, file_path, stdio=stdio) except: - stdio.exception('') + stdio.exception("") return False @property @@ -544,43 +626,57 @@ def _put_file(self): return self._client_put_file def _client_put_file(self, local_path, remote_path, stdio=None): - if self.execute_command('mkdir -p %s && rm -fr %s' % (os.path.dirname(remote_path), remote_path), stdio=stdio): - stdio.verbose('send %s to %s' % (local_path, remote_path)) - if self.sftp.put(local_path.replace('~', os.getenv('HOME')), remote_path.replace('~', os.getenv('HOME'))): - return self.execute_command('chmod %s %s' % (oct(os.stat(local_path).st_mode)[-3:], remote_path)) + if self.execute_command( + "mkdir -p %s && rm -fr %s" % (os.path.dirname(remote_path), remote_path), + stdio=stdio, + ): + stdio.verbose("send %s to %s" % (local_path, remote_path)) + if self.sftp.put( + local_path.replace("~", os.getenv("HOME")), + remote_path.replace("~", os.getenv("HOME")), + ): + return self.execute_command( + "chmod %s %s" % (oct(os.stat(local_path).st_mode)[-3:], remote_path) + ) return False def _rsync(self, source, target, stdio=None): identity_option = "" if self.config.key_filename: - identity_option += '-i {key_filename} '.format(key_filename=self.config.key_filename) + identity_option += "-i {key_filename} ".format( + key_filename=self.config.key_filename + ) if self.config.port: - identity_option += '-p {}'.format(self.config.port) + identity_option += "-p {}".format(self.config.port) cmd = 'yes | rsync -a -W -e "ssh {identity_option}" {source} {target}'.format( - identity_option=identity_option, - source=source, - target=target + identity_option=identity_option, source=source, target=target ) ret = LocalClient.execute_command(cmd, stdio=stdio) return bool(ret) def _rsync_put_dir(self, local_path, remote_path, stdio=None): - stdio.verbose('send %s to %s by rsync' % (local_path, remote_path)) - source = os.path.join(local_path, '*') + stdio.verbose("send %s to %s by rsync" % (local_path, remote_path)) + source = os.path.join(local_path, "*") if os.path.exists(os.path.dirname(source)) and not glob(source): stdio.verbose("%s is empty" % source) return True - target = "{user}@{host}:{remote_path}".format(user=self.config.username, host=self.config.host, remote_path=remote_path) + target = "{user}@{host}:{remote_path}".format( + user=self.config.username, host=self.config.host, remote_path=remote_path + ) if self._rsync(source, target, stdio=stdio): return True else: return False def _rsync_put_file(self, local_path, remote_path, stdio=None): - if not self.execute_command('mkdir -p %s' % os.path.dirname(remote_path), stdio=stdio): + if not self.execute_command( + "mkdir -p %s" % os.path.dirname(remote_path), stdio=stdio + ): return False - stdio.verbose('send %s to %s by rsync' % (local_path, remote_path)) - target = "{user}@{host}:{remote_path}".format(user=self.config.username, host=self.config.host, remote_path=remote_path) + stdio.verbose("send %s to %s by rsync" % (local_path, remote_path)) + target = "{user}@{host}:{remote_path}".format( + user=self.config.username, host=self.config.host, remote_path=remote_path + ) if self._rsync(local_path, target, stdio=stdio): return True else: @@ -591,11 +687,11 @@ def put_dir(self, local_dir, remote_dir, stdio=None): return LocalClient.put_dir(local_dir, remote_dir, stdio=stdio) if not self._open_sftp(stdio=stdio): return False - if not self.execute_command('mkdir -p %s' % remote_dir, stdio=stdio): + if not self.execute_command("mkdir -p %s" % remote_dir, stdio=stdio): return False - stdio.start_loading('Send %s to %s' % (local_dir, remote_dir)) + stdio.start_loading("Send %s to %s" % (local_dir, remote_dir)) ret = self._put_dir(local_dir, remote_dir, stdio=stdio) - stdio.stop_loading('succeed' if ret else 'fail') + stdio.stop_loading("succeed" if ret else "fail") return ret @property @@ -607,24 +703,30 @@ def _put_dir(self): def _client_put_dir(self, local_dir, remote_dir, stdio=None): has_failed = False - ret = LocalClient.execute_command('find -L %s -type f' % local_dir) + ret = LocalClient.execute_command("find -L %s -type f" % local_dir) if not ret: has_failed = True - all_files = ret.stdout.strip().split('\n') if ret.stdout else [] - ret = LocalClient.execute_command('find %s -type d' % local_dir) + all_files = ret.stdout.strip().split("\n") if ret.stdout else [] + ret = LocalClient.execute_command("find %s -type d" % local_dir) if not ret: has_failed = True - all_dirs = ret.stdout.strip().split('\n') if ret.stdout else [] + all_dirs = ret.stdout.strip().split("\n") if ret.stdout else [] self._filter_dir_in_file_path(all_files, all_dirs) for local_path in all_files: - remote_path = os.path.join(remote_dir, os.path.relpath(local_path, local_dir)) + remote_path = os.path.join( + remote_dir, os.path.relpath(local_path, local_dir) + ) if not self._client_put_file(local_path, remote_path, stdio=stdio): - stdio.error('Fail to get %s' % remote_path) + stdio.error("Fail to get %s" % remote_path) has_failed = True for local_path in all_dirs: - remote_path = os.path.join(remote_dir, os.path.relpath(local_path, local_dir)) + remote_path = os.path.join( + remote_dir, os.path.relpath(local_path, local_dir) + ) stat = oct(os.stat(local_path).st_mode)[-3:] - cmd = '[ -d "{remote_path}" ] || (mkdir -p {remote_path}; chmod {stat} {remote_path})'.format(remote_path=remote_path, stat=stat) + cmd = '[ -d "{remote_path}" ] || (mkdir -p {remote_path}; chmod {stat} {remote_path})'.format( + remote_path=remote_path, stat=stat + ) if not self.execute_command(cmd): has_failed = True return not has_failed @@ -636,12 +738,12 @@ def get_file(self, local_path, remote_path, stdio=None): local_path = os.path.join(dirname, local_path) if os.path.exists(dirname): if not os.path.isdir(dirname): - stdio.error('%s is not directory' % dirname) + stdio.error("%s is not directory" % dirname) return False elif not DirectoryUtil.mkdir(dirname, stdio=stdio): return False if os.path.exists(local_path) and not os.path.isfile(local_path): - stdio.error('path: %s is not file' % local_path) + stdio.error("path: %s is not file" % local_path) return False if self._is_local: return LocalClient.get_file(local_path, remote_path, stdio=stdio) @@ -657,20 +759,26 @@ def _get_file(self): return self._client_get_file def _rsync_get_dir(self, local_path, remote_path, stdio=None): - source = "{user}@{host}:{remote_path}".format(user=self.config.username, host=self.config.host, remote_path=remote_path) + source = "{user}@{host}:{remote_path}".format( + user=self.config.username, host=self.config.host, remote_path=remote_path + ) if "*" not in remote_path: source = os.path.join(source, "*") target = local_path - stdio.verbose('get %s from %s by rsync' % (local_path, remote_path)) - if LocalClient.execute_command('mkdir -p {}'.format(local_path), stdio=stdio) and self._rsync(source, target, stdio=stdio): + stdio.verbose("get %s from %s by rsync" % (local_path, remote_path)) + if LocalClient.execute_command( + "mkdir -p {}".format(local_path), stdio=stdio + ) and self._rsync(source, target, stdio=stdio): return True else: return False def _rsync_get_file(self, local_path, remote_path, stdio=None): - source = "{user}@{host}:{remote_path}".format(user=self.config.username, host=self.config.host, remote_path=remote_path) + source = "{user}@{host}:{remote_path}".format( + user=self.config.username, host=self.config.host, remote_path=remote_path + ) target = local_path - stdio.verbose('get %s from %s by rsync' % (local_path, remote_path)) + stdio.verbose("get %s from %s by rsync" % (local_path, remote_path)) if self._rsync(source, target, stdio=stdio): return True else: @@ -683,7 +791,10 @@ def _client_get_file(self, local_path, remote_path, stdio=None): os.chmod(local_path, stat.st_mode) return True except Exception as e: - stdio.exception('get %s from %s@%s:%s failed: %s' % (local_path, self.config.username, self.config.host, remote_path, e)) + stdio.exception( + "get %s from %s@%s:%s failed: %s" + % (local_path, self.config.username, self.config.host, remote_path, e) + ) return False def get_dir(self, local_dir, remote_dir, stdio=None): @@ -692,24 +803,24 @@ def get_dir(self, local_dir, remote_dir, stdio=None): dirname = os.getcwd() local_dir = os.path.join(dirname, local_dir) if "*" in dirname: - stdio.error('Invalid directory {}'.format(dirname)) + stdio.error("Invalid directory {}".format(dirname)) return False if os.path.exists(dirname): if not os.path.isdir(dirname): - stdio.error('%s is not directory' % dirname) + stdio.error("%s is not directory" % dirname) return False elif not DirectoryUtil.mkdir(dirname, stdio=stdio): return False if os.path.exists(local_dir) and not os.path.isdir(local_dir): - stdio.error('%s is not directory' % local_dir) + stdio.error("%s is not directory" % local_dir) return False if self._is_local: return LocalClient.get_dir(local_dir, remote_dir, stdio=stdio) if not self._open_sftp(stdio=stdio): return False - stdio.start_loading('Get %s from %s' % (local_dir, remote_dir)) + stdio.start_loading("Get %s from %s" % (local_dir, remote_dir)) ret = self._get_dir(local_dir, remote_dir, stdio=stdio) - stdio.stop_loading('succeed' if ret else 'fail') + stdio.stop_loading("succeed" if ret else "fail") return ret @property @@ -724,15 +835,15 @@ def _client_get_dir(self, local_dir, remote_dir, stdio=None): has_failed = False if DirectoryUtil.mkdir(local_dir, stdio=stdio): try: - ret = self.execute_command('find %s -type f' % remote_dir) + ret = self.execute_command("find %s -type f" % remote_dir) if not ret: stdio.verbose(ret.stderr) has_failed = True - all_files = ret.stdout.strip().split('\n') if ret.stdout else [] - ret = self.execute_command('find %s -type d' % remote_dir) + all_files = ret.stdout.strip().split("\n") if ret.stdout else [] + ret = self.execute_command("find %s -type d" % remote_dir) if not ret: has_failed = True - all_dirs = ret.stdout.strip().split('\n') if ret.stdout else [] + all_dirs = ret.stdout.strip().split("\n") if ret.stdout else [] self._filter_dir_in_file_path(all_files, all_dirs) for f in all_files: task_queue.append(f) @@ -741,22 +852,28 @@ def _client_get_dir(self, local_dir, remote_dir, stdio=None): else: remote_base_dir = remote_dir for remote_path in task_queue: - local_path = os.path.join(local_dir, os.path.relpath(remote_path, remote_dir)) + local_path = os.path.join( + local_dir, os.path.relpath(remote_path, remote_dir) + ) if not self._client_get_file(local_path, remote_path, stdio=stdio): - stdio.error('Fail to get %s' % remote_path) + stdio.error("Fail to get %s" % remote_path) has_failed = True for remote_path in all_dirs: try: - local_path = os.path.join(local_dir, os.path.relpath(remote_path, remote_base_dir)) + local_path = os.path.join( + local_dir, os.path.relpath(remote_path, remote_base_dir) + ) if not os.path.exists(local_path): stat = self.sftp.stat(remote_path) os.makedirs(local_path, mode=stat.st_mode) except Exception as e: - stdio.exception('Fail to make directory %s in local: %s' % (remote_path, e)) + stdio.exception( + "Fail to make directory %s in local: %s" % (remote_path, e) + ) has_failed = True return not has_failed except Exception as e: - stdio.exception('Fail to get %s: %s' % (remote_dir, e)) + stdio.exception("Fail to get %s: %s" % (remote_dir, e)) @staticmethod def _filter_dir_in_file_path(files, directories): @@ -778,16 +895,18 @@ def file_downloader(self, local_dir, remote_dir, stdio=None): client._remote_transporter = self.remote_transporter while True: remote_path = self.task_queue.get(block=False) - local_path = os.path.join(local_dir, os.path.relpath(remote_path, remote_dir)) + local_path = os.path.join( + local_dir, os.path.relpath(remote_path, remote_dir) + ) if client.get_file(local_path, remote_path, stdio=stdio): self.result_queue.put(remote_path) else: - stdio.error('Fail to get %s' % remote_path) + stdio.error("Fail to get %s" % remote_path) except Empty: return except: stdio.exception("") - stdio.exception('Failed to get %s' % remote_dir) + stdio.exception("Failed to get %s" % remote_dir) def file_uploader(self, local_dir, remote_dir, stdio=None): try: @@ -795,35 +914,54 @@ def file_uploader(self, local_dir, remote_dir, stdio=None): client._remote_transporter = self.remote_transporter while True: local_path, is_dir = self.task_queue.get(block=False) - remote_path = os.path.join(remote_dir, os.path.relpath(local_path, local_dir)) + remote_path = os.path.join( + remote_dir, os.path.relpath(local_path, local_dir) + ) if is_dir: stat = oct(os.stat(local_path).st_mode)[-3:] - cmd = '[ -d "{remote_path}" ] || (mkdir -p {remote_path}; chmod {stat} {remote_path})'.format(remote_path=remote_path, stat=stat) + cmd = '[ -d "{remote_path}" ] || (mkdir -p {remote_path}; chmod {stat} {remote_path})'.format( + remote_path=remote_path, stat=stat + ) if client.execute_command(cmd): self.result_queue.put(remote_path) else: if client.put_file(local_path, remote_path, stdio=stdio): self.result_queue.put(remote_path) else: - stdio.error('Fail to get %s' % remote_path) + stdio.error("Fail to get %s" % remote_path) except Empty: return except: stdio.exception("") - stdio.verbose('Failed to get %s' % remote_dir) + stdio.verbose("Failed to get %s" % remote_dir) + + # TODO ENV_DISABLE_RSA_ALGORITHMS need get by context.inner_context -ENV_DISABLE_RSA_ALGORITHMS=0 +ENV_DISABLE_RSA_ALGORITHMS = 0 + + def dis_rsa_algorithms(state=0): """ Disable RSA algorithms in OpenSSH server. """ global ENV_DISABLE_RSA_ALGORITHMS - ENV_DISABLE_RSA_ALGORITHMS=state + ENV_DISABLE_RSA_ALGORITHMS = state + + class SshHelper(object): - def __init__(self, is_ssh=None, host_ip=None, username=None, password=None, ssh_port=None, key_file=None, - node=None, stdio=None): + def __init__( + self, + is_ssh=None, + host_ip=None, + username=None, + password=None, + ssh_port=None, + key_file=None, + node=None, + stdio=None, + ): if node is None: - node={} + node = {} self.is_ssh = is_ssh self.stdio = stdio self.host_ip = host_ip @@ -832,24 +970,40 @@ def __init__(self, is_ssh=None, host_ip=None, username=None, password=None, ssh_ self.need_password = True self.password = node.get("ssh_password") or password self.key_file = node.get("ssh_key_file") or key_file - self.key_file=os.path.expanduser(self.key_file) + self.key_file = os.path.expanduser(self.key_file) self.ssh_type = node.get("ssh_type") or "remote" self._ssh_fd = None self._sftp_client = None if "ssh_type" in node and node.get("ssh_type") == "docker": try: self.ssh_type = node["ssh_type"] - self.stdio.verbose("use ssh_type:{0} , node info : {1}".format(self.ssh_type, StringUtils.node_cut_passwd_for_log(node))) + self.stdio.verbose( + "use ssh_type:{0} , node info : {1}".format( + self.ssh_type, StringUtils.node_cut_passwd_for_log(node) + ) + ) self.node = node # docker_permissions_check if self.ssh_type == "docker": self.client = docker.from_env() if "container_name" not in node: - self.stdio.error("SshHelper init docker Exception: 'container_name' not in node") - raise Exception("SshHelper init docker Exception: 'container_name' not in node") + self.stdio.error( + "SshHelper init docker Exception: 'container_name' not in node" + ) + raise Exception( + "SshHelper init docker Exception: 'container_name' not in node" + ) else: - self.stdio.error("SshHelper init not support the ssh_type : {0}".format(self.ssh_type)) - raise Exception("SshHelper init not support the ssh_type : {0}".format(self.ssh_type)) + self.stdio.error( + "SshHelper init not support the ssh_type : {0}".format( + self.ssh_type + ) + ) + raise Exception( + "SshHelper init not support the ssh_type : {0}".format( + self.ssh_type + ) + ) except Exception as e: self.stdio.error("SshHelper init docker Exception: {0}".format(e)) @@ -858,7 +1012,7 @@ def __init__(self, is_ssh=None, host_ip=None, username=None, password=None, ssh_ return if self.is_ssh: - self._disabled_rsa_algorithms=None + self._disabled_rsa_algorithms = None DISABLED_ALGORITHMS = dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]) if ENV_DISABLE_RSA_ALGORITHMS == 1: self._disabled_rsa_algorithms = DISABLED_ALGORITHMS @@ -866,26 +1020,60 @@ def __init__(self, is_ssh=None, host_ip=None, username=None, password=None, ssh_ if len(self.key_file) > 0: try: self._ssh_fd = paramiko.SSHClient() - self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) + self._ssh_fd.set_missing_host_key_policy( + paramiko.client.AutoAddPolicy() + ) self._ssh_fd.load_system_host_keys() - self._ssh_fd.connect(hostname=host_ip, username=username, key_filename=self.key_file, port=ssh_port,disabled_algorithms=self._disabled_rsa_algorithms) + self._ssh_fd.connect( + hostname=host_ip, + username=username, + key_filename=self.key_file, + port=ssh_port, + disabled_algorithms=self._disabled_rsa_algorithms, + ) except AuthenticationException: - self.password = input("Authentication failed, Input {0}@{1} password:\n".format(username, host_ip)) + self.password = input( + "Authentication failed, Input {0}@{1} password:\n".format( + username, host_ip + ) + ) self.need_password = True - self._ssh_fd.connect(hostname=host_ip, username=username, password=password, port=ssh_port,disabled_algorithms=self._disabled_rsa_algorithms) + self._ssh_fd.connect( + hostname=host_ip, + username=username, + password=password, + port=ssh_port, + disabled_algorithms=self._disabled_rsa_algorithms, + ) except Exception as e: - raise OBDIAGSSHConnException("ssh {0}@{1}: failed, exception:{2}".format(username, host_ip, e)) + raise OBDIAGSSHConnException( + "ssh {0}@{1}: failed, exception:{2}".format( + username, host_ip, e + ) + ) else: self._ssh_fd = paramiko.SSHClient() - self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) + self._ssh_fd.set_missing_host_key_policy( + paramiko.client.AutoAddPolicy() + ) self._ssh_fd.load_system_host_keys() self.need_password = True - self._ssh_fd.connect(hostname=host_ip, username=username, password=password, port=ssh_port,disabled_algorithms=self._disabled_rsa_algorithms) + self._ssh_fd.connect( + hostname=host_ip, + username=username, + password=password, + port=ssh_port, + disabled_algorithms=self._disabled_rsa_algorithms, + ) def ssh_exec_cmd(self, cmd): if self.ssh_type == "docker": try: - self.stdio.verbose("ssh_exec_cmd docker {0} cmd: {1}".format(self.node.get("container_name"), cmd)) + self.stdio.verbose( + "ssh_exec_cmd docker {0} cmd: {1}".format( + self.node.get("container_name"), cmd + ) + ) client_result = self.client.containers.get(self.node["container_name"]) result = client_result.exec_run( cmd=["bash", "-c", cmd], @@ -894,25 +1082,38 @@ def ssh_exec_cmd(self, cmd): stderr=True, ) if result.exit_code != 0: - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " - "command=[{1}], exception:{2}".format(self.node["container_name"], cmd, - result.output.decode('utf-8'))) + raise OBDIAGShellCmdException( + "Execute Shell command on server {0} failed, " + "command=[{1}], exception:{2}".format( + self.node["container_name"], + cmd, + result.output.decode("utf-8"), + ) + ) except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) + self.stdio.error( + "sshHelper ssh_exec_cmd docker Exception: {0}".format(e) + ) + raise Exception( + "sshHelper ssh_exec_cmd docker Exception: {0}".format(e) + ) - return result.output.decode('utf-8') + return result.output.decode("utf-8") try: stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) err_text = stderr.read() if len(err_text): - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " - "command=[{1}], exception:{2}".format(self.host_ip, cmd, err_text)) + raise OBDIAGShellCmdException( + "Execute Shell command on server {0} failed, " + "command=[{1}], exception:{2}".format(self.host_ip, cmd, err_text) + ) except SSHException as e: - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " - "command=[{1}], exception:{2}".format(self.host_ip, cmd, e)) - return stdout.read().decode('utf-8') + raise OBDIAGShellCmdException( + "Execute Shell command on server {0} failed, " + "command=[{1}], exception:{2}".format(self.host_ip, cmd, e) + ) + return stdout.read().decode("utf-8") def ssh_exec_cmd_ignore_err(self, cmd): if self.ssh_type == "docker": @@ -925,16 +1126,24 @@ def ssh_exec_cmd_ignore_err(self, cmd): stderr=True, ) except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) + self.stdio.error( + "sshHelper ssh_exec_cmd docker Exception: {0}".format(e) + ) + raise Exception( + "sshHelper ssh_exec_cmd docker Exception: {0}".format(e) + ) - return result.output.decode('utf-8') + return result.output.decode("utf-8") try: stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - return stdout.read().decode('utf-8') + return stdout.read().decode("utf-8") except SSHException as e: - print("Execute Shell command on server {0} failed,command=[{1}], exception:{2}".format(self.node, cmd, e)) + print( + "Execute Shell command on server {0} failed,command=[{1}], exception:{2}".format( + self.node, cmd, e + ) + ) def ssh_exec_cmd_ignore_exception(self, cmd): if self.ssh_type == "docker": @@ -946,16 +1155,20 @@ def ssh_exec_cmd_ignore_exception(self, cmd): stdout=True, stderr=True, ) - return result.output.decode('utf-8') + return result.output.decode("utf-8") except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format(e)) + self.stdio.error( + "sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format( + e + ) + ) pass # raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) return try: stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - return stderr.read().decode('utf-8') + return stderr.read().decode("utf-8") except SSHException as e: pass @@ -969,35 +1182,62 @@ def ssh_exec_cmd_get_stderr(self, cmd): stdout=True, stderr=True, ) - return result.output.decode('utf-8') + return result.output.decode("utf-8") except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format(e)) + self.stdio.error( + "sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format( + e + ) + ) pass # raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) return try: stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - return stderr.read().decode('utf-8') + return stderr.read().decode("utf-8") except SSHException as e: pass - def progress_bar(self, transferred, to_be_transferred, suffix=''): + def progress_bar(self, transferred, to_be_transferred, suffix=""): bar_len = 20 filled_len = int(round(bar_len * transferred / float(to_be_transferred))) percents = round(20.0 * transferred / float(to_be_transferred), 1) - bar = '\033[32;1m%s\033[0m' % '=' * filled_len + '-' * (bar_len - filled_len) + bar = "\033[32;1m%s\033[0m" % "=" * filled_len + "-" * (bar_len - filled_len) print_percents = round((percents * 5), 1) sys.stdout.flush() - sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) + sys.stdout.write( + "Downloading [%s] %s%s%s %s %s\r" + % ( + bar, + "\033[32;1m%s\033[0m" % print_percents, + "% [", + self.translate_byte(transferred), + "]", + suffix, + ) + ) if transferred == to_be_transferred: - sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % ( - bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) + sys.stdout.write( + "Downloading [%s] %s%s%s %s %s\r" + % ( + bar, + "\033[32;1m%s\033[0m" % print_percents, + "% [", + self.translate_byte(transferred), + "]", + suffix, + ) + ) print() def download(self, remote_path, local_path): if self.ssh_type == "docker": try: - self.stdio.verbose("remote_path: {0}:{1} to local_path:{2}".format(self.node["container_name"], remote_path, local_path)) + self.stdio.verbose( + "remote_path: {0}:{1} to local_path:{2}".format( + self.node["container_name"], remote_path, local_path + ) + ) client_result = self.client.containers.get(self.node["container_name"]) data, stat = client_result.get_archive(remote_path) with open(local_path, "wb") as f: @@ -1011,33 +1251,39 @@ def download(self, remote_path, local_path): transport = self._ssh_fd.get_transport() self._sftp_client = paramiko.SFTPClient.from_transport(transport) - print('Download {0}:{1}'.format(self.host_ip,remote_path)) + print("Download {0}:{1}".format(self.host_ip, remote_path)) self._sftp_client.get(remote_path, local_path, callback=self.progress_bar) self._sftp_client.close() def translate_byte(self, B): B = float(B) KB = float(1024) - MB = float(KB ** 2) - GB = float(MB ** 2) - TB = float(GB ** 2) + MB = float(KB**2) + GB = float(MB**2) + TB = float(GB**2) if B < KB: - return '{} {}'.format(B, 'bytes' if B > 1 else "byte") + return "{} {}".format(B, "bytes" if B > 1 else "byte") elif KB < B < MB: - return '{:.2f} KB'.format(B / KB) + return "{:.2f} KB".format(B / KB) elif MB < B < GB: - return '{:.2f} MB'.format(B / MB) + return "{:.2f} MB".format(B / MB) elif GB < B < TB: - return '{:.2f} GB'.format(B / GB) + return "{:.2f} GB".format(B / GB) else: - return '{:.2f} TB'.format(B / TB) + return "{:.2f} TB".format(B / TB) def upload(self, remote_path, local_path): if self.ssh_type == "docker": try: - self.stdio.verbose(" local_path:{0} to remote_path:{1}:{2}".format(local_path, self.node["container_name"], remote_path)) + self.stdio.verbose( + " local_path:{0} to remote_path:{1}:{2}".format( + local_path, self.node["container_name"], remote_path + ) + ) - self.client.containers.get(self.node["container_name"]).put_archive(remote_path, local_path) + self.client.containers.get(self.node["container_name"]).put_archive( + remote_path, local_path + ) return except Exception as e: @@ -1065,27 +1311,40 @@ def __del__(self): def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): if self.ssh_type == "docker": try: - exec_id = self.client.exec_create(container=self.node["container_name"], command=['su', '- ' + new_user]) + exec_id = self.client.exec_create( + container=self.node["container_name"], + command=["su", "- " + new_user], + ) response = self.client.exec_start(exec_id) return response except Exception as e: - self.stdio.error("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e)) - raise Exception("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e)) + self.stdio.error( + "sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format( + e + ) + ) + raise Exception( + "sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format( + e + ) + ) return try: ssh = self._ssh_fd.invoke_shell() - ssh.send('su {0}\n'.format(new_user)) - ssh.send('{}\n'.format(cmd)) + ssh.send("su {0}\n".format(new_user)) + ssh.send("{}\n".format(cmd)) time.sleep(time_out) self._ssh_fd.close() result = ssh.recv(65535) except SSHException as e: - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " - "command=[{1}], exception:{2}".format(self.host_ip, cmd, e)) + raise OBDIAGShellCmdException( + "Execute Shell command on server {0} failed, " + "command=[{1}], exception:{2}".format(self.host_ip, cmd, e) + ) return result def get_name(self): if self.ssh_type == "docker": - return "(docker)"+self.node.get("container_name") - return self.host_ip \ No newline at end of file + return "(docker)" + self.node.get("container_name") + return self.host_ip diff --git a/common/tool.py b/common/tool.py index bc8d7e6d..c1d102d9 100644 --- a/common/tool.py +++ b/common/tool.py @@ -54,32 +54,33 @@ from ruamel.yaml import YAML from err import EC_SQL_EXECUTE_FAILED from stdio import SafeStdio + _open = open encoding_open = open __all__ = ( -"Timeout", -"DynamicLoading", -"ConfigUtil", -"DirectoryUtil", -"FileUtil", -"YamlLoader", -"OrderedDict", -"COMMAND_ENV", -"TimeUtils", -"NetUtils", -"StringUtils", -"YamlUtils", -"Util" + "Timeout", + "DynamicLoading", + "ConfigUtil", + "DirectoryUtil", + "FileUtil", + "YamlLoader", + "OrderedDict", + "COMMAND_ENV", + "TimeUtils", + "NetUtils", + "StringUtils", + "YamlUtils", + "Util", ) -_WINDOWS = os.name == 'nt' +_WINDOWS = os.name == "nt" class Timeout(object): - def __init__(self, seconds=1, error_message='Timeout'): + def __init__(self, seconds=1, error_message="Timeout"): self.seconds = seconds self.error_message = error_message @@ -149,15 +150,17 @@ def remove_libs_path(libs): def import_module(name, stdio=None): if name not in DynamicLoading.MODULES: try: - stdio and getattr(stdio, 'verbose', print)('import %s' % name) + stdio and getattr(stdio, "verbose", print)("import %s" % name) module = __import__(name) DynamicLoading.MODULES[name] = DynamicLoading.Module(module) except: - stdio and getattr(stdio, 'exception', print)('import %s failed' % name) - stdio and getattr(stdio, 'verbose', print)('sys.path: %s' % sys.path) + stdio and getattr(stdio, "exception", print)("import %s failed" % name) + stdio and getattr(stdio, "verbose", print)("sys.path: %s" % sys.path) return None DynamicLoading.MODULES[name].count += 1 - stdio and getattr(stdio, 'verbose', print)('add %s ref count to %s' % (name, DynamicLoading.MODULES[name].count)) + stdio and getattr(stdio, "verbose", print)( + "add %s ref count to %s" % (name, DynamicLoading.MODULES[name].count) + ) return DynamicLoading.MODULES[name].module @staticmethod @@ -168,13 +171,15 @@ def export_module(name, stdio=None): return try: DynamicLoading.MODULES[name].count -= 1 - stdio and getattr(stdio, 'verbose', print)('sub %s ref count to %s' % (name, DynamicLoading.MODULES[name].count)) + stdio and getattr(stdio, "verbose", print)( + "sub %s ref count to %s" % (name, DynamicLoading.MODULES[name].count) + ) if DynamicLoading.MODULES[name].count == 0: - stdio and getattr(stdio, 'verbose', print)('export %s' % name) + stdio and getattr(stdio, "verbose", print)("export %s" % name) del sys.modules[name] del DynamicLoading.MODULES[name] except: - stdio and getattr(stdio, 'exception', print)('export %s failed' % name) + stdio and getattr(stdio, "exception", print)("export %s failed" % name) class ConfigUtil(object): @@ -184,7 +189,9 @@ def get_value_from_dict(conf, key, default=None, transform_func=None): try: # 不要使用 conf.get(key, default)来替换，这里还有类型转换的需求 value = conf[key] - return transform_func(value) if value is not None and transform_func else value + return ( + transform_func(value) if value is not None and transform_func else value + ) except: return default @@ -208,7 +215,9 @@ def get_random_pwd_by_total_length(pwd_length=10): return pwd @staticmethod - def get_random_pwd_by_rule(lowercase_length=2, uppercase_length=2, digits_length=2, punctuation_length=2): + def get_random_pwd_by_rule( + lowercase_length=2, uppercase_length=2, digits_length=2, punctuation_length=2 + ): pwd = "" for i in range(lowercase_length): pwd += random.choice(string.ascii_lowercase) @@ -217,10 +226,10 @@ def get_random_pwd_by_rule(lowercase_length=2, uppercase_length=2, digits_length for i in range(digits_length): pwd += random.choice(string.digits) for i in range(punctuation_length): - pwd += random.choice('(._+@#%)') + pwd += random.choice("(._+@#%)") pwd_list = list(pwd) random.shuffle(pwd_list) - return ''.join(pwd_list) + return "".join(pwd_list) @staticmethod def passwd_format(passwd): @@ -248,12 +257,16 @@ def list_dir(path, stdio=None): @staticmethod def copy(src, dst, stdio=None): if not os.path.isdir(src): - stdio and getattr(stdio, 'error', print)("cannot copy tree '%s': not a directory" % src) + stdio and getattr(stdio, "error", print)( + "cannot copy tree '%s': not a directory" % src + ) return False try: names = os.listdir(src) except: - stdio and getattr(stdio, 'exception', print)("error listing files in '%s':" % (src)) + stdio and getattr(stdio, "exception", print)( + "error listing files in '%s':" % (src) + ) return False if DirectoryUtil.mkdir(dst, stdio): @@ -278,7 +291,7 @@ def copy(src, dst, stdio=None): @staticmethod def mkdir(path, mode=0o755, stdio=None): - stdio and getattr(stdio, 'verbose', print)('mkdir %s' % path) + stdio and getattr(stdio, "verbose", print)("mkdir %s" % path) try: os.makedirs(path, mode=mode) return True @@ -286,18 +299,22 @@ def mkdir(path, mode=0o755, stdio=None): if e.errno == 17: return True elif e.errno == 20: - stdio and getattr(stdio, 'error', print)('%s is not a directory', path) + stdio and getattr(stdio, "error", print)("%s is not a directory", path) else: - stdio and getattr(stdio, 'error', print)('failed to create directory %s', path) - stdio and getattr(stdio, 'exception', print)('') + stdio and getattr(stdio, "error", print)( + "failed to create directory %s", path + ) + stdio and getattr(stdio, "exception", print)("") except: - stdio and getattr(stdio, 'exception', print)('') - stdio and getattr(stdio, 'error', print)('failed to create directory %s', path) + stdio and getattr(stdio, "exception", print)("") + stdio and getattr(stdio, "error", print)( + "failed to create directory %s", path + ) return False @staticmethod def rm(path, stdio=None): - stdio and getattr(stdio, 'verbose', print)('rm %s' % path) + stdio and getattr(stdio, "verbose", print)("rm %s" % path) try: if os.path.exists(path): if os.path.islink(path): @@ -306,8 +323,8 @@ def rm(path, stdio=None): shutil.rmtree(path) return True except Exception as e: - stdio and getattr(stdio, 'exception', print)('') - stdio and getattr(stdio, 'error', print)('failed to remove %s', path) + stdio and getattr(stdio, "exception", print)("") + stdio and getattr(stdio, "error", print)("failed to remove %s", path) return False @@ -318,19 +335,20 @@ class FileUtil(object): @staticmethod def checksum(target_path, stdio=None): from common.ssh import LocalClient + if not os.path.isfile(target_path): - info = 'No such file: ' + target_path + info = "No such file: " + target_path if stdio: - getattr(stdio, 'error', print)(info) + getattr(stdio, "error", print)(info) return False else: raise IOError(info) - ret = LocalClient.execute_command('md5sum {}'.format(target_path), stdio=stdio) + ret = LocalClient.execute_command("md5sum {}".format(target_path), stdio=stdio) if ret: - return ret.stdout.strip().split(' ')[0].encode('utf-8') + return ret.stdout.strip().split(" ")[0].encode("utf-8") else: m = hashlib.md5() - with open(target_path, 'rb') as f: + with open(target_path, "rb") as f: m.update(f.read()) return m.hexdigest().encode(sys.getdefaultencoding()) @@ -346,11 +364,11 @@ def copy_fileobj(fsrc, fdst): @staticmethod def copy(src, dst, stdio=None): - stdio and getattr(stdio, 'verbose', print)('copy %s %s' % (src, dst)) + stdio and getattr(stdio, "verbose", print)("copy %s %s" % (src, dst)) if os.path.exists(src) and os.path.exists(dst) and os.path.samefile(src, dst): info = "`%s` and `%s` are the same file" % (src, dst) if stdio: - getattr(stdio, 'error', print)(info) + getattr(stdio, "error", print)(info) return False else: raise IOError(info) @@ -364,7 +382,7 @@ def copy(src, dst, stdio=None): if stat.S_ISFIFO(st.st_mode): info = "`%s` is a named pipe" % fn if stdio: - getattr(stdio, 'error', print)(info) + getattr(stdio, "error", print)(info) return False else: raise IOError(info) @@ -373,96 +391,101 @@ def copy(src, dst, stdio=None): if os.path.islink(src): FileUtil.symlink(os.readlink(src), dst) return True - with FileUtil.open(src, 'rb') as fsrc, FileUtil.open(dst, 'wb') as fdst: - FileUtil.copy_fileobj(fsrc, fdst) - os.chmod(dst, os.stat(src).st_mode) - return True + with FileUtil.open(src, "rb") as fsrc, FileUtil.open(dst, "wb") as fdst: + FileUtil.copy_fileobj(fsrc, fdst) + os.chmod(dst, os.stat(src).st_mode) + return True except Exception as e: - if int(getattr(e, 'errno', -1)) == 26: + if int(getattr(e, "errno", -1)) == 26: from common.ssh import LocalClient - if LocalClient.execute_command('/usr/bin/cp -f %s %s' % (src, dst), stdio=stdio): + + if LocalClient.execute_command( + "/usr/bin/cp -f %s %s" % (src, dst), stdio=stdio + ): return True elif stdio: - getattr(stdio, 'exception', print)('copy error: %s' % e) + getattr(stdio, "exception", print)("copy error: %s" % e) else: raise e return False @staticmethod def symlink(src, dst, stdio=None): - stdio and getattr(stdio, 'verbose', print)('link %s %s' % (src, dst)) + stdio and getattr(stdio, "verbose", print)("link %s %s" % (src, dst)) try: if DirectoryUtil.rm(dst, stdio): os.symlink(src, dst) return True except Exception as e: if stdio: - getattr(stdio, 'exception', print)('link error: %s' % e) + getattr(stdio, "exception", print)("link error: %s" % e) else: raise e return False @staticmethod - def open(path, _type='r', encoding=None, stdio=None): - stdio and getattr(stdio, 'verbose', print)('open %s for %s' % (path, _type)) + def open(path, _type="r", encoding=None, stdio=None): + stdio and getattr(stdio, "verbose", print)("open %s for %s" % (path, _type)) if os.path.exists(path): if os.path.isfile(path): return encoding_open(path, _type, encoding=encoding) - info = '%s is not file' % path + info = "%s is not file" % path if stdio: - getattr(stdio, 'error', print)(info) + getattr(stdio, "error", print)(info) return None else: raise IOError(info) dir_path, file_name = os.path.split(path) if not dir_path or DirectoryUtil.mkdir(dir_path, stdio=stdio): return encoding_open(path, _type, encoding=encoding) - info = '%s is not file' % path + info = "%s is not file" % path if stdio: - getattr(stdio, 'error', print)(info) + getattr(stdio, "error", print)(info) return None else: raise IOError(info) @staticmethod def unzip(source, ztype=None, stdio=None): - stdio and getattr(stdio, 'verbose', print)('unzip %s' % source) + stdio and getattr(stdio, "verbose", print)("unzip %s" % source) if not ztype: - ztype = source.split('.')[-1] + ztype = source.split(".")[-1] try: - if ztype == 'bz2': - s_fn = bz2.BZ2File(source, 'r') - elif ztype == 'xz': - s_fn = lzma.LZMAFile(source, 'r') - elif ztype == 'gz': - s_fn = gzip.GzipFile(source, 'r') + if ztype == "bz2": + s_fn = bz2.BZ2File(source, "r") + elif ztype == "xz": + s_fn = lzma.LZMAFile(source, "r") + elif ztype == "gz": + s_fn = gzip.GzipFile(source, "r") else: - s_fn = open(source, 'r') + s_fn = open(source, "r") return s_fn except: - stdio and getattr(stdio, 'exception', print)('failed to unzip %s' % source) + stdio and getattr(stdio, "exception", print)("failed to unzip %s" % source) return None def extract_tar(tar_path, output_path, stdio=None): if not os.path.exists(output_path): os.makedirs(output_path) try: - with tarfile.open(tar_path, 'r') as tar: + with tarfile.open(tar_path, "r") as tar: tar.extractall(path=output_path) except: - stdio and getattr(stdio, 'exception', print)('failed to extract tar file %s' % tar_path) + stdio and getattr(stdio, "exception", print)( + "failed to extract tar file %s" % tar_path + ) return None @staticmethod def rm(path, stdio=None): - stdio and getattr(stdio, 'verbose', print)('rm %s' % path) + stdio and getattr(stdio, "verbose", print)("rm %s" % path) if not os.path.exists(path): return True try: os.remove(path) return True except: - stdio and getattr(stdio, 'exception', print)('failed to remove %s' % path) + stdio and getattr(stdio, "exception", print)("failed to remove %s" % path) return False @staticmethod @@ -471,27 +494,31 @@ def move(src, dst, stdio=None): @staticmethod def share_lock_obj(obj, stdio=None): - stdio and getattr(stdio, 'verbose', print)('try to get share lock %s' % obj.name) + stdio and getattr(stdio, "verbose", print)( + "try to get share lock %s" % obj.name + ) fcntl.flock(obj, fcntl.LOCK_SH | fcntl.LOCK_NB) return obj @classmethod - def share_lock(cls, path, _type='w', stdio=None): + def share_lock(cls, path, _type="w", stdio=None): return cls.share_lock_obj(cls.open(path, _type=_type, stdio=stdio)) @staticmethod def exclusive_lock_obj(obj, stdio=None): - stdio and getattr(stdio, 'verbose', print)('try to get exclusive lock %s' % obj.name) + stdio and getattr(stdio, "verbose", print)( + "try to get exclusive lock %s" % obj.name + ) fcntl.flock(obj, fcntl.LOCK_EX | fcntl.LOCK_NB) return obj @classmethod - def exclusive_lock(cls, path, _type='w', stdio=None): + def exclusive_lock(cls, path, _type="w", stdio=None): return cls.exclusive_lock_obj(cls.open(path, _type=_type, stdio=stdio)) @staticmethod def unlock(obj, stdio=None): - stdio and getattr(stdio, 'verbose', print)('unlock %s' % obj.name) + stdio and getattr(stdio, "verbose", print)("unlock %s" % obj.name) fcntl.flock(obj, fcntl.LOCK_UN) return obj @@ -503,7 +530,7 @@ def size_format(num, unit="B", output_str=False, stdio=None): unit_idx = units.index(unit) except KeyError: raise ValueError("unit {0} is illegal!".format(unit)) - new_num = float(num) * (1024 ** unit_idx) + new_num = float(num) * (1024**unit_idx) unit_idx = 0 while new_num > 1024: new_num = float(new_num) / 1024 @@ -513,23 +540,27 @@ def size_format(num, unit="B", output_str=False, stdio=None): if output_str: return "".join(["%.3f" % new_num, units[unit_idx]]) return new_num, units[unit_idx] - + @staticmethod def show_file_size_tabulate(ip, file_size, stdio=None): - format_file_size = FileUtil.size_format(int(file_size), output_str=True, stdio=stdio) + format_file_size = FileUtil.size_format( + int(file_size), output_str=True, stdio=stdio + ) summary_tab = [] field_names = ["Node", "LogSize"] summary_tab.append((ip, format_file_size)) - return "\nZipFileInfo:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + return "\nZipFileInfo:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) @staticmethod def show_file_list_tabulate(ip, file_list, stdio=None): summary_tab = [] field_names = ["Node", "LogList"] summary_tab.append((ip, file_list)) - return "\nFileListInfo:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + return "\nFileListInfo:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) @staticmethod def find_all_file(base, stdio=None): @@ -546,7 +577,7 @@ def calculate_sha256(filepath, stdio=None): sha256 = hashlib.sha256() try: filepath = os.path.expanduser(filepath) - with open(filepath, 'rb') as file: + with open(filepath, "rb") as file: while True: data = file.read(8192) if not data: @@ -556,7 +587,7 @@ def calculate_sha256(filepath, stdio=None): except Exception as e: return "" - def size(size_str, unit='B', stdio=None): + def size(size_str, unit="B", stdio=None): unit_size_dict = { "b": 1, "B": 1, @@ -571,31 +602,40 @@ def size(size_str, unit='B', stdio=None): } unit_str = size_str.strip()[-1] if unit_str not in unit_size_dict: - raise ValueError('unit {0} not in {1}'.format(unit_str, unit_size_dict.keys())) + raise ValueError( + "unit {0} not in {1}".format(unit_str, unit_size_dict.keys()) + ) real_size = float(size_str.strip()[:-1]) * unit_size_dict[unit_str] if real_size < 0: - raise ValueError('size cannot be negative!') + raise ValueError("size cannot be negative!") return real_size / unit_size_dict[unit] def write_append(filename, result, stdio=None): - with io.open(filename, 'a', encoding='utf-8') as fileobj: - fileobj.write(u'{}'.format(result)) + with io.open(filename, "a", encoding="utf-8") as fileobj: + fileobj.write("{}".format(result)) class YamlLoader(YAML): def __init__(self, stdio=None, typ=None, pure=False, output=None, plug_ins=None): - super(YamlLoader, self).__init__(typ=typ, pure=pure, output=output, plug_ins=plug_ins) + super(YamlLoader, self).__init__( + typ=typ, pure=pure, output=output, plug_ins=plug_ins + ) self.stdio = stdio - if not self.Representer.yaml_multi_representers and self.Representer.yaml_representers: - self.Representer.yaml_multi_representers = self.Representer.yaml_representers + if ( + not self.Representer.yaml_multi_representers + and self.Representer.yaml_representers + ): + self.Representer.yaml_multi_representers = ( + self.Representer.yaml_representers + ) def load(self, stream): try: return super(YamlLoader, self).load(stream) except Exception as e: - if getattr(self.stdio, 'exception', False): - self.stdio.exception('Parsing error:\n%s' % e) + if getattr(self.stdio, "exception", False): + self.stdio.exception("Parsing error:\n%s" % e) raise e def loads(self, yaml_content): @@ -606,16 +646,18 @@ def loads(self, yaml_content): stream.seek(0) return self.load(stream) except Exception as e: - if getattr(self.stdio, 'exception', False): - self.stdio.exception('Parsing error:\n%s' % e) + if getattr(self.stdio, "exception", False): + self.stdio.exception("Parsing error:\n%s" % e) raise e def dump(self, data, stream=None, transform=None): try: - return super(YamlLoader, self).dump(data, stream=stream, transform=transform) + return super(YamlLoader, self).dump( + data, stream=stream, transform=transform + ) except Exception as e: - if getattr(self.stdio, 'exception', False): - self.stdio.exception('dump error:\n%s' % e) + if getattr(self.stdio, "exception", False): + self.stdio.exception("dump error:\n%s" % e) raise e def dumps(self, data, transform=None): @@ -628,17 +670,18 @@ def dumps(self, data, transform=None): return content return content.decode() except Exception as e: - if getattr(self.stdio, 'exception', False): - self.stdio.exception('dumps error:\n%s' % e) + if getattr(self.stdio, "exception", False): + self.stdio.exception("dumps error:\n%s" % e) raise e + class YamlUtils(object): @staticmethod def is_yaml_file(path, stdio=None): if not os.path.isfile(path): return False - if path.endswith(('.yaml', '.yml')): + if path.endswith((".yaml", ".yml")): return True else: return False @@ -647,7 +690,7 @@ def is_yaml_file(path, stdio=None): def read_yaml_data(file_path, stdio=None): if YamlUtils.is_yaml_file(file_path): try: - with open(file_path, 'r') as f: + with open(file_path, "r") as f: data = yaml.load(f, Loader=yaml.FullLoader) return data except yaml.YAMLError as exc: @@ -655,12 +698,12 @@ def read_yaml_data(file_path, stdio=None): @staticmethod def write_yaml_data(data, file_path, stdio=None): - with open(file_path, 'w') as f: + with open(file_path, "w") as f: yaml.safe_dump(data, f, allow_unicode=True, sort_keys=False) @staticmethod def write_yaml_data_append(data, file_path, stdio=None): - with open(file_path, 'a+') as f: + with open(file_path, "a+") as f: yaml.safe_dump(data, f, allow_unicode=True, sort_keys=False) @@ -678,7 +721,7 @@ def load(self, source_path, stdio=None): self.source_path = source_path try: if os.path.exists(source_path): - with FileUtil.open(source_path, 'r') as f: + with FileUtil.open(source_path, "r") as f: self._cmd_env = json.load(f) except: stdio.exception("Failed to load environments from {}".format(source_path)) @@ -691,10 +734,10 @@ def save(self, stdio=None): return False stdio.verbose("save environment variables {}".format(self._cmd_env)) try: - with FileUtil.open(self.source_path, 'w', stdio=stdio) as f: + with FileUtil.open(self.source_path, "w", stdio=stdio) as f: json.dump(self._cmd_env, f) except: - stdio.exception('Failed to save environment variables') + stdio.exception("Failed to save environment variables") return False return True @@ -760,7 +803,7 @@ def get_inner_ip(stdio=None): return localhost_ip except Exception as e: return localhost_ip - + @staticmethod def network_connectivity(url="", stdio=None): try: @@ -772,27 +815,29 @@ def network_connectivity(url="", stdio=None): return False except Exception as e: return False - + @staticmethod def download_file(url, local_filename, stdio=None): with requests.get(url, stream=True) as r: r.raise_for_status() - with open(local_filename, 'wb') as f: + with open(local_filename, "wb") as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) return local_filename -COMMAND_ENV=CommandEnv() + +COMMAND_ENV = CommandEnv() class DateTimeEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): # 将datetime对象转换为字符串 - return obj.strftime('%Y-%m-%d %H:%M:%S') + return obj.strftime("%Y-%m-%d %H:%M:%S") # 其他类型按默认处理 return super().default(obj) + class TimeUtils(object): @staticmethod @@ -808,7 +853,7 @@ def parse_time_sec(time_str): elif unit == "d": value *= 3600 * 24 else: - raise Exception('%s parse time to second fialed:' % (time_str)) + raise Exception("%s parse time to second fialed:" % (time_str)) return value @staticmethod @@ -816,23 +861,24 @@ def get_format_time(time_str, stdio=None): try: return datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S") except Exception as e: - stdio.exception('%s parse time fialed, error:\n%s, time format need to be %s' % (time_str, e, '%Y-%m-%d %H:%M:%S')) - + stdio.exception( + "%s parse time fialed, error:\n%s, time format need to be %s" + % (time_str, e, "%Y-%m-%d %H:%M:%S") + ) @staticmethod def sub_minutes(t, delta, stdio=None): try: - return (t - datetime.timedelta(minutes=delta)).strftime('%Y-%m-%d %H:%M:%S') + return (t - datetime.timedelta(minutes=delta)).strftime("%Y-%m-%d %H:%M:%S") except Exception as e: - stdio.exception('%s get time fialed, error:\n%s' % (t, e)) - + stdio.exception("%s get time fialed, error:\n%s" % (t, e)) @staticmethod def add_minutes(t, delta, stdio=None): try: - return (t + datetime.timedelta(minutes=delta)).strftime('%Y-%m-%d %H:%M:%S') + return (t + datetime.timedelta(minutes=delta)).strftime("%Y-%m-%d %H:%M:%S") except Exception as e: - stdio.exception('%s get time fialed, error:\n%s' % (t, e)) + stdio.exception("%s get time fialed, error:\n%s" % (t, e)) @staticmethod def parse_time_from_to(from_time=None, to_time=None, stdio=None): @@ -841,7 +887,11 @@ def parse_time_from_to(from_time=None, to_time=None, stdio=None): sucess = False if from_time: format_from_time = TimeUtils.get_format_time(from_time, stdio) - format_to_time = TimeUtils.get_format_time(to_time, stdio) if to_time else TimeUtils.add_minutes(format_from_time, 30) + format_to_time = ( + TimeUtils.get_format_time(to_time, stdio) + if to_time + else TimeUtils.add_minutes(format_from_time, 30) + ) else: if to_time: format_to_time = TimeUtils.get_format_time(to_time, stdio) @@ -853,11 +903,15 @@ def parse_time_from_to(from_time=None, to_time=None, stdio=None): @staticmethod def parse_time_since(since=None, stdio=None): now_time = datetime.datetime.now() - format_to_time = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + format_to_time = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) try: - format_from_time = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_sec(since))).strftime('%Y-%m-%d %H:%M:%S') + format_from_time = ( + now_time - datetime.timedelta(seconds=TimeUtils.parse_time_sec(since)) + ).strftime("%Y-%m-%d %H:%M:%S") except Exception as e: - stdio.exception('%s parse time fialed, error:\n%s' % (since, e)) + stdio.exception("%s parse time fialed, error:\n%s" % (since, e)) format_from_time = TimeUtils.sub_minutes(format_to_time, 30) return format_from_time, format_to_time @@ -869,7 +923,7 @@ def get_current_us_timestamp(stdio=None): @staticmethod def parse_time_length_to_sec(time_length_str, stdio=None): unit = time_length_str[-1] - if unit != "m" and unit != "h" and unit != "d": + if unit != "m" and unit != "h" and unit != "d": raise Exception("time length must be format 'n'") value = int(time_length_str[:-1]) if unit == "m": @@ -887,18 +941,22 @@ def datetime_to_timestamp(datetime_str, stdio=None): # yyyy-mm-dd hh:mm:ss.uuuuus or yyyy-mm-dd hh:mm:ss try: if len(datetime_str) > 19: - dt = datetime.datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S.%f') + dt = datetime.datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S.%f") else: - dt = datetime.datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S') + dt = datetime.datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S") return int(dt.timestamp() * 1000000) except Exception as e: return 0 @staticmethod def trans_datetime_utc_to_local(datetime_str, stdio=None): - utct_date = datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S") # 2020-12-01 03:21:57 + utct_date = datetime.datetime.strptime( + datetime_str, "%Y-%m-%dT%H:%M:%S" + ) # 2020-12-01 03:21:57 local_date = utct_date + datetime.timedelta(hours=8) # 加上时区 - local_date_srt = datetime.datetime.strftime(local_date, "%Y-%m-%d %H:%M:%S") # 2020-12-01 11:21:57 + local_date_srt = datetime.datetime.strftime( + local_date, "%Y-%m-%d %H:%M:%S" + ) # 2020-12-01 11:21:57 trans_res = datetime.datetime.strptime(local_date_srt, "%Y-%m-%d %H:%M:%S") return str(trans_res) @@ -906,31 +964,45 @@ def trans_datetime_utc_to_local(datetime_str, stdio=None): def timestamp_to_filename_time(timestamp, stdio=None): second_timestamp = timestamp / 1000000 time_obj = time.localtime(int(second_timestamp)) - filename_time_str = time.strftime('%Y%m%d%H%M%S', time_obj) + filename_time_str = time.strftime("%Y%m%d%H%M%S", time_obj) return filename_time_str @staticmethod def parse_time_str(arg_time, stdio=None): - format_time = '' + format_time = "" try: format_time = datetime.datetime.strptime(arg_time, "%Y-%m-%d %H:%M:%S") except ValueError as e: - raise ValueError("time option {0} must be formatted as {1}".format(arg_time, '"%Y-%m-%d %H:%M:%S"')) + raise ValueError( + "time option {0} must be formatted as {1}".format( + arg_time, '"%Y-%m-%d %H:%M:%S"' + ) + ) return format_time @staticmethod def filename_time_to_datetime(filename_time, stdio=None): - """ transform yyyymmddhhmmss to yyyy-mm-dd hh:mm:ss""" + """transform yyyymmddhhmmss to yyyy-mm-dd hh:mm:ss""" if filename_time != "": - return "{0}-{1}-{2} {3}:{4}:{5}".format(filename_time[0:4], filename_time[4:6], filename_time[6:8], filename_time[8:10], filename_time[10:12], filename_time[12:14]) + return "{0}-{1}-{2} {3}:{4}:{5}".format( + filename_time[0:4], + filename_time[4:6], + filename_time[6:8], + filename_time[8:10], + filename_time[10:12], + filename_time[12:14], + ) else: return "" @staticmethod def extract_filename_time_from_log_name(log_name, stdio=None): - """ eg: xxx.20221226231617 """ + """eg: xxx.20221226231617""" log_name_fields = log_name.split(".") - if bytes.isdigit(log_name_fields[-1].encode("utf-8")) and len(log_name_fields[-1]) >= 14: + if ( + bytes.isdigit(log_name_fields[-1].encode("utf-8")) + and len(log_name_fields[-1]) >= 14 + ): return log_name_fields[-1] return "" @@ -939,13 +1011,17 @@ def extract_time_from_log_file_text(log_text, stdio=None): # 因为 yyyy-mm-dd hh:mm:ss.000000 的格式已经占了27个字符，所以如果传进来的字符串包含时间信息，那长度一定大于27 if len(log_text) > 27: if log_text.startswith("["): - time_str = log_text[1: log_text.find(']')] + time_str = log_text[1 : log_text.find("]")] else: - time_str = log_text[0: log_text.find(',')] - time_without_us = time_str[0: time_str.find('.')] + time_str = log_text[0 : log_text.find(",")] + time_without_us = time_str[0 : time_str.find(".")] try: - format_time = datetime.datetime.strptime(time_without_us, "%Y-%m-%d %H:%M:%S") - format_time_str = time.strftime("%Y-%m-%d %H:%M:%S", format_time.timetuple()) + format_time = datetime.datetime.strptime( + time_without_us, "%Y-%m-%d %H:%M:%S" + ) + format_time_str = time.strftime( + "%Y-%m-%d %H:%M:%S", format_time.timetuple() + ) except Exception as e: format_time_str = "" else: @@ -966,16 +1042,48 @@ def get_time_rounding(dt, step=0, rounding_level="s", stdio=None): :return: 处理后的时间 """ if rounding_level == "days": - td = timedelta(days=-step, seconds=dt.second, microseconds=dt.microsecond, milliseconds=0, minutes=dt.minute, hours=dt.hour, weeks=0) + td = timedelta( + days=-step, + seconds=dt.second, + microseconds=dt.microsecond, + milliseconds=0, + minutes=dt.minute, + hours=dt.hour, + weeks=0, + ) new_dt = dt - td elif rounding_level == "hour": - td = timedelta(days=0, seconds=dt.second, microseconds=dt.microsecond, milliseconds=0, minutes=dt.minute, hours=-step, weeks=0) + td = timedelta( + days=0, + seconds=dt.second, + microseconds=dt.microsecond, + milliseconds=0, + minutes=dt.minute, + hours=-step, + weeks=0, + ) new_dt = dt - td elif rounding_level == "min": - td = timedelta(days=0, seconds=dt.second, microseconds=dt.microsecond, milliseconds=0, minutes=-step, hours=0, weeks=0) + td = timedelta( + days=0, + seconds=dt.second, + microseconds=dt.microsecond, + milliseconds=0, + minutes=-step, + hours=0, + weeks=0, + ) new_dt = dt - td elif rounding_level == "s": - td = timedelta(days=0, seconds=-step, microseconds=dt.microsecond, milliseconds=0, minutes=0, hours=0, weeks=0) + td = timedelta( + days=0, + seconds=-step, + microseconds=dt.microsecond, + milliseconds=0, + minutes=0, + hours=0, + weeks=0, + ) new_dt = dt - td else: new_dt = dt @@ -989,29 +1097,30 @@ def trans_time(size: int): :return: 转化后的字符串 """ if size < 0: - return 'NO_END' + return "NO_END" mapping = [ - (86400000000, 'd'), - (3600000000, 'h'), - (60000000, 'm'), - (1000000, 's'), - (1000, 'ms'), - (1, 'μs'), + (86400000000, "d"), + (3600000000, "h"), + (60000000, "m"), + (1000000, "s"), + (1000, "ms"), + (1, "μs"), ] for unit, unit_str in mapping: if size >= unit: if unit == 1: - return '{} {}'.format(size, unit_str) + return "{} {}".format(size, unit_str) else: - return '{:.3f} {}'.format(size / unit, unit_str) - return '0' + return "{:.3f} {}".format(size / unit, unit_str) + return "0" @staticmethod def str_2_timestamp(t, stdio=None): if isinstance(t, int): return t - temp = datetime.datetime.strptime(t, '%Y-%m-%d %H:%M:%S.%f') - return int(datetime.datetime.timestamp(temp) * 10 ** 6) + temp = datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S.%f") + return int(datetime.datetime.timestamp(temp) * 10**6) + class StringUtils(object): @@ -1019,59 +1128,64 @@ class StringUtils(object): def parse_mysql_conn(cli_conn_str, stdio=None): db_info = {} # 处理密码选项，注意区分短选项和长选项的密码 - password_pattern = re.compile(r'(-p\s*|--password=)([^ ]*)') + password_pattern = re.compile(r"(-p\s*|--password=)([^ ]*)") password_match = password_pattern.search(cli_conn_str) if password_match: password = password_match.group(2) - db_info['password'] = password + db_info["password"] = password # 去除密码部分，避免后续解析出错 - cli_conn_str = cli_conn_str[:password_match.start()] + cli_conn_str[password_match.end():] + cli_conn_str = ( + cli_conn_str[: password_match.start()] + + cli_conn_str[password_match.end() :] + ) # 模式匹配短选项 - short_opt_pattern = re.compile(r'-(\w)\s*(\S*)') + short_opt_pattern = re.compile(r"-(\w)\s*(\S*)") matches = short_opt_pattern.finditer(cli_conn_str) for match in matches: opt = match.group(1) value = match.group(2) - if opt == 'h': - db_info['host'] = value - elif opt == 'u': - db_info['user'] = value - elif opt == 'P': - db_info['port'] = int(value) - elif opt == 'D': - db_info['database'] = value + if opt == "h": + db_info["host"] = value + elif opt == "u": + db_info["user"] = value + elif opt == "P": + db_info["port"] = int(value) + elif opt == "D": + db_info["database"] = value # 模式匹配长选项 - long_opt_pattern = re.compile(r'--(\w+)=([^ ]+)') + long_opt_pattern = re.compile(r"--(\w+)=([^ ]+)") long_matches = long_opt_pattern.finditer(cli_conn_str) for match in long_matches: opt = match.group(1) value = match.group(2) - if opt == 'host': - db_info['host'] = value - elif opt == 'user': - db_info['user'] = value - elif opt == 'port': - db_info['port'] = int(value) - elif opt in ['dbname', 'database']: - db_info['database'] = value + if opt == "host": + db_info["host"] = value + elif opt == "user": + db_info["user"] = value + elif opt == "port": + db_info["port"] = int(value) + elif opt in ["dbname", "database"]: + db_info["database"] = value # 如果存在命令行最后的参数，且不是一个选项，则认为是数据库名 last_param = cli_conn_str.split()[-1] - if last_param[0] != '-' and 'database' not in db_info: - db_info['database'] = last_param + if last_param[0] != "-" and "database" not in db_info: + db_info["database"] = last_param return db_info @staticmethod def validate_db_info(db_info, stdio=None): - required_keys = {'database', 'host', 'user', 'port'} - if not required_keys.issubset(db_info.keys()) or any(not value for value in db_info.values()): + required_keys = {"database", "host", "user", "port"} + if not required_keys.issubset(db_info.keys()) or any( + not value for value in db_info.values() + ): return False - if not isinstance(db_info['port'], int): + if not isinstance(db_info["port"], int): return False for key, value in db_info.items(): - if key != 'port' and not isinstance(value, str): + if key != "port" and not isinstance(value, str): return False return True @@ -1079,9 +1193,9 @@ def validate_db_info(db_info, stdio=None): def parse_env(env_string, stdio=None): env_dict = {} inner_str = env_string[1:-1] - pairs = inner_str.split(',') + pairs = inner_str.split(",") for pair in pairs: - key_value = pair.strip().split('=') + key_value = pair.strip().split("=") if len(key_value) == 2: key, value = key_value if value.startswith('"') and value.endswith('"'): @@ -1093,12 +1207,18 @@ def parse_env(env_string, stdio=None): @staticmethod def get_observer_ip_from_trace_id(content, stdio=None): - if content[0] == 'Y' and len(content) >= 12: - sep = content.find('-') + if content[0] == "Y" and len(content) >= 12: + sep = content.find("-") uval = int(content[1:sep], 16) - ip = uval & 0xffffffff - port = (uval >> 32) & 0xffff - return "%d.%d.%d.%d:%d" % ((ip >> 24 & 0xff), (ip >> 16 & 0xff), (ip >> 8 & 0xff), (ip >> 0 & 0xff), port) + ip = uval & 0xFFFFFFFF + port = (uval >> 32) & 0xFFFF + return "%d.%d.%d.%d:%d" % ( + (ip >> 24 & 0xFF), + (ip >> 16 & 0xFF), + (ip >> 8 & 0xFF), + (ip >> 0 & 0xFF), + port, + ) else: return "" @@ -1109,7 +1229,7 @@ def parse_range_string(range_str, nu, stdio=None): nu = int(nu) range_str = range_str.replace(" ", "") # range_str = range_str.replace(".", "") - start, end = range_str[1:-1].split(',') + start, end = range_str[1:-1].split(",") need_less = True need_than = True # 将数字转换为整数 @@ -1121,7 +1241,7 @@ def parse_range_string(range_str, nu, stdio=None): need_than = False else: end = float(end.strip()) - stdio and getattr(stdio, 'verbose', print)('range_str is %s' % range_str) + stdio and getattr(stdio, "verbose", print)("range_str is %s" % range_str) if need_less: if range_str[0] == "(": @@ -1143,30 +1263,36 @@ def parse_range_string(range_str, nu, stdio=None): def build_str_on_expr_by_dict(expr, variable_dict, stdio=None): s = expr d = variable_dict + def replacer(match): key = match.group(1) return str(d.get(key, match.group(0))) - return re.sub(r'#\{(\w+)\}', replacer, s) + + return re.sub(r"#\{(\w+)\}", replacer, s) @staticmethod def build_str_on_expr_by_dict_2(expr, variable_dict, stdio=None): s = expr d = variable_dict + def replacer(match): key = match.group(1) value = str(d.get(key, match.group(0))) return f"{value}" - return re.sub(r'\$\{(\w+)\}', replacer, s) + + return re.sub(r"\$\{(\w+)\}", replacer, s) @staticmethod def build_sql_on_expr_by_dict(expr, variable_dict, stdio=None): s = expr d = variable_dict + def replacer(match): key = match.group(1) value = str(d.get(key, match.group(0))) return f'"{value}"' - return re.sub(r'\$\{(\w+)\}', replacer, s) + + return re.sub(r"\$\{(\w+)\}", replacer, s) @staticmethod def node_cut_passwd_for_log(obj, stdio=None): @@ -1184,14 +1310,14 @@ def node_cut_passwd_for_log(obj, stdio=None): @staticmethod def split_ip(ip_str, stdio=None): - pattern = r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" result = re.findall(pattern, ip_str) return result @staticmethod def is_chinese(s, stdio=None): try: - s.encode('ascii') + s.encode("ascii") except UnicodeEncodeError: return True else: @@ -1214,10 +1340,9 @@ def compare_versions_lower(v1, v2, stdio=None): return len(v1.split(".")) < len(v2.split(".")) - class Cursor(SafeStdio): - def __init__(self, ip, port, user='root', tenant='sys', password='', stdio=None): + def __init__(self, ip, port, user="root", tenant="sys", password="", stdio=None): self.stdio = stdio self.ip = ip self.port = port @@ -1249,30 +1374,75 @@ def raise_cursor(self): return raise_cursor if sys.version_info.major == 2: + def _connect(self): - self.stdio.verbose('connect %s -P%s -u%s -p%s' % (self.ip, self.port, self.user, self.password)) - self.db = mysql.connect(host=self.ip, user=self.user, port=int(self.port), passwd=str(self.password)) + self.stdio.verbose( + "connect %s -P%s -u%s -p%s" + % (self.ip, self.port, self.user, self.password) + ) + self.db = mysql.connect( + host=self.ip, + user=self.user, + port=int(self.port), + passwd=str(self.password), + ) self.cursor = self.db.cursor(cursorclass=mysql.cursors.DictCursor) + else: + def _connect(self): - self.stdio.verbose('connect %s -P%s -u%s -p%s' % (self.ip, self.port, self.user, self.password)) - self.db = mysql.connect(host=self.ip, user=self.user, port=int(self.port), password=str(self.password), - cursorclass=mysql.cursors.DictCursor) + self.stdio.verbose( + "connect %s -P%s -u%s -p%s" + % (self.ip, self.port, self.user, self.password) + ) + self.db = mysql.connect( + host=self.ip, + user=self.user, + port=int(self.port), + password=str(self.password), + cursorclass=mysql.cursors.DictCursor, + ) self.cursor = self.db.cursor() - def new_cursor(self, tenant='sys', user='root', password='', ip='', port='', print_exception=True): + def new_cursor( + self, + tenant="sys", + user="root", + password="", + ip="", + port="", + print_exception=True, + ): try: ip = ip if ip else self.ip port = port if port else self.port - return Cursor(ip=ip, port=port, user=user, tenant=tenant, password=password, stdio=self.stdio) + return Cursor( + ip=ip, + port=port, + user=user, + tenant=tenant, + password=password, + stdio=self.stdio, + ) except: - print_exception and self.stdio.exception('') - self.stdio.verbose('fail to connect %s -P%s -u%s@%s -p%s' % (ip, port, user, tenant, password)) + print_exception and self.stdio.exception("") + self.stdio.verbose( + "fail to connect %s -P%s -u%s@%s -p%s" + % (ip, port, user, tenant, password) + ) return None - def execute(self, sql, args=None, execute_func=None, raise_exception=False, exc_level='error', stdio=None): + def execute( + self, + sql, + args=None, + execute_func=None, + raise_exception=False, + exc_level="error", + stdio=None, + ): try: - stdio.verbose('execute sql: %s. args: %s' % (sql, args)) + stdio.verbose("execute sql: %s. args: %s" % (sql, args)) self.cursor.execute(sql, args) if not execute_func: return self.cursor @@ -1282,15 +1452,33 @@ def execute(self, sql, args=None, execute_func=None, raise_exception=False, exc_ if raise_exception is None: raise_exception = self._raise_exception if raise_exception: - stdio.exception('') + stdio.exception("") raise e return False - def fetchone(self, sql, args=None, raise_exception=False, exc_level='error', stdio=None): - return self.execute(sql, args=args, execute_func='fetchone', raise_exception=raise_exception, exc_level=exc_level, stdio=stdio) - - def fetchall(self, sql, args=None, raise_exception=False, exc_level='error', stdio=None): - return self.execute(sql, args=args, execute_func='fetchall', raise_exception=raise_exception, exc_level=exc_level, stdio=stdio) + def fetchone( + self, sql, args=None, raise_exception=False, exc_level="error", stdio=None + ): + return self.execute( + sql, + args=args, + execute_func="fetchone", + raise_exception=raise_exception, + exc_level=exc_level, + stdio=stdio, + ) + + def fetchall( + self, sql, args=None, raise_exception=False, exc_level="error", stdio=None + ): + return self.execute( + sql, + args=args, + execute_func="fetchall", + raise_exception=raise_exception, + exc_level=exc_level, + stdio=stdio, + ) def close(self): if self.cursor: @@ -1305,22 +1493,22 @@ class Util(object): @staticmethod def get_option(options, key, default=None): - if not hasattr(options, key) : + if not hasattr(options, key): return default value = getattr(options, key) if value is None: value = default return value - + @staticmethod def set_option(options, key, value): setattr(options, key, value) - + @staticmethod def convert_to_number(s, stdio=None): if isinstance(s, (int, float)): return s - if isinstance(s,decimal.Decimal): + if isinstance(s, decimal.Decimal): try: return float(s) except: @@ -1345,20 +1533,30 @@ def convert_to_number(s, stdio=None): @staticmethod def print_scene(scene_dict, stdio=None): - columns_to_print = ['command', 'info_en', 'info_cn'] + columns_to_print = ["command", "info_en", "info_cn"] keys = columns_to_print table_data = [[value[key] for key in keys] for value in scene_dict.values()] - column_widths = [max(len(str(item)) * (StringUtils.is_chinese(item) or 1) for item in column) for column in zip(*table_data)] + column_widths = [ + max(len(str(item)) * (StringUtils.is_chinese(item) or 1) for item in column) + for column in zip(*table_data) + ] table_data.insert(0, keys) - Util.print_line(length= sum(column_widths) + 5) + Util.print_line(length=sum(column_widths) + 5) for i in range(len(table_data)): - print(Fore.GREEN + " ".join(f"{item:<{width}}" for item, width in zip(table_data[i], column_widths)) + Style.RESET_ALL) + print( + Fore.GREEN + + " ".join( + f"{item:<{width}}" + for item, width in zip(table_data[i], column_widths) + ) + + Style.RESET_ALL + ) if i == 0: - Util.print_line(length= sum(column_widths) + 5) - Util.print_line(length= sum(column_widths) + 5) + Util.print_line(length=sum(column_widths) + 5) + Util.print_line(length=sum(column_widths) + 5) @staticmethod - def print_line(char='-', length=50, stdio=None): + def print_line(char="-", length=50, stdio=None): print(char * length) @staticmethod @@ -1367,7 +1565,7 @@ def print_title(name, stdio=None): @staticmethod def gen_password(length=8, chars=string.ascii_letters + string.digits, stdio=None): - return ''.join([choice(chars) for i in range(length)]) + return "".join([choice(chars) for i in range(length)]) def retry(retry_count=3, retry_interval=2, stdio=None): def real_decorator(decor_method): @@ -1377,18 +1575,25 @@ def wrapper(*args, **kwargs): return_values = decor_method(*args, **kwargs) return return_values except Exception as e: - if getattr(stdio, "Function execution %s retry: %s " %(decor_method.__name__, count + 1), False): - stdio.exception('dumps error:\n%s' % e) + if getattr( + stdio, + "Function execution %s retry: %s " + % (decor_method.__name__, count + 1), + False, + ): + stdio.exception("dumps error:\n%s" % e) time.sleep(retry_interval) if count == retry_count - 1: raise e + return wrapper + return real_decorator @staticmethod def get_nodes_list(context, nodes, stdio=None): - ctx_nodes = context.get_variable("filter_nodes_list",None) - if ctx_nodes is not None and len(ctx_nodes)>0: + ctx_nodes = context.get_variable("filter_nodes_list", None) + if ctx_nodes is not None and len(ctx_nodes) > 0: new_nodes = [] for node in nodes: if node in ctx_nodes: diff --git a/common/types.py b/common/types.py index 2baef5f0..080816c3 100644 --- a/common/types.py +++ b/common/types.py @@ -24,8 +24,27 @@ import traceback __all__ = ( -"Moment", "Time", "Capacity", "CapacityWithB", "CapacityMB", "StringList", "Dict", "List", "StringOrKvList", "Double", -"Boolean", "Integer", "String", "Path", "SafeString", "PathList", "SafeStringList", "DBUrl", "WebUrl", "OBUser") + "Moment", + "Time", + "Capacity", + "CapacityWithB", + "CapacityMB", + "StringList", + "Dict", + "List", + "StringOrKvList", + "Double", + "Boolean", + "Integer", + "String", + "Path", + "SafeString", + "PathList", + "SafeStringList", + "DBUrl", + "WebUrl", + "OBUser", +) class Null(object): @@ -52,7 +71,7 @@ def __init__(self, s): @property def _type_str(self): if self.TYPE_STR is None: - self.TYPE_STR = str(self.__class__.__name__).split('.')[-1] + self.TYPE_STR = str(self.__class__.__name__).split(".")[-1] return self.TYPE_STR def _format(self): @@ -98,29 +117,29 @@ class Moment(ConfigItemType): def _format(self): if self._origin: - if self._origin.upper() == 'DISABLE': + if self._origin.upper() == "DISABLE": self._value = 0 else: - r = re.match('^(\d{1,2}):(\d{1,2})$', self._origin) + r = re.match("^(\d{1,2}):(\d{1,2})$", self._origin) h, m = r.groups() h, m = int(h), int(m) if 0 <= h <= 23 and 0 <= m <= 60: self._value = h * 60 + m else: - raise Exception('Invalid Value') + raise Exception("Invalid Value") else: self._value = 0 class Time(ConfigItemType): UNITS = { - 'ns': 0.000000001, - 'us': 0.000001, - 'ms': 0.001, - 's': 1, - 'm': 60, - 'h': 3600, - 'd': 86400 + "ns": 0.000000001, + "us": 0.000001, + "ms": 0.001, + "s": 1, + "m": 60, + "h": 3600, + "d": 86400, } def _format(self): @@ -128,15 +147,15 @@ def _format(self): self._origin = str(self._origin).strip() if self._origin.isdigit(): n = self._origin - unit = self.UNITS['s'] + unit = self.UNITS["s"] else: - r = re.match('^(\d+)(\w+)$', self._origin.lower()) + r = re.match("^(\d+)(\w+)$", self._origin.lower()) n, u = r.groups() unit = self.UNITS.get(u.lower()) if unit: self._value = int(n) * unit else: - raise Exception('Invalid Value') + raise Exception("Invalid Value") else: self._value = 0 @@ -157,31 +176,46 @@ def __repr__(self): def __add__(self, other): if isinstance(other, DecimalValue): - return DecimalValue(self.value + other.value, - self.precision if self.precision is not None else other.precision) + return DecimalValue( + self.value + other.value, + self.precision if self.precision is not None else other.precision, + ) return DecimalValue(self.value + other, self.precision) def __sub__(self, other): if isinstance(other, DecimalValue): - return DecimalValue(self.value - other.value, - self.precision if self.precision is not None else other.precision) + return DecimalValue( + self.value - other.value, + self.precision if self.precision is not None else other.precision, + ) return DecimalValue(self.value - other, self.precision) def __mul__(self, other): if isinstance(other, DecimalValue): - return DecimalValue(self.value * other.value, - self.precision if self.precision is not None else other.precision) + return DecimalValue( + self.value * other.value, + self.precision if self.precision is not None else other.precision, + ) return DecimalValue(self.value * other, self.precision) def __truediv__(self, other): if isinstance(other, DecimalValue): - return DecimalValue(self.value / other.value, - self.precision if self.precision is not None else other.precision) + return DecimalValue( + self.value / other.value, + self.precision if self.precision is not None else other.precision, + ) return DecimalValue(self.value / other, self.precision) class Capacity(ConfigItemType): - UNITS = {"B": 1, "K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40, "P": 1 << 50} + UNITS = { + "B": 1, + "K": 1 << 10, + "M": 1 << 20, + "G": 1 << 30, + "T": 1 << 40, + "P": 1 << 50, + } LENGTHS = {"B": 4, "K": 8, "M": 12, "G": 16, "T": 20, "P": 24} @@ -201,23 +235,27 @@ def _format(self): if not isinstance(self._origin, str) or self._origin.strip().isdigit(): self._origin = int(float(self._origin)) n = self._origin - unit = self.UNITS['B'] + unit = self.UNITS["B"] for u in self.LENGTHS: if len(str(self._origin)) < self.LENGTHS[u]: break else: - u = 'P' + u = "P" else: - groups = re.match("^(\d+)\s*([BKMGTP])((IB)|B)?\s*$", self._origin.upper()) + groups = re.match( + "^(\d+)\s*([BKMGTP])((IB)|B)?\s*$", self._origin.upper() + ) if not groups: raise ValueError("Invalid capacity string: %s" % self._origin) n, u, _, _ = groups.groups() unit = self.UNITS.get(u.upper()) if unit: self._value = int(n) * unit - self.value = str(DecimalValue(self._value, self.precision) / self.UNITS[u]) + u + self.value = ( + str(DecimalValue(self._value, self.precision) / self.UNITS[u]) + u + ) else: - raise Exception('Invalid Value') + raise Exception("Invalid Value") else: self._value = 0 self.value = str(DecimalValue(0, self.precision)) @@ -230,7 +268,7 @@ def __init__(self, s): def _format(self): super(CapacityWithB, self)._format() - self.value = self.value + 'B' + self.value = self.value + "B" class CapacityMB(Capacity): @@ -238,10 +276,10 @@ class CapacityMB(Capacity): def _format(self): super(CapacityMB, self)._format() if isinstance(self._origin, str) and self._origin.isdigit(): - self.value = self._origin + 'M' - self._value *= self.UNITS['M'] + self.value = self._origin + "M" + self._value *= self.UNITS["M"] if not self._origin: - self.value = '0M' + self.value = "0M" class StringList(ConfigItemType): @@ -249,7 +287,7 @@ class StringList(ConfigItemType): def _format(self): if self._origin: self._origin = str(self._origin).strip() - self._value = self._origin.split(';') + self._value = self._origin.split(";") else: self._value = [] @@ -286,10 +324,18 @@ def _format(self): if not item: continue if not isinstance(item, (str, dict)): - raise Exception("Invalid value: {} should be string or key-value format.".format(item)) + raise Exception( + "Invalid value: {} should be string or key-value format.".format( + item + ) + ) if isinstance(item, dict): if len(item.keys()) != 1: - raise Exception("Invalid value: {} should be single key-value format".format(item)) + raise Exception( + "Invalid value: {} should be single key-value format".format( + item + ) + ) self._value = self._origin else: self._value = self.value = [] @@ -308,14 +354,14 @@ def _format(self): self._value = self._origin else: _origin = str(self._origin).lower() - if _origin == 'true': + if _origin == "true": self._value = True - elif _origin == 'false': + elif _origin == "false": self._value = False elif _origin.isdigit(): self._value = bool(self._origin) else: - raise Exception('%s is not Boolean' % _origin) + raise Exception("%s is not Boolean" % _origin) self.value = self._value @@ -330,23 +376,25 @@ def _format(self): try: self.value = self._value = int(_origin) except: - raise Exception('%s is not Integer' % _origin) + raise Exception("%s is not Integer" % _origin) class String(ConfigItemType): def _format(self): - self.value = self._value = str(self._origin) if self._origin else '' + self.value = self._value = str(self._origin) if self._origin else "" # this type is used to ensure the parameter is a valid oceanbase user class OBUser(ConfigItemType): - OB_USER_PATTERN = re.compile("^[a-zA-Z0-9_\.-]+(@[a-zA-Z0-9_\.-]+)?(#[a-zA-Z0-9_\.-]+)?$") + OB_USER_PATTERN = re.compile( + "^[a-zA-Z0-9_\.-]+(@[a-zA-Z0-9_\.-]+)?(#[a-zA-Z0-9_\.-]+)?$" + ) def _format(self): if not self.OB_USER_PATTERN.match(str(self._origin)): raise Exception("%s is not a valid config" % self._origin) - self.value = self._value = str(self._origin) if self._origin else '' + self.value = self._value = str(self._origin) if self._origin else "" # this type is used to ensure the parameter not containing special characters to inject command @@ -356,7 +404,7 @@ class SafeString(ConfigItemType): def _format(self): if not self.SAFE_STRING_PATTERN.match(str(self._origin)): raise Exception("%s is not a valid config" % self._origin) - self.value = self._value = str(self._origin) if self._origin else '' + self.value = self._value = str(self._origin) if self._origin else "" # this type is used to ensure the parameter not containing special characters to inject command @@ -366,7 +414,7 @@ class SafeStringList(ConfigItemType): def _format(self): if self._origin: self._origin = str(self._origin).strip() - self._value = self._origin.split(';') + self._value = self._origin.split(";") for v in self._value: if not self.SAFE_STRING_PATTERN.match(v): raise Exception("%s is not a valid config" % v) @@ -383,9 +431,12 @@ def _format(self): absolute_path = "/".join([parent_path, str(self._origin)]) normalized_path = os.path.normpath(absolute_path) - if not (self.PATH_PATTERN.match(str(self._origin)) and normalized_path.startswith(parent_path)): + if not ( + self.PATH_PATTERN.match(str(self._origin)) + and normalized_path.startswith(parent_path) + ): raise Exception("%s is not a valid path" % self._origin) - self.value = self._value = str(self._origin) if self._origin else '' + self.value = self._value = str(self._origin) if self._origin else "" # this type is used to ensure the parameter is a valid path by checking it's only certaining certain characters and not crossing path @@ -396,11 +447,14 @@ def _format(self): parent_path = "/{0}".format(uuid.uuid4().hex) if self._origin: self._origin = str(self._origin).strip() - self._value = self._origin.split(';') + self._value = self._origin.split(";") for v in self._value: absolute_path = "/".join([parent_path, v]) normalized_path = os.path.normpath(absolute_path) - if not (self.PATH_PATTERN.match(v) and normalized_path.startswith(parent_path)): + if not ( + self.PATH_PATTERN.match(v) + and normalized_path.startswith(parent_path) + ): raise Exception("%s is not a valid path" % v) else: self._value = [] @@ -409,19 +463,22 @@ def _format(self): # this type is used to ensure the parameter is a valid database connection url class DBUrl(ConfigItemType): DBURL_PATTERN = re.compile( - "^jdbc:(mysql|oceanbase):(\/\/)([a-zA-Z0-9_.-]+)(:[0-9]{1,5})?\/([a-zA-Z0-9_\-]+)(\?[a-zA-Z0-9_&;=.-]*)?$") + "^jdbc:(mysql|oceanbase):(\/\/)([a-zA-Z0-9_.-]+)(:[0-9]{1,5})?\/([a-zA-Z0-9_\-]+)(\?[a-zA-Z0-9_&;=.-]*)?$" + ) def _format(self): if not self.DBURL_PATTERN.match(str(self._origin)): raise Exception("%s is not a valid config" % self._origin) - self.value = self._value = str(self._origin) if self._origin else '' + self.value = self._value = str(self._origin) if self._origin else "" # this type is used to ensure the parameter is a valid web url class WebUrl(ConfigItemType): - WEBURL_PATTERN = re.compile("^(https?:\/\/)?([\da-z_.-]+)(:[0-9]{1,5})?([\/\w \.-]*)*\/?(?:\?[\w&=_.-]*)?$") + WEBURL_PATTERN = re.compile( + "^(https?:\/\/)?([\da-z_.-]+)(:[0-9]{1,5})?([\/\w \.-]*)*\/?(?:\?[\w&=_.-]*)?$" + ) def _format(self): if not self.WEBURL_PATTERN.match(str(self._origin)): raise Exception("%s is not a valid config" % self._origin) - self.value = self._value = str(self._origin) if self._origin else '' + self.value = self._value = str(self._origin) if self._origin else "" diff --git a/common/version.py b/common/version.py index 74af20df..94bcd4ff 100644 --- a/common/version.py +++ b/common/version.py @@ -17,17 +17,19 @@ """ # obdiag version -OBDIAG_VERSION = '' +OBDIAG_VERSION = "" # obdiag build time -OBDIAG_BUILD_TIME = '' +OBDIAG_BUILD_TIME = "" def get_obdiag_version(): - version = '''OceanBase Diagnostic Tool: %s + version = """OceanBase Diagnostic Tool: %s BUILD_TIME: %s Copyright (C) 2022 OceanBase License Mulan PSL v2: http://license.coscl.org.cn/MulanPSL2. You can use this software according to the terms and conditions of the Mulan PSL v2. -There is NO WARRANTY, to the extent permitted by law.''' % (OBDIAG_VERSION, OBDIAG_BUILD_TIME) +There is NO WARRANTY, to the extent permitted by law.""" % ( + OBDIAG_VERSION, + OBDIAG_BUILD_TIME, + ) return version - diff --git a/config.py b/config.py index d79fcb9c..3f5ebc33 100644 --- a/config.py +++ b/config.py @@ -25,13 +25,13 @@ from collections import defaultdict -if getattr(sys, 'frozen', False): +if getattr(sys, "frozen", False): absPath = os.path.dirname(os.path.abspath(sys.executable)) else: absPath = os.path.dirname(os.path.abspath(__file__)) INNER_CONFIG_FILE = os.path.join(absPath, "conf/inner_config.yml") -DEFAULT_CONFIG_DATA = ''' +DEFAULT_CONFIG_DATA = """ obcluster: ob_cluster_name: obcluster db_host: 127.0.0.1 @@ -55,48 +55,48 @@ ssh_username: '' ssh_password: '' home_path: /root/obproxy -''' +""" DEFAULT_INNER_CONFIG = { - 'obdiag': { - 'basic': { - 'config_path': '~/.obdiag/config.yml', - 'config_backup_dir': '~/.obdiag/backup_conf', - 'file_number_limit': 20, - 'file_size_limit': '2G', - 'dis_rsa_algorithms':0, - }, - 'logger': { - 'log_dir': '~/.obdiag/log', - 'log_filename': 'obdiag.log', - 'file_handler_log_level': 'DEBUG', - 'log_level': 'INFO', - 'mode': 'obdiag', - 'stdout_handler_log_level': 'INFO', - }, - }, - 'check': { - 'ignore_version': False, - 'work_path': '~/.obdiag/check', - 'report': { - 'report_path': './check_report/', - 'export_type': 'table', - }, - 'package_file': '~/.obdiag/check/check_package.yaml', - 'tasks_base_path': '~/.obdiag/check/tasks/', + "obdiag": { + "basic": { + "config_path": "~/.obdiag/config.yml", + "config_backup_dir": "~/.obdiag/backup_conf", + "file_number_limit": 20, + "file_size_limit": "2G", + "dis_rsa_algorithms": 0, }, - 'gather': { - 'scenes_base_path': '~/.obdiag/gather/tasks', + "logger": { + "log_dir": "~/.obdiag/log", + "log_filename": "obdiag.log", + "file_handler_log_level": "DEBUG", + "log_level": "INFO", + "mode": "obdiag", + "stdout_handler_log_level": "INFO", }, - 'rca': { - 'result_path': './rca/', + }, + "check": { + "ignore_version": False, + "work_path": "~/.obdiag/check", + "report": { + "report_path": "./check_report/", + "export_type": "table", }, - } + "package_file": "~/.obdiag/check/check_package.yaml", + "tasks_base_path": "~/.obdiag/check/tasks/", + }, + "gather": { + "scenes_base_path": "~/.obdiag/gather/tasks", + }, + "rca": { + "result_path": "./rca/", + }, +} class Manager(SafeStdio): - - RELATIVE_PATH = '' + + RELATIVE_PATH = "" def __init__(self, home_path, stdio=None): self.stdio = stdio @@ -111,7 +111,7 @@ def _rm(self, path): def load_config(self): try: - with open(self.path, 'r') as file: + with open(self.path, "r") as file: return yaml.safe_load(file) except FileNotFoundError: self.stdio.exception(f"Configuration file '{self.path}' not found.") @@ -121,7 +121,7 @@ def load_config(self): def load_config_with_defaults(self, defaults_dict): default_config = defaultdict(lambda: None, defaults_dict) try: - with open(self.path, 'r') as stream: + with open(self.path, "r") as stream: loaded_config = yaml.safe_load(stream) except FileNotFoundError: self.stdio.exception(f"Configuration file '{self.path}' not found.") @@ -133,7 +133,10 @@ def load_config_with_defaults(self, defaults_dict): for section, values in default_config.items(): if isinstance(values, dict): for key, default_value in values.items(): - if section not in combined_config or key not in combined_config[section]: + if ( + section not in combined_config + or key not in combined_config[section] + ): combined_config[section][key] = default_value else: if section not in combined_config: @@ -142,14 +145,18 @@ def load_config_with_defaults(self, defaults_dict): class ConfigManager(Manager): - + def __init__(self, config_file=None, stdio=None): - default_config_path = os.path.join(os.path.expanduser("~"), ".obdiag", "config.yml") + default_config_path = os.path.join( + os.path.expanduser("~"), ".obdiag", "config.yml" + ) if config_file is None or not os.path.exists(config_file): config_file = default_config_path - pathlib.Path(os.path.dirname(default_config_path)).mkdir(parents=True, exist_ok=True) - with open(default_config_path, 'w') as f: + pathlib.Path(os.path.dirname(default_config_path)).mkdir( + parents=True, exist_ok=True + ) + with open(default_config_path, "w") as f: f.write(DEFAULT_CONFIG_DATA) super(ConfigManager, self).__init__(config_file, stdio) self.config_file = config_file @@ -167,90 +174,125 @@ def _safe_get(self, dictionary, *keys, default=None): @property def get_ocp_config(self): - ocp = self._safe_get(self.config_data, 'ocp', 'login', default={}) + ocp = self._safe_get(self.config_data, "ocp", "login", default={}) return { - 'url': ocp.get('url'), - 'user': ocp.get('user'), - 'password': ocp.get('password'), + "url": ocp.get("url"), + "user": ocp.get("user"), + "password": ocp.get("password"), } @property def get_ob_cluster_config(self): - ob_cluster = self.config_data.get('obcluster', {}) - nodes = ob_cluster.get('servers', {}).get('nodes', []) + ob_cluster = self.config_data.get("obcluster", {}) + nodes = ob_cluster.get("servers", {}).get("nodes", []) def create_ob_cluster_node(node_config, global_config): return { - 'ip': node_config.get('ip'), - 'ssh_username': node_config.get('ssh_username', global_config.get('ssh_username')), - 'ssh_password': node_config.get('ssh_password', global_config.get('ssh_password')), - 'ssh_port': node_config.get('ssh_port', global_config.get('ssh_port', 22)), - 'home_path': node_config.get('home_path', global_config.get('home_path', '/root/observer')), - 'data_dir': node_config.get('data_dir', global_config.get('data_dir', '/root/observer/store')), - 'redo_dir': node_config.get('redo_dir', global_config.get('redo_dir', '/root/observer/store')), - 'ssh_key_file': node_config.get('ssh_key_file', global_config.get('ssh_key_file', '')), - 'ssh_type': node_config.get('ssh_type', global_config.get('ssh_type', 'remote')), - 'container_name': node_config.get('container_name', global_config.get('container_name')), - 'host_type': 'OBSERVER', + "ip": node_config.get("ip"), + "ssh_username": node_config.get( + "ssh_username", global_config.get("ssh_username") + ), + "ssh_password": node_config.get( + "ssh_password", global_config.get("ssh_password") + ), + "ssh_port": node_config.get( + "ssh_port", global_config.get("ssh_port", 22) + ), + "home_path": node_config.get( + "home_path", global_config.get("home_path", "/root/observer") + ), + "data_dir": node_config.get( + "data_dir", global_config.get("data_dir", "/root/observer/store") + ), + "redo_dir": node_config.get( + "redo_dir", global_config.get("redo_dir", "/root/observer/store") + ), + "ssh_key_file": node_config.get( + "ssh_key_file", global_config.get("ssh_key_file", "") + ), + "ssh_type": node_config.get( + "ssh_type", global_config.get("ssh_type", "remote") + ), + "container_name": node_config.get( + "container_name", global_config.get("container_name") + ), + "host_type": "OBSERVER", } - global_config = ob_cluster.get('servers', {}).get('global', {}) - ob_cluster_nodes = [create_ob_cluster_node(node, global_config) for node in nodes] + global_config = ob_cluster.get("servers", {}).get("global", {}) + ob_cluster_nodes = [ + create_ob_cluster_node(node, global_config) for node in nodes + ] return { - 'ob_cluster_name': ob_cluster.get('ob_cluster_name'), - 'db_host': ob_cluster.get('db_host'), - 'db_port': ob_cluster.get('db_port'), - 'tenant_sys': { - 'user': ob_cluster.get('tenant_sys', {}).get('user'), - 'password': ob_cluster.get('tenant_sys', {}).get('password'), + "ob_cluster_name": ob_cluster.get("ob_cluster_name"), + "db_host": ob_cluster.get("db_host"), + "db_port": ob_cluster.get("db_port"), + "tenant_sys": { + "user": ob_cluster.get("tenant_sys", {}).get("user"), + "password": ob_cluster.get("tenant_sys", {}).get("password"), }, - 'servers': ob_cluster_nodes, + "servers": ob_cluster_nodes, } @property def get_obproxy_config(self): - ob_proxy = self.config_data.get('obproxy', {}) - nodes = ob_proxy.get('servers', {}).get('nodes', []) + ob_proxy = self.config_data.get("obproxy", {}) + nodes = ob_proxy.get("servers", {}).get("nodes", []) def create_ob_proxy_node(node_config, global_config): return { - 'ip': node_config.get('ip'), - 'ssh_username': node_config.get('ssh_username', global_config.get('ssh_username', '')), - 'ssh_password': node_config.get('ssh_password', global_config.get('ssh_password', '')), - 'ssh_port': node_config.get('ssh_port', global_config.get('ssh_port', 22)), - 'home_path': node_config.get('home_path', global_config.get('home_path', '/root/obproxy')), - 'ssh_key_file': node_config.get('ssh_key_file', global_config.get('ssh_key_file', '')), - 'ssh_type': node_config.get('ssh_type', global_config.get('ssh_type', 'ssh')), - 'container_name': node_config.get('container_name', global_config.get('container_name')), - 'host_type': 'OBPROXY', + "ip": node_config.get("ip"), + "ssh_username": node_config.get( + "ssh_username", global_config.get("ssh_username", "") + ), + "ssh_password": node_config.get( + "ssh_password", global_config.get("ssh_password", "") + ), + "ssh_port": node_config.get( + "ssh_port", global_config.get("ssh_port", 22) + ), + "home_path": node_config.get( + "home_path", global_config.get("home_path", "/root/obproxy") + ), + "ssh_key_file": node_config.get( + "ssh_key_file", global_config.get("ssh_key_file", "") + ), + "ssh_type": node_config.get( + "ssh_type", global_config.get("ssh_type", "ssh") + ), + "container_name": node_config.get( + "container_name", global_config.get("container_name") + ), + "host_type": "OBPROXY", } - global_config = ob_proxy.get('servers', {}).get('global', {}) + global_config = ob_proxy.get("servers", {}).get("global", {}) ob_proxy_nodes = [create_ob_proxy_node(node, global_config) for node in nodes] return { - 'obproxy_cluster_name': ob_proxy.get('obproxy_cluster_name'), - 'servers': ob_proxy_nodes, + "obproxy_cluster_name": ob_proxy.get("obproxy_cluster_name"), + "servers": ob_proxy_nodes, } @property def get_node_config(self, type, node_ip, config_item): - if type == 'ob_cluster': - nodes = self.get_ob_cluster_config()['servers'] - elif type == 'ob_proxy': - nodes = self.get_obproxy_config()['servers'] + if type == "ob_cluster": + nodes = self.get_ob_cluster_config()["servers"] + elif type == "ob_proxy": + nodes = self.get_obproxy_config()["servers"] else: self.stdio.exception(f"Unsupported cluster type: {type}") for node in nodes: - if node['ip'] == node_ip: + if node["ip"] == node_ip: return node.get(config_item) return None + class InnerConfigManager(Manager): - + def __init__(self, stdio=None): inner_config_abs_path = os.path.abspath(INNER_CONFIG_FILE) super().__init__(inner_config_abs_path, stdio=stdio) - self.config = self.load_config_with_defaults(DEFAULT_INNER_CONFIG) \ No newline at end of file + self.config = self.load_config_with_defaults(DEFAULT_INNER_CONFIG) diff --git a/context.py b/context.py index 2e4e7b59..ebe7eaa8 100644 --- a/context.py +++ b/context.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function from optparse import Values + class HandlerContextNamespace: def __init__(self, spacename): @@ -101,7 +102,19 @@ def return_false(self, *args, **kwargs): class HandlerContext(object): - def __init__(self, handler_name=None, namespace=None, namespaces=None, cluster_config=None, obproxy_config=None, ocp_config=None, inner_config=None, cmd=None, options=None, stdio=None): + def __init__( + self, + handler_name=None, + namespace=None, + namespaces=None, + cluster_config=None, + obproxy_config=None, + ocp_config=None, + inner_config=None, + cmd=None, + options=None, + stdio=None, + ): self.namespace = HandlerContextNamespace(namespace) self.namespaces = namespaces self.handler_name = handler_name @@ -149,4 +162,4 @@ def get_option(self, name, spacename=None, default=None): return namespace.get_option(name, default) if namespace else None def set_option(self, name, value): - self.namespace.set_option(name, value) \ No newline at end of file + self.namespace.set_option(name, value) diff --git a/core.py b/core.py index af760090..7dbca106 100644 --- a/core.py +++ b/core.py @@ -53,7 +53,9 @@ class ObdiagHome(object): - def __init__(self, stdio=None, config_path=os.path.expanduser('~/.obdiag/config.yml')): + def __init__( + self, stdio=None, config_path=os.path.expanduser("~/.obdiag/config.yml") + ): self._optimize_manager = None self.stdio = None self._stdio_func = None @@ -64,13 +66,32 @@ def __init__(self, stdio=None, config_path=os.path.expanduser('~/.obdiag/config. self.context = None self.inner_config_manager = InnerConfigManager(stdio) self.config_manager = ConfigManager(config_path, stdio) - if self.inner_config_manager.config.get("obdiag") is not None and self.inner_config_manager.config.get("obdiag").get( - "basic") is not None and self.inner_config_manager.config.get("obdiag").get("basic").get( - "telemetry") is not None and self.inner_config_manager.config.get("obdiag").get("basic").get("telemetry") is False: + if ( + self.inner_config_manager.config.get("obdiag") is not None + and self.inner_config_manager.config.get("obdiag").get("basic") is not None + and self.inner_config_manager.config.get("obdiag") + .get("basic") + .get("telemetry") + is not None + and self.inner_config_manager.config.get("obdiag") + .get("basic") + .get("telemetry") + is False + ): telemetry.work_tag = False - if self.inner_config_manager.config.get("obdiag") is not None and self.inner_config_manager.config.get("obdiag").get( - "basic") is not None and self.inner_config_manager.config.get("obdiag").get("basic").get("dis_rsa_algorithms") is not None : - disable_rsa_algorithms=self.inner_config_manager.config.get("obdiag").get("basic").get("dis_rsa_algorithms") + if ( + self.inner_config_manager.config.get("obdiag") is not None + and self.inner_config_manager.config.get("obdiag").get("basic") is not None + and self.inner_config_manager.config.get("obdiag") + .get("basic") + .get("dis_rsa_algorithms") + is not None + ): + disable_rsa_algorithms = ( + self.inner_config_manager.config.get("obdiag") + .get("basic") + .get("dis_rsa_algorithms") + ) dis_rsa_algorithms(disable_rsa_algorithms) def fork(self, cmds=None, options=None, stdio=None): @@ -91,16 +112,29 @@ def set_options(self, options): def set_stdio(self, stdio): def _print(msg, *arg, **kwarg): - sep = kwarg['sep'] if 'sep' in kwarg else None - end = kwarg['end'] if 'end' in kwarg else None - return print(msg, sep='' if sep is None else sep, end='\n' if end is None else end) + sep = kwarg["sep"] if "sep" in kwarg else None + end = kwarg["end"] if "end" in kwarg else None + return print( + msg, sep="" if sep is None else sep, end="\n" if end is None else end + ) self.stdio = stdio self._stdio_func = {} if not self.stdio: return - for func in ['start_loading', 'stop_loading', 'print', 'confirm', 'verbose', 'warn', 'exception', 'error', - 'critical', 'print_list', 'read']: + for func in [ + "start_loading", + "stop_loading", + "print", + "confirm", + "verbose", + "warn", + "exception", + "error", + "critical", + "print_list", + "read", + ]: self._stdio_func[func] = getattr(self.stdio, func, _print) def set_context(self, handler_name, namespace, config): @@ -113,7 +147,7 @@ def set_context(self, handler_name, namespace, config): cmd=self.cmds, options=self.options, stdio=self.stdio, - inner_config=self.inner_config_manager.config + inner_config=self.inner_config_manager.config, ) telemetry.set_cluster_conn(config.get_ob_cluster_config) @@ -127,7 +161,7 @@ def set_context_skip_cluster_conn(self, handler_name, namespace, config): cmd=self.cmds, options=self.options, stdio=self.stdio, - inner_config=self.inner_config_manager.config + inner_config=self.inner_config_manager.config, ) def set_offline_context(self, handler_name, namespace): @@ -137,7 +171,7 @@ def set_offline_context(self, handler_name, namespace): cmd=self.cmds, options=self.options, stdio=self.stdio, - inner_config=self.inner_config_manager.config + inner_config=self.inner_config_manager.config, ) def get_namespace(self, spacename): @@ -150,18 +184,18 @@ def get_namespace(self, spacename): def call_plugin(self, plugin, spacename=None, target_servers=None, **kwargs): args = { - 'namespace': spacename, - 'namespaces': self.namespaces, - 'cluster_config': None, - 'obproxy_config': None, - 'ocp_config': None, - 'cmd': self.cmds, - 'options': self.options, - 'stdio': self.stdio, - 'target_servers': target_servers + "namespace": spacename, + "namespaces": self.namespaces, + "cluster_config": None, + "obproxy_config": None, + "ocp_config": None, + "cmd": self.cmds, + "options": self.options, + "stdio": self.stdio, + "target_servers": target_servers, } args.update(kwargs) - self._call_stdio('verbose', 'Call %s ' % (plugin)) + self._call_stdio("verbose", "Call %s " % (plugin)) return plugin(**args) def _call_stdio(self, func, msg, *arg, **kwarg): @@ -170,7 +204,7 @@ def _call_stdio(self, func, msg, *arg, **kwarg): return self._stdio_func[func](msg, *arg, **kwarg) def ssh_clients_connect(self, servers, ssh_clients, user_config, fail_exit=False): - self._call_stdio('start_loading', 'Open ssh connection') + self._call_stdio("start_loading", "Open ssh connection") connect_io = self.stdio if fail_exit else self.stdio.sub_io() connect_status = {} success = True @@ -183,9 +217,9 @@ def ssh_clients_connect(self, servers, ssh_clients, user_config, fail_exit=False user_config.password, user_config.key_file, user_config.port, - user_config.timeout + user_config.timeout, ), - self.stdio + self.stdio, ) error = client.connect(stdio=connect_io) connect_status[server] = status = CheckStatus() @@ -197,43 +231,43 @@ def ssh_clients_connect(self, servers, ssh_clients, user_config, fail_exit=False else: status.status = CheckStatus.PASS ssh_clients[server] = client - self._call_stdio('stop_loading', 'succeed' if success else 'fail') + self._call_stdio("stop_loading", "succeed" if success else "fail") return connect_status def gather_function(self, function_type, opt): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: self.stdio.print("{0} start ...".format(function_type)) - self.set_context(function_type, 'gather', config) + self.set_context(function_type, "gather", config) timestamp = TimeUtils.get_current_us_timestamp() - self.context.set_variable('gather_timestamp', timestamp) - if function_type == 'gather_log': + self.context.set_variable("gather_timestamp", timestamp) + if function_type == "gather_log": handler = GatherLogHandler(self.context) return handler.handle() - elif function_type == 'gather_awr': + elif function_type == "gather_awr": handler = GatherAwrHandler(self.context) return handler.handle() - elif function_type == 'gather_clog': - self.context.set_variable('gather_obadmin_mode', 'clog') + elif function_type == "gather_clog": + self.context.set_variable("gather_obadmin_mode", "clog") handler = GatherObAdminHandler(self.context) return handler.handle() - elif function_type == 'gather_slog': - self.context.set_variable('gather_obadmin_mode', 'slog') + elif function_type == "gather_slog": + self.context.set_variable("gather_obadmin_mode", "slog") handler = GatherObAdminHandler(self.context) return handler.handle() - elif function_type == 'gather_obstack': + elif function_type == "gather_obstack": handler = GatherObstack2Handler(self.context) return handler.handle() - elif function_type == 'gather_perf': + elif function_type == "gather_perf": handler = GatherPerfHandler(self.context) return handler.handle() - elif function_type == 'gather_plan_monitor': + elif function_type == "gather_plan_monitor": handler = GatherPlanMonitorHandler(self.context) return handler.handle() - elif function_type == 'gather_all': + elif function_type == "gather_all": handler_sysstat = GatherOsInfoHandler(self.context) handler_sysstat.handle() handler_stack = GatherObstack2Handler(self.context) @@ -245,105 +279,135 @@ def gather_function(self, function_type, opt): handler_obproxy = GatherObProxyLogHandler(self.context) handler_obproxy.handle() return True - elif function_type == 'gather_sysstat': + elif function_type == "gather_sysstat": handler = GatherOsInfoHandler(self.context) return handler.handle() - elif function_type == 'gather_scenes_run': + elif function_type == "gather_scenes_run": handler = GatherSceneHandler(self.context) return handler.handle() - elif function_type == 'gather_ash_report': - handler =GatherAshReportHandler(self.context) + elif function_type == "gather_ash_report": + handler = GatherAshReportHandler(self.context) return handler.handle() else: - self._call_stdio('error', 'Not support gather function: {0}'.format(function_type)) + self._call_stdio( + "error", "Not support gather function: {0}".format(function_type) + ) return False def gather_obproxy_log(self, opt): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: - self.set_context_skip_cluster_conn('gather_obproxy_log', 'gather', config) + self.set_context_skip_cluster_conn("gather_obproxy_log", "gather", config) handler = GatherObProxyLogHandler(self.context) return handler.handle() def gather_scenes_list(self, opt): - self.set_offline_context('gather_scenes_list', 'gather') + self.set_offline_context("gather_scenes_list", "gather") handler = GatherScenesListHandler(self.context) return handler.handle() def analyze_fuction(self, function_type, opt): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: self.stdio.print("{0} start ...".format(function_type)) - if function_type == 'analyze_log': - self.set_context(function_type, 'analyze', config) + if function_type == "analyze_log": + self.set_context(function_type, "analyze", config) handler = AnalyzeLogHandler(self.context) handler.handle() - elif function_type == 'analyze_log_offline': - self.set_context_skip_cluster_conn(function_type, 'analyze', config) + elif function_type == "analyze_log_offline": + self.set_context_skip_cluster_conn(function_type, "analyze", config) handler = AnalyzeLogHandler(self.context) handler.handle() - elif function_type == 'analyze_flt_trace': - self.set_context(function_type, 'analyze', config) + elif function_type == "analyze_flt_trace": + self.set_context(function_type, "analyze", config) handler = AnalyzeFltTraceHandler(self.context) handler.handle() else: - self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type)) + self._call_stdio( + "error", "Not support analyze function: {0}".format(function_type) + ) return False - + def check(self, opts): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: self.stdio.print("check start ...") - self.set_context('check', 'check', config) + self.set_context("check", "check", config) obproxy_check_handler = None observer_check_handler = None - if self.context.obproxy_config.get("servers") is not None and len(self.context.obproxy_config.get("servers"))>0: - obproxy_check_handler = CheckHandler(self.context,check_target_type="obproxy") + if ( + self.context.obproxy_config.get("servers") is not None + and len(self.context.obproxy_config.get("servers")) > 0 + ): + obproxy_check_handler = CheckHandler( + self.context, check_target_type="obproxy" + ) obproxy_check_handler.handle() obproxy_check_handler.execute() - if self.context.cluster_config.get("servers") is not None and len(self.context.cluster_config.get("servers"))>0: - observer_check_handler = CheckHandler(self.context,check_target_type="observer") + if ( + self.context.cluster_config.get("servers") is not None + and len(self.context.cluster_config.get("servers")) > 0 + ): + observer_check_handler = CheckHandler( + self.context, check_target_type="observer" + ) observer_check_handler.handle() observer_check_handler.execute() if obproxy_check_handler is not None: - obproxy_report_path = os.path.expanduser(obproxy_check_handler.report.get_report_path()) + obproxy_report_path = os.path.expanduser( + obproxy_check_handler.report.get_report_path() + ) if os.path.exists(obproxy_report_path): self.stdio.print( - "Check obproxy finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format( - obproxy_check_handler.report.get_report_path()) + Style.RESET_ALL + "'") + "Check obproxy finished. For more details, please run cmd '" + + Fore.YELLOW + + " cat {0} ".format( + obproxy_check_handler.report.get_report_path() + ) + + Style.RESET_ALL + + "'" + ) if observer_check_handler is not None: - observer_report_path = os.path.expanduser(observer_check_handler.report.get_report_path()) + observer_report_path = os.path.expanduser( + observer_check_handler.report.get_report_path() + ) if os.path.exists(observer_report_path): self.stdio.print( - "Check observer finished. For more details, please run cmd'" + Fore.YELLOW + " cat {0} ".format( - observer_check_handler.report.get_report_path()) + Style.RESET_ALL + "'") + "Check observer finished. For more details, please run cmd'" + + Fore.YELLOW + + " cat {0} ".format( + observer_check_handler.report.get_report_path() + ) + + Style.RESET_ALL + + "'" + ) def check_list(self, opts): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: - self.set_offline_context('check_list', 'check_list') + self.set_offline_context("check_list", "check_list") handler = CheckListHandler(self.context) handler.handle() def rca_run(self, opts): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: - self.set_context('rca_run', 'rca_run', config) + self.set_context("rca_run", "rca_run", config) try: handler = RCAHandler(self.context) handler.handle() @@ -354,30 +418,30 @@ def rca_run(self, opts): def rca_list(self, opts): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: - self.set_offline_context('rca_list', 'rca_list') + self.set_offline_context("rca_list", "rca_list") handler = RcaScenesListHandler(context=self.context) handler.handle() def update(self, opts): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: self.stdio.print("update start ...") - self.set_offline_context('update', 'update') + self.set_offline_context("update", "update") handler = UpdateHandler(self.context) handler.execute() def config(self, opt): config = self.config_manager if not config: - self._call_stdio('error', 'No such custum config') + self._call_stdio("error", "No such custum config") return False else: - self.set_offline_context('config', 'config') + self.set_offline_context("config", "config") config_helper = ConfigHelper(context=self.context) config_helper.build_configuration() diff --git a/dependencies/check_dependencies.py b/dependencies/check_dependencies.py index 07e59e25..6fd69f4c 100644 --- a/dependencies/check_dependencies.py +++ b/dependencies/check_dependencies.py @@ -34,6 +34,7 @@ def check_client_dependencies(): import argparse import paramiko import traceback + install_flag = True except Exception as err: print("import error!!!,cause:[{0}]".format(err)) @@ -44,4 +45,6 @@ def check_client_dependencies(): if __name__ == "__main__": if not check_client_dependencies(): - print("\033[1;31m check dependencies failed, you need resolve dependencies \033[0m") + print( + "\033[1;31m check dependencies failed, you need resolve dependencies \033[0m" + ) diff --git a/err.py b/err.py index 0fc92745..262ff9be 100644 --- a/err.py +++ b/err.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function + class OBDIAGErrorCode(object): def __init__(self, code, msg): @@ -24,7 +25,7 @@ def __init__(self, code, msg): self.msg = msg def __str__(self): - return self.msg + return self.msg class OBDIAGErrorCodeTemplate(object): @@ -32,7 +33,7 @@ class OBDIAGErrorCodeTemplate(object): def __init__(self, code, msg): self.code = code self.msg = msg - self._str_ = ('OBDIAG-%04d: ' % code) + msg + self._str_ = ("OBDIAG-%04d: " % code) + msg def format(self, *args, **kwargs): return OBDIAGErrorCode( @@ -55,6 +56,7 @@ def __init__(self, operation, key, value=None, is_global=False): self.value = value self.is_global = is_global + class OBDIAGErrorSuggestion(object): def __init__(self, msg, auto_fix=False, fix_eval=[]): @@ -73,12 +75,13 @@ def __init__(self, msg, auto_fix=False, fix_eval=[]): def format(self, *args, **kwargs): return OBDIAGErrorSuggestion( self._msg.format(*args, **kwargs), - auto_fix=kwargs.get('auto_fix', self.auto_fix), - fix_eval=kwargs.get('fix_eval', self.fix_eval) + auto_fix=kwargs.get("auto_fix", self.auto_fix), + fix_eval=kwargs.get("fix_eval", self.fix_eval), ) + class CheckStatus(object): - + FAIL = "FAIL" PASS = "PASS" WAIT = "WAIT" @@ -88,6 +91,7 @@ def __init__(self, status=WAIT, error=None, suggests=[]): self.error = error self.suggests = suggests -SUG_SSH_FAILED = OBDIAGErrorSuggestionTemplate('Please check user config and network') -EC_SSH_CONNECT = OBDIAGErrorCodeTemplate(1013, '{user}@{ip} connect failed: {message}') -EC_SQL_EXECUTE_FAILED = OBDIAGErrorCodeTemplate(5000, "{sql} execute failed") \ No newline at end of file + +SUG_SSH_FAILED = OBDIAGErrorSuggestionTemplate("Please check user config and network") +EC_SSH_CONNECT = OBDIAGErrorCodeTemplate(1013, "{user}@{ip} connect failed: {message}") +EC_SQL_EXECUTE_FAILED = OBDIAGErrorCodeTemplate(5000, "{sql} execute failed") diff --git a/handler/__init__.py b/handler/__init__.py index 2b33595e..4f2405d9 100644 --- a/handler/__init__.py +++ b/handler/__init__.py @@ -14,4 +14,4 @@ @time: 2022/6/20 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/analyzer/__init__.py b/handler/analyzer/__init__.py index 29f4a072..d3a64b03 100644 --- a/handler/analyzer/__init__.py +++ b/handler/analyzer/__init__.py @@ -14,4 +14,4 @@ @time: 2023/9/20 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/analyzer/analyze_flt_trace.py b/handler/analyzer/analyze_flt_trace.py index 3faf5b4f..83c53401 100644 --- a/handler/analyzer/analyze_flt_trace.py +++ b/handler/analyzer/analyze_flt_trace.py @@ -40,7 +40,7 @@ def __init__(self, context, gather_pack_dir=None): self.is_ssh = True self.gather_ob_log_temporary_dir = const.GATHER_LOG_TEMPORARY_DIR_DEFAULT self.gather_pack_dir = gather_pack_dir - self.flt_trace_id = '' + self.flt_trace_id = "" self.nodes = [] self.workers = const.FLT_TRACE_WORKER self.max_recursion = const.FLT_TRACE_TREE_MAX_RECURSION @@ -50,21 +50,25 @@ def __init__(self, context, gather_pack_dir=None): self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] self.inner_config = self.context.inner_config return True def init_option(self): options = self.context.options - files_option = Util.get_option(options, 'files') - store_dir_option = Util.get_option(options, 'store_dir') - flt_trace_id_option = Util.get_option(options, 'flt_trace_id') - top_option = Util.get_option(options, 'top') - recursion_option = Util.get_option(options, 'recursion') - output_option = Util.get_option(options, 'output') + files_option = Util.get_option(options, "files") + store_dir_option = Util.get_option(options, "store_dir") + flt_trace_id_option = Util.get_option(options, "flt_trace_id") + top_option = Util.get_option(options, "top") + recursion_option = Util.get_option(options, "recursion") + output_option = Util.get_option(options, "output") if store_dir_option is not None: if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('Warning: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "Warning: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) if files_option: @@ -84,22 +88,28 @@ def init_option(self): self.output = int(output_option) return True - def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False - local_store_parent_dir = os.path.join(self.gather_pack_dir, "analyze_flt_result_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + local_store_parent_dir = os.path.join( + self.gather_pack_dir, + "analyze_flt_result_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] node_files = [] old_files = [] def handle_from_node(node): - resp, node_file_s = self.__handle_from_node(node, old_files, local_store_parent_dir) + resp, node_file_s = self.__handle_from_node( + node, old_files, local_store_parent_dir + ) old_files.extend(node_file_s) for node_file in node_file_s: node_files.append([node, node_file]) @@ -109,14 +119,18 @@ def handle_from_node(node): for node in self.nodes: handle_from_node(node) else: - local_ip = '127.0.0.1' + local_ip = "127.0.0.1" node = self.nodes[0] node["ip"] = local_ip handle_from_node(node) tree = Tree() - with ProcessPoolExecutor(max(min(self.workers, len(node_files)), 1)) as executor: - future_to_url = [executor.submit(self.parse_file, file) for file in node_files] + with ProcessPoolExecutor( + max(min(self.workers, len(node_files)), 1) + ) as executor: + future_to_url = [ + executor.submit(self.parse_file, file) for file in node_files + ] for future in as_completed(future_to_url): data = future.result() tree.build(data) @@ -125,32 +139,42 @@ def handle_from_node(node): return analyze_tuples def __handle_from_node(self, node, old_files, local_store_parent_dir): - resp = { - "skip": False, - "error": "" - } - remote_ip = node.get("ip") if self.is_ssh else '127.0.0.1' + resp = {"skip": False, "error": ""} + remote_ip = node.get("ip") if self.is_ssh else "127.0.0.1" remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") node_files = [] - self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) + self.stdio.verbose( + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=local_store_parent_dir, stdio=self.stdio) if "ssh_type" in node and node["ssh_type"] == "docker": - local_store_dir = "{0}/docker_{1}".format(local_store_parent_dir, node["container_name"]) + local_store_dir = "{0}/docker_{1}".format( + local_store_parent_dir, node["container_name"] + ) else: local_store_dir = "{0}/{1}".format(local_store_parent_dir, remote_ip) DirectoryUtil.mkdir(path=local_store_dir, stdio=self.stdio) ssh_failed = False try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node) + ssh = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + ) except Exception as e: ssh = None - self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.exception( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) @@ -159,10 +183,20 @@ def __handle_from_node(self, node, old_files, local_store_parent_dir): gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) if self.is_ssh: - self.__get_online_log_file(ssh, node, gather_dir_full_path, local_store_dir) + self.__get_online_log_file( + ssh, node, gather_dir_full_path, local_store_dir + ) else: self.__get_offline_log_file(ssh, gather_dir_full_path, local_store_dir) - delete_file(self.is_ssh, ssh, os.path.join(gather_dir_full_path, str(node.get("host_type")) + '-' + str(self.flt_trace_id)), self.stdio) + delete_file( + self.is_ssh, + ssh, + os.path.join( + gather_dir_full_path, + str(node.get("host_type")) + "-" + str(self.flt_trace_id), + ), + self.stdio, + ) ssh.ssh_close() for file in FileUtil.find_all_file(local_store_dir): if self.flt_trace_id in file and (file not in old_files): @@ -176,7 +210,10 @@ def __get_online_log_file(self, ssh_helper, node, gather_path, local_store_dir): """ home_path = node.get("home_path") log_path = os.path.join(home_path, "log") - local_store_path = "{0}/{1}".format(local_store_dir, str(node.get("host_type")) + '-' + str(self.flt_trace_id)) + local_store_path = "{0}/{1}".format( + local_store_dir, str(node.get("host_type")) + "-" + str(self.flt_trace_id) + ) + def check_filename(filename): if os.path.exists(filename): # 文件已存在，尝试添加后缀 @@ -196,12 +233,12 @@ def check_filename(filename): grep_args=self.flt_trace_id, gather_path=gather_path, log_name=self.flt_trace_id, - log_dir=log_path) + log_dir=log_path, + ) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) SshClient(self.stdio).run(ssh_helper, grep_cmd) log_full_path = "{gather_path}/{log_name}".format( - log_name=self.flt_trace_id, - gather_path=gather_path + log_name=self.flt_trace_id, gather_path=gather_path ) download_file(True, ssh_helper, log_full_path, local_store_path, self.stdio) @@ -215,10 +252,13 @@ def __get_offline_log_file(self, ssh_helper, log_full_path, local_store_dir): if self.flt_trace_id is not None and (len(log_name_list) > 0): grep_cmd = "grep -e '{grep_args}' {log_file} > {local_store_path} ".format( grep_args=self.flt_trace_id, - log_file=' '.join(log_name_list), - local_store_path=local_store_path) + log_file=" ".join(log_name_list), + local_store_path=local_store_path, + ) LocalClient(self.stdio).run(grep_cmd) - download_file(False, ssh_helper, log_full_path, local_store_path, self.stdio) + download_file( + False, ssh_helper, log_full_path, local_store_path, self.stdio + ) def __get_log_name_list_offline(self): """ @@ -241,7 +281,7 @@ def __get_log_name_list_offline(self): def __parse_log_file(self, node, file, trace): counter = 0 li = [] - with open(file, 'r', encoding='utf-8') as f: + with open(file, "r", encoding="utf-8") as f: while True: line = f.readline() if line: @@ -250,17 +290,19 @@ def __parse_log_file(self, node, file, trace): counter += 1 li.append(parsed) else: - self.stdio.verbose('file:{} trace:{} total:{}'.format(file, trace, counter)) + self.stdio.verbose( + "file:{} trace:{} total:{}".format(file, trace, counter) + ) break return li def __parse_json_file(self, node, file, trace): key_mapping = { - 'span_id': 'id', - 'parent': 'parent_id', - 'span_name': 'name', + "span_id": "id", + "parent": "parent_id", + "span_name": "name", } - time_keys = ['start_ts', 'end_ts'] + time_keys = ["start_ts", "end_ts"] def remap_key(di): for key, new_key in key_mapping.items(): @@ -269,8 +311,9 @@ def remap_key(di): di[new_key] = temp di.pop(key) return di + li = [] - with open(file, 'r', encoding='utf-8') as f: + with open(file, "r", encoding="utf-8") as f: content = f.read() try: data = json.loads(content) @@ -278,9 +321,9 @@ def remap_key(di): self.stdio.verbose(traceback.format_exc()) sys.exit() if not isinstance(data, list): - raise ValueError('json file is not a list') + raise ValueError("json file is not a list") for item in data: - if trace == item['trace_id']: + if trace == item["trace_id"]: li.append(remap_key(item)) for key in time_keys: item[key] = TimeUtils.str_2_timestamp(item[key]) @@ -288,9 +331,15 @@ def remap_key(di): def parse_line(self, node, line, trace): traced_prefix = '"trace_id"' - traced = '{' + ('"trace_id":"%s"' % trace if trace else '') - data_start = '{"host_ip": "' + node.get("ip") + '", "host_type": "' + node.get("host_type") + ' ", "trace_data":' - data_end = '}' + traced = "{" + ('"trace_id":"%s"' % trace if trace else "") + data_start = ( + '{"host_ip": "' + + node.get("ip") + + '", "host_type": "' + + node.get("host_type") + + ' ", "trace_data":' + ) + data_end = "}" idx = line.find(traced) if idx == -1: return @@ -300,26 +349,26 @@ def parse_line(self, node, line, trace): if countStr == 1: return json.loads(data_start + line[idx:-1] + data_end) else: - line_last = line[idx + 1:len(line) + 1] + line_last = line[idx + 1 : len(line) + 1] idx_last = line_last.find(traced_prefix) - new_line = line[idx:(idx + idx_last + 1)] + new_line = line[idx : (idx + idx_last + 1)] if len(new_line) > 10: return json.loads(data_start + new_line + data_end) else: pass except: - if line.endswith(']}\n'): - new_line_data = line[idx:-3] + "...\"}]}" + if line.endswith("]}\n"): + new_line_data = line[idx:-3] + '..."}]}' else: - new_line_data = line[idx:-1] + '}' + new_line_data = line[idx:-1] + "}" try: return json.loads(data_start + new_line_data + data_end) except: - new_line_data = line.replace('\t', '\\t')[idx:-5] + '..."}]}' + new_line_data = line.replace("\t", "\\t")[idx:-5] + '..."}]}' return json.loads(data_start + new_line_data + data_end) - def __scan_trace_file(self, path='.'): - keyword = 'trace.log' + def __scan_trace_file(self, path="."): + keyword = "trace.log" for entry in os.listdir(path): if keyword in entry: yield os.path.join(path, entry) @@ -329,13 +378,13 @@ def __file_mapping(self, args): results = [] for log_dir in args.log_dirs: if not os.path.isdir(log_dir): - self.stdio.verbose('Dir not exist: {}'.format(log_dir)) + self.stdio.verbose("Dir not exist: {}".format(log_dir)) continue for file in self.__scan_trace_file(log_dir): results.append((file, trace_id)) for file in args.log_files: if not os.path.isfile(file): - self.stdio.verbose('File not exist: {}'.format(file)) + self.stdio.verbose("File not exist: {}".format(file)) continue if (file, trace_id) not in results: results.append((file, trace_id)) @@ -345,27 +394,31 @@ def __output(self, result_dir, tree, output_terminal=60): if not tree.nodes: self.stdio.warn("The analysis result is empty") return - filename = os.path.join(result_dir, '{}.txt'.format(self.flt_trace_id)) + filename = os.path.join(result_dir, "{}.txt".format(self.flt_trace_id)) line_counter = 0 - with open(filename, 'w', encoding='utf-8') as f: + with open(filename, "w", encoding="utf-8") as f: for line in tree.traverse(self.max_recursion, self.top): f.write(line) - f.write('\n') + f.write("\n") line_counter += 1 if line_counter < output_terminal: if len(line) > 100: - self.stdio.print("{0} {1}".format(line[:97], '...')) + self.stdio.print("{0} {1}".format(line[:97], "...")) else: self.stdio.print(line) elif line_counter == output_terminal: - self.stdio.print('Result too large, wait a moment ...\n') - self.stdio.verbose('Result saved: {}'.format(os.path.abspath(filename))) - last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(filename) + self.stdio.print("Result too large, wait a moment ...\n") + self.stdio.verbose("Result saved: {}".format(os.path.abspath(filename))) + last_info = ( + "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + filename + ) + ) self.stdio.print(last_info) def parse_file(self, file): - self.stdio.verbose('parse file: {}'.format(file[1])) - if file[1].endswith('.json'): + self.stdio.verbose("parse file: {}".format(file[1])) + if file[1].endswith(".json"): return self.__parse_json_file(file[0], file[1], self.flt_trace_id) else: - return self.__parse_log_file(file[0], file[1], self.flt_trace_id) \ No newline at end of file + return self.__parse_log_file(file[0], file[1], self.flt_trace_id) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index b658d51d..575754ef 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -55,28 +55,28 @@ def __init__(self, context): self.config_path = const.DEFAULT_CONFIG_PATH def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] self.inner_config = self.context.inner_config if self.inner_config is None: self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] return True def init_option(self): options = self.context.options - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - since_option = Util.get_option(options, 'since') - store_dir_option = Util.get_option(options, 'store_dir') - grep_option = Util.get_option(options, 'grep') - scope_option = Util.get_option(options, 'scope') - log_level_option = Util.get_option(options, 'log_level') - files_option = Util.get_option(options, 'files') + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + since_option = Util.get_option(options, "since") + store_dir_option = Util.get_option(options, "store_dir") + grep_option = Util.get_option(options, "grep") + scope_option = Util.get_option(options, "scope") + log_level_option = Util.get_option(options, "log_level") + files_option = Util.get_option(options, "files") if files_option: self.is_ssh = False self.directly_analyze_files = True @@ -88,28 +88,64 @@ def init_option(self): self.from_time_str = from_option self.to_time_str = to_option except OBDIAGFormatException: - self.stdio.exception('Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}'.format(from_option, to_option)) + self.stdio.exception( + "Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}".format( + from_option, to_option + ) + ) return False if to_timestamp <= from_timestamp: - self.stdio.exception('Error: from datetime is larger than to datetime, please check.') + self.stdio.exception( + "Error: from datetime is larger than to datetime, please check." + ) return False elif (from_option is None or to_option is None) and since_option is not None: now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('analyze log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "analyze log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 30 minutes') + self.stdio.warn( + "No time option provided, default processing is based on the last 30 minutes" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) if since_option is not None: - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") else: - self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('analyze log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.from_time_str = ( + now_time - datetime.timedelta(minutes=30) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "analyze log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) if store_dir_option is not None: if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('Error: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "Error: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) if grep_option is not None: @@ -122,14 +158,19 @@ def init_option(self): def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False - local_store_parent_dir = os.path.join(self.gather_pack_dir, - "analyze_pack_{0}".format(TimeUtils.timestamp_to_filename_time( - TimeUtils.get_current_us_timestamp()))) + local_store_parent_dir = os.path.join( + self.gather_pack_dir, + "analyze_pack_{0}".format( + TimeUtils.timestamp_to_filename_time( + TimeUtils.get_current_us_timestamp() + ) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] @@ -141,81 +182,126 @@ def handle_from_node(node): for node in self.nodes: handle_from_node(node) else: - local_ip = '127.0.0.1' + local_ip = "127.0.0.1" node = self.nodes[0] node["ip"] = local_ip handle_from_node(node) - self.stdio.start_loading('analyze result start') - title, field_names, summary_list, summary_details_list = self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) - table = tabulate.tabulate(summary_list, headers=field_names, tablefmt="grid", showindex=False) - self.stdio.stop_loading('analyze result sucess') + self.stdio.start_loading("analyze result start") + title, field_names, summary_list, summary_details_list = ( + self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) + ) + table = tabulate.tabulate( + summary_list, headers=field_names, tablefmt="grid", showindex=False + ) + self.stdio.stop_loading("analyze result sucess") self.stdio.print(title) self.stdio.print(table) - FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), title + str(table) + "\n\nDetails:\n\n") - + FileUtil.write_append( + os.path.join(local_store_parent_dir, "result_details.txt"), + title + str(table) + "\n\nDetails:\n\n", + ) + for m in range(len(summary_details_list)): for n in range(len(field_names)): - extend = "\n\n" if n == len(field_names) -1 else "\n" - FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), field_names[n] + ": " + str(summary_details_list[m][n]) + extend) - last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(local_store_parent_dir, "result_details.txt")) + extend = "\n\n" if n == len(field_names) - 1 else "\n" + FileUtil.write_append( + os.path.join(local_store_parent_dir, "result_details.txt"), + field_names[n] + ": " + str(summary_details_list[m][n]) + extend, + ) + last_info = ( + "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + os.path.join(local_store_parent_dir, "result_details.txt") + ) + ) self.stdio.print(last_info) return analyze_tuples def __handle_from_node(self, node, local_store_parent_dir): - resp = { - "skip": False, - "error": "" - } + resp = {"skip": False, "error": ""} node_results = [] - remote_ip = node.get("ip") if self.is_ssh else '127.0.0.1' + remote_ip = node.get("ip") if self.is_ssh else "127.0.0.1" remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") remote_home_path = node.get("home_path") - self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) + self.stdio.verbose( + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=local_store_parent_dir, stdio=self.stdio) - if "ssh_type" in node and node["ssh_type"]=="docker": - local_store_dir= "{0}/docker_{1}".format(local_store_parent_dir, node["container_name"]) + if "ssh_type" in node and node["ssh_type"] == "docker": + local_store_dir = "{0}/docker_{1}".format( + local_store_parent_dir, node["container_name"] + ) else: - local_store_dir = "{0}/{1}".format(local_store_parent_dir, remote_ip.replace(".", "_")) - DirectoryUtil.mkdir(path=local_store_dir,stdio=self.stdio) + local_store_dir = "{0}/{1}".format( + local_store_parent_dir, remote_ip.replace(".", "_") + ) + DirectoryUtil.mkdir(path=local_store_dir, stdio=self.stdio) ssh_failed = False ssh = None try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key,node, self.stdio) + ssh = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.error("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.error( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) if not ssh_failed: - from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) - to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) - gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp) + from_datetime_timestamp = TimeUtils.timestamp_to_filename_time( + TimeUtils.datetime_to_timestamp(self.from_time_str) + ) + to_datetime_timestamp = TimeUtils.timestamp_to_filename_time( + TimeUtils.datetime_to_timestamp(self.to_time_str) + ) + gather_dir_name = "ob_log_{0}_{1}_{2}".format( + ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp + ) gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) log_list, resp = self.__handle_log_list(ssh, node, resp) if resp["skip"]: return resp, node_results - self.stdio.print(FileUtil.show_file_list_tabulate(remote_ip, log_list, self.stdio)) + self.stdio.print( + FileUtil.show_file_list_tabulate(remote_ip, log_list, self.stdio) + ) for log_name in log_list: if self.directly_analyze_files: - self.__pharse_offline_log_file(ssh_helper=ssh, log_name=log_name, local_store_dir=local_store_dir) - analyze_log_full_path = "{0}/{1}".format(local_store_dir, str(log_name).strip(".").replace("/", "_")) + self.__pharse_offline_log_file( + ssh_helper=ssh, + log_name=log_name, + local_store_dir=local_store_dir, + ) + analyze_log_full_path = "{0}/{1}".format( + local_store_dir, str(log_name).strip(".").replace("/", "_") + ) else: - self.__pharse_log_file(ssh_helper=ssh, node=node, log_name=log_name, - gather_path=gather_dir_full_path, - local_store_dir=local_store_dir) + self.__pharse_log_file( + ssh_helper=ssh, + node=node, + log_name=log_name, + gather_path=gather_dir_full_path, + local_store_dir=local_store_dir, + ) analyze_log_full_path = "{0}/{1}".format(local_store_dir, log_name) - self.stdio.start_loading('analyze log start') + self.stdio.start_loading("analyze log start") file_result = self.__parse_log_lines(analyze_log_full_path) - self.stdio.stop_loading('analyze log sucess') + self.stdio.stop_loading("analyze log sucess") node_results.append(file_result) delete_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) ssh.ssh_close() @@ -227,21 +313,31 @@ def __handle_log_list(self, ssh, node, resp): else: log_list = self.__get_log_name_list(ssh, node) if len(log_list) > self.file_number_limit: - self.stdio.warn("{0} The number of log files is {1}, out of range (0,{2}]".format(node.get("ip"), len(log_list), - self.file_number_limit)) - resp["skip"] = True, - resp["error"] = "Too many files {0} > {1}, Please adjust the analyze time range".format(len(log_list), - self.file_number_limit) + self.stdio.warn( + "{0} The number of log files is {1}, out of range (0,{2}]".format( + node.get("ip"), len(log_list), self.file_number_limit + ) + ) + resp["skip"] = (True,) + resp["error"] = ( + "Too many files {0} > {1}, Please adjust the analyze time range".format( + len(log_list), self.file_number_limit + ) + ) if self.directly_analyze_files: - resp["error"] = "Too many files {0} > {1}, " \ - "Please adjust the number of incoming files".format(len(log_list), - self.file_number_limit) + resp["error"] = ( + "Too many files {0} > {1}, " + "Please adjust the number of incoming files".format( + len(log_list), self.file_number_limit + ) + ) return log_list, resp elif len(log_list) == 0: self.stdio.warn( "{0} The number of log files is {1}, No files found, " - "Please adjust the query limit".format(node.get("ip"), len(log_list))) - resp["skip"] = True, + "Please adjust the query limit".format(node.get("ip"), len(log_list)) + ) + resp["skip"] = (True,) resp["error"] = "No files found" return log_list, resp return log_list, resp @@ -253,18 +349,40 @@ def __get_log_name_list(self, ssh_helper, node): """ home_path = node.get("home_path") log_path = os.path.join(home_path, "log") - if self.scope == "observer" or self.scope == "rootservice" or self.scope == "election": - get_oblog = "ls -1 -F %s/*%s.log* | awk -F '/' '{print $NF}'" % (log_path, self.scope) + if ( + self.scope == "observer" + or self.scope == "rootservice" + or self.scope == "election" + ): + get_oblog = "ls -1 -F %s/*%s.log* | awk -F '/' '{print $NF}'" % ( + log_path, + self.scope, + ) else: - get_oblog = "ls -1 -F %s/observer.log* %s/rootservice.log* %s/election.log* | awk -F '/' '{print $NF}'" % \ - (log_path, log_path, log_path) + get_oblog = ( + "ls -1 -F %s/observer.log* %s/rootservice.log* %s/election.log* | awk -F '/' '{print $NF}'" + % (log_path, log_path, log_path) + ) log_name_list = [] - log_files = SshClient(self.stdio).run(ssh_helper, get_oblog) if self.is_ssh else LocalClient(self.stdio).run(get_oblog) + log_files = ( + SshClient(self.stdio).run(ssh_helper, get_oblog) + if self.is_ssh + else LocalClient(self.stdio).run(get_oblog) + ) if log_files: - log_name_list = get_logfile_name_list(self.is_ssh, ssh_helper, self.from_time_str, self.to_time_str, - log_path, log_files, self.stdio) + log_name_list = get_logfile_name_list( + self.is_ssh, + ssh_helper, + self.from_time_str, + self.to_time_str, + log_path, + log_files, + self.stdio, + ) else: - self.stdio.error("Unable to find the log file. Please provide the correct --ob_install_dir, the default is [/home/admin/oceanbase]") + self.stdio.error( + "Unable to find the log file. Please provide the correct --ob_install_dir, the default is [/home/admin/oceanbase]" + ) return log_name_list def __get_log_name_list_offline(self): @@ -285,7 +403,9 @@ def __get_log_name_list_offline(self): self.stdio.verbose("get log list {}".format(log_name_list)) return log_name_list - def __pharse_log_file(self, ssh_helper, node, log_name, gather_path, local_store_dir): + def __pharse_log_file( + self, ssh_helper, node, log_name, gather_path, local_store_dir + ): """ :param ssh_helper, log_name, gather_path :return: @@ -298,44 +418,79 @@ def __pharse_log_file(self, ssh_helper, node, log_name, gather_path, local_store grep_args=self.grep_args, gather_path=gather_path, log_name=log_name, - log_dir=log_path) + log_dir=log_path, + ) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ( + SshClient(self.stdio).run(ssh_helper, grep_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(grep_cmd) + ) log_full_path = "{gather_path}/{log_name}".format( - log_name=log_name, - gather_path=gather_path + log_name=log_name, gather_path=gather_path + ) + download_file( + self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio ) - download_file(self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio) else: - real_time_logs = ["observer.log", "rootservice.log", "election.log", "trace.log", "observer.log.wf", "rootservice.log.wf", "election.log.wf", "trace.log.wf"] + real_time_logs = [ + "observer.log", + "rootservice.log", + "election.log", + "trace.log", + "observer.log.wf", + "rootservice.log.wf", + "election.log.wf", + "trace.log.wf", + ] if log_name in real_time_logs: cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format( - gather_path=gather_path, - log_name=log_name, - log_dir=log_path) + gather_path=gather_path, log_name=log_name, log_dir=log_path + ) self.stdio.verbose("copy files, run cmd = [{0}]".format(cp_cmd)) - SshClient(self.stdio).run(ssh_helper, cp_cmd) if self.is_ssh else LocalClient(self.stdio).run(cp_cmd) - log_full_path = "{gather_path}/{log_name}".format(log_name=log_name, gather_path=gather_path) - download_file(self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio) + ( + SshClient(self.stdio).run(ssh_helper, cp_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cp_cmd) + ) + log_full_path = "{gather_path}/{log_name}".format( + log_name=log_name, gather_path=gather_path + ) + download_file( + self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio + ) else: - log_full_path = "{log_dir}/{log_name}".format(log_name=log_name, log_dir=log_path) - download_file(self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio) + log_full_path = "{log_dir}/{log_name}".format( + log_name=log_name, log_dir=log_path + ) + download_file( + self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio + ) def __pharse_offline_log_file(self, ssh_helper, log_name, local_store_dir): """ :param ssh_helper, log_name :return: """ - local_store_path = "{0}/{1}".format(local_store_dir, str(log_name).strip(".").replace("/", "_")) + local_store_path = "{0}/{1}".format( + local_store_dir, str(log_name).strip(".").replace("/", "_") + ) if self.grep_args is not None: grep_cmd = "grep -e '{grep_args}' {log_name} >> {local_store_path} ".format( grep_args=self.grep_args, log_name=log_name, - local_store_path=local_store_path) + local_store_path=local_store_path, + ) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ( + SshClient(self.stdio).run(ssh_helper, grep_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(grep_cmd) + ) else: - download_file(self.is_ssh, ssh_helper, log_name, local_store_path, self.stdio) + download_file( + self.is_ssh, ssh_helper, log_name, local_store_path, self.stdio + ) def __get_observer_ret_code(self, log_line): """ @@ -353,7 +508,7 @@ def __get_observer_ret_code(self, log_line): end = start while end < len(log_line): c = log_line[end] - if c < '0' or c > '9': + if c < "0" or c > "9": break end = end + 1 return "-" + log_line[start:end] @@ -366,7 +521,7 @@ def __parse_log_lines(self, file_full_path): """ error_dict = {} self.stdio.verbose("start parse log {0}".format(file_full_path)) - with open(file_full_path, 'r', encoding='utf8', errors='ignore') as file: + with open(file_full_path, "r", encoding="utf8", errors="ignore") as file: line_num = 0 for line in file: line_num = line_num + 1 @@ -387,14 +542,22 @@ def __parse_log_lines(self, file_full_path): "count": 1, "first_found_time": line_time, "last_found_time": line_time, - "trace_id_list": {trace_id} if len(trace_id) > 0 else {} + "trace_id_list": ( + {trace_id} if len(trace_id) > 0 else {} + ), } else: count = error_dict[ret_code]["count"] + 1 - first_found_time = error_dict[ret_code]["first_found_time"] if error_dict[ret_code][ - "first_found_time"] < line_time else line_time - last_found_time = error_dict[ret_code]["last_found_time"] if error_dict[ret_code][ - "last_found_time"] > line_time else line_time + first_found_time = ( + error_dict[ret_code]["first_found_time"] + if error_dict[ret_code]["first_found_time"] < line_time + else line_time + ) + last_found_time = ( + error_dict[ret_code]["last_found_time"] + if error_dict[ret_code]["last_found_time"] > line_time + else line_time + ) trace_id_list = list(error_dict[ret_code]["trace_id_list"]) if not (trace_id in trace_id_list): trace_id_list.append(trace_id) @@ -403,7 +566,7 @@ def __parse_log_lines(self, file_full_path): "count": count, "first_found_time": first_found_time, "last_found_time": last_found_time, - "trace_id_list": trace_id_list + "trace_id_list": trace_id_list, } self.stdio.verbose("complete parse log {0}".format(file_full_path)) return error_dict @@ -416,7 +579,7 @@ def __get_time_from_ob_log_line(self, log_line): """ time_str = "" if len(log_line) >= 28: - time_str = log_line[1: log_line.find(']')] + time_str = log_line[1 : log_line.find("]")] return time_str def __get_trace_id(self, log_line): @@ -425,11 +588,10 @@ def __get_trace_id(self, log_line): :param log_line :return: trace_id """ - pattern = re.compile(r'\[Y(.*?)\]') + pattern = re.compile(r"\[Y(.*?)\]") find = pattern.search(log_line) if find and find.group(1): - return find.group(1).strip('[').strip(']') - + return find.group(1).strip("[").strip("]") def __get_log_level(self, log_line): """ @@ -437,7 +599,16 @@ def __get_log_level(self, log_line): :param log_line :return: log level """ - level_lits = ["DEBUG ", "TRACE ", "INFO ", "WDIAG ", "WARN ", "EDIAG ", "ERROR ", "FATAL "] + level_lits = [ + "DEBUG ", + "TRACE ", + "INFO ", + "WDIAG ", + "WARN ", + "EDIAG ", + "ERROR ", + "FATAL ", + ] length = len(log_line) if length > 38: length = 38 @@ -454,14 +625,13 @@ def __get_overall_summary(node_summary_tuples, is_files=False): :param node_summary_tuple :return: a string indicating the overall summary """ - field_names = [ - "Node", "Status", "FileName", "ErrorCode", - "Message", "Count" - ] + field_names = ["Node", "Status", "FileName", "ErrorCode", "Message", "Count"] t = [] t_details = [] field_names_details = field_names - field_names_details.extend(["Cause", "Solution", "First Found Time", "Last Found Time", "Trace_IDS"]) + field_names_details.extend( + ["Cause", "Solution", "First Found Time", "Last Found Time", "Trace_IDS"] + ) for tup in node_summary_tuples: is_empty = True node = tup[0] @@ -469,37 +639,84 @@ def __get_overall_summary(node_summary_tuples, is_files=False): node_results = tup[3] if is_err: is_empty = False - t.append([node, "Error:" + tup[2] if is_err else "Completed", None, None, None, None]) - t_details.append([node, "Error:" + tup[2] if is_err else "Completed", None, None, None, None, None, None, None, None, None]) + t.append( + [ + node, + "Error:" + tup[2] if is_err else "Completed", + None, + None, + None, + None, + ] + ) + t_details.append( + [ + node, + "Error:" + tup[2] if is_err else "Completed", + None, + None, + None, + None, + None, + None, + None, + None, + None, + ] + ) for log_result in node_results: for ret_key, ret_value in log_result.items(): if ret_key is not None: error_code_info = OB_RET_DICT.get(ret_key, "") if len(error_code_info) > 3: is_empty = False - t.append([node, - "Error:" + tup[2] if is_err else "Completed", - ret_value["file_name"], - ret_key, - error_code_info[1], - ret_value["count"] - ]) - t_details.append([node, - "Error:" + tup[2] if is_err else "Completed", - ret_value["file_name"], - ret_key, - error_code_info[1], - ret_value["count"], - error_code_info[2], - error_code_info[3], - ret_value["first_found_time"], - ret_value["last_found_time"], - str(ret_value["trace_id_list"]) - ]) + t.append( + [ + node, + "Error:" + tup[2] if is_err else "Completed", + ret_value["file_name"], + ret_key, + error_code_info[1], + ret_value["count"], + ] + ) + t_details.append( + [ + node, + "Error:" + tup[2] if is_err else "Completed", + ret_value["file_name"], + ret_key, + error_code_info[1], + ret_value["count"], + error_code_info[2], + error_code_info[3], + ret_value["first_found_time"], + ret_value["last_found_time"], + str(ret_value["trace_id_list"]), + ] + ) if is_empty: t.append([node, "\033[32mPASS\033[0m", None, None, None, None]) - t_details.append([node, "\033[32mPASS\033[0m", None, None, None, None, None, None, None, None, None]) - title = "\nAnalyze OceanBase Offline Log Summary:\n" if is_files else "\nAnalyze OceanBase Online Log Summary:\n" + t_details.append( + [ + node, + "\033[32mPASS\033[0m", + None, + None, + None, + None, + None, + None, + None, + None, + None, + ] + ) + title = ( + "\nAnalyze OceanBase Offline Log Summary:\n" + if is_files + else "\nAnalyze OceanBase Online Log Summary:\n" + ) t.sort(key=lambda x: (x[0], x[1], x[2], x[3]), reverse=False) t_details.sort(key=lambda x: (x[0], x[1], x[2], x[3]), reverse=False) - return title, field_names, t, t_details \ No newline at end of file + return title, field_names, t, t_details diff --git a/handler/analyzer/log_parser/__init__.py b/handler/analyzer/log_parser/__init__.py index f5e16f89..5a69f20f 100644 --- a/handler/analyzer/log_parser/__init__.py +++ b/handler/analyzer/log_parser/__init__.py @@ -14,4 +14,4 @@ @time: 2023/11/07 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/analyzer/log_parser/log_entry.py b/handler/analyzer/log_parser/log_entry.py index 405192e0..84f6085f 100644 --- a/handler/analyzer/log_parser/log_entry.py +++ b/handler/analyzer/log_parser/log_entry.py @@ -34,14 +34,33 @@ def find_field_end(data, end_chs=",)}({|][", start=0, end=-1): class LogEntry(object): WF_LEVELS = {"ERROR", "WARN", "FATAL"} - __slots__ = ("id", "log_type", "log_file_offset", "log_text_length", "timestamp_us", "parse_succ", "is_trace", - "log_level", "component", "source_func", "source_file", "source_line", "th_id", "co_id", "trace_id", - "lt", "dc", "log_text", "content") + __slots__ = ( + "id", + "log_type", + "log_file_offset", + "log_text_length", + "timestamp_us", + "parse_succ", + "is_trace", + "log_level", + "component", + "source_func", + "source_file", + "source_line", + "th_id", + "co_id", + "trace_id", + "lt", + "dc", + "log_text", + "content", + ) """ Log entry parsed from the log text. This entry should be complete, meaning that we can resume the original log from this entry. At this stage we store the raw text in the field "log_text". """ + def __init__(self, log_id, log_type, file_offset, file_length): # key_info: only these vars dumped to files self.id = log_id @@ -83,7 +102,7 @@ def parse_from_data(self, data, time_slice): while idx != -1: next_idx = data.find(next_brac, idx) if next_idx == -1: - bare_elems.append(data[idx:len(data)].strip()) + bare_elems.append(data[idx : len(data)].strip()) break if cur_brac == "[": in_brac_elems.append(data[idx:next_idx].strip()) @@ -93,17 +112,17 @@ def parse_from_data(self, data, time_slice): break else: bare_elem = data[idx:next_idx].strip() - if bare_elem != '': + if bare_elem != "": bare_elems.append(bare_elem) idx = next_idx + 1 cur_brac, next_brac = next_brac, cur_brac - time_slice[0] += (time.time()-st) + time_slice[0] += time.time() - st st = time.time() if len(in_brac_elems) < 3 and "(tid:" in data: return if len(bare_elems[0]) > 6: # 处理某些log开头混乱的情况，需要把这部分包含的Level, Component, Source分开 - real_bare_content = bare_elems[0].split(' ') + real_bare_content = bare_elems[0].split(" ") self.log_level = real_bare_content[0] # no component name and no func name, level is INFO DEBUG if len(real_bare_content) == 2: @@ -113,20 +132,35 @@ def parse_from_data(self, data, time_slice): elif len(real_bare_content) == 3: if self.log_level in LogEntry.WF_LEVELS: # no func but have component - bare_elems = [real_bare_content[0], real_bare_content[2]] + bare_elems[1:] - in_brac_elems = [in_brac_elems[0], real_bare_content[1]] + in_brac_elems[1:] + bare_elems = [ + real_bare_content[0], + real_bare_content[2], + ] + bare_elems[1:] + in_brac_elems = [ + in_brac_elems[0], + real_bare_content[1], + ] + in_brac_elems[1:] else: # no component but have func - bare_elems = [real_bare_content[0], real_bare_content[2]] + bare_elems[1:] + bare_elems = [ + real_bare_content[0], + real_bare_content[2], + ] + bare_elems[1:] in_brac_elems = [in_brac_elems[0], ""] + in_brac_elems[1:] # has func and component, level is WARN ERROR FATAL elif len(real_bare_content) == 4: - bare_elems = [real_bare_content[0], real_bare_content[2]+" "+real_bare_content[3]] + bare_elems[1:] - in_brac_elems = [in_brac_elems[0], real_bare_content[1]] + in_brac_elems[1:] + bare_elems = [ + real_bare_content[0], + real_bare_content[2] + " " + real_bare_content[3], + ] + bare_elems[1:] + in_brac_elems = [ + in_brac_elems[0], + real_bare_content[1], + ] + in_brac_elems[1:] else: - time_slice[1] += (time.time() - st) + time_slice[1] += time.time() - st raise Exception() - time_slice[1] += (time.time()-st) + time_slice[1] += time.time() - st st = time.time() self.timestamp_us = TimeUtils.datetime_to_timestamp(in_brac_elems[0]) self.log_level = bare_elems[0] @@ -135,13 +169,13 @@ def parse_from_data(self, data, time_slice): if self.log_level in LogEntry.WF_LEVELS: func_file_tuple = location.split(" ") self.source_func = func_file_tuple[0] - file_line_str = func_file_tuple[1].strip('()') + file_line_str = func_file_tuple[1].strip("()") else: file_line_str = location - fl_tup = file_line_str.split(':') + fl_tup = file_line_str.split(":") self.source_file = fl_tup[0] self.source_line = int(fl_tup[1]) - time_slice[2] += (time.time()-st) + time_slice[2] += time.time() - st st = time.time() th_idx = 2 # 寻找lt和dc的位置 @@ -168,7 +202,11 @@ def parse_from_data(self, data, time_slice): break t_idx -= 1 if t_idx == -1: - raise Exception('No dc= field, no lt= field and no [T] field. data={0}'.format(data)) + raise Exception( + "No dc= field, no lt= field and no [T] field. data={0}".format( + data + ) + ) self.is_trace = True self.trace_id = in_brac_elems[t_idx - 1] if t_idx - 2 == th_idx: @@ -181,10 +219,10 @@ def parse_from_data(self, data, time_slice): else: # 无dc 有lt self.lt = in_brac_elems[lt_idx] - self.trace_id = in_brac_elems[lt_idx-1] + self.trace_id = in_brac_elems[lt_idx - 1] self.th_id = int(in_brac_elems[th_idx]) if lt_idx - 2 != th_idx: - self.co_id = int(in_brac_elems[lt_idx-2]) + self.co_id = int(in_brac_elems[lt_idx - 2]) else: # 有dc @@ -199,7 +237,7 @@ def parse_from_data(self, data, time_slice): self.co_id = in_brac_elems[dc_idx - 3] self.th_id = in_brac_elems[th_idx] - time_slice[3] += (time.time()-st) + time_slice[3] += time.time() - st st = time.time() # 日志内容解析 # 将非头部分提取为content @@ -215,13 +253,14 @@ def parse_from_data(self, data, time_slice): self.co_id = int(self.co_id) if self.co_id is not None else None # self.lt = int(self.lt.split("=")[1]) if self.lt is not None else None # self.dc = int(self.dc.split("=")[1]) if self.dc is not None else None - time_slice[4] += (time.time()-st) + time_slice[4] += time.time() - st self.parse_succ = True return @staticmethod def generate_log_entries_from_string_reader( - reader_io, parse_time_slice, max_read_n=-1, start_offset=0, start_log_id=0): + reader_io, parse_time_slice, max_read_n=-1, start_offset=0, start_log_id=0 + ): """ parse log entries from start point that user specified. A posix read-like stream reader interface. @@ -245,14 +284,19 @@ def generate_log_entries_from_string_reader( max_read_n = (1 << 64) - 1 while n_read < max_read_n: line = reader_io.readline() - if line == '': + if line == "": if len(log_entry_texts) == 0: # 最后一个且没有文本内容 break # make up the last record log_entry_text = "\n".join(log_entry_texts) try: - log_entry = LogEntry(log_id, None, log_entry_begin_offset, line_offset - log_entry_begin_offset) + log_entry = LogEntry( + log_id, + None, + log_entry_begin_offset, + line_offset - log_entry_begin_offset, + ) log_entry.parse_from_data(log_entry_text, parse_time_slice) success_log_entries.append(log_entry) n_read += 1 @@ -260,22 +304,26 @@ def generate_log_entries_from_string_reader( except Exception as e: irregular_logs.append(log_entry_text) break - if len(line) > 28 and line[0] == '[' and line[5] == "-" and line[8] == "-": + if len(line) > 28 and line[0] == "[" and line[5] == "-" and line[8] == "-": # if current at the first line, this does not mean the end of a log entry. # So we need to wait for another line that marks the begin of another log entry. if line_idx != 0: log_entry_text = "\n".join(log_entry_texts) st = time.time() try: - log_entry = LogEntry(log_id, None, log_entry_begin_offset, - line_offset - log_entry_begin_offset) + log_entry = LogEntry( + log_id, + None, + log_entry_begin_offset, + line_offset - log_entry_begin_offset, + ) log_entry.parse_from_data(log_entry_text, parse_time_slice) success_log_entries.append(log_entry) log_id += 1 n_read += 1 except Exception as e: irregular_logs.append(log_entry_text) - parse_time_slice[5] += (time.time() - st) + parse_time_slice[5] += time.time() - st log_entry_begin_offset = line_offset log_entry_texts = [line] else: diff --git a/handler/analyzer/log_parser/log_enum.py b/handler/analyzer/log_parser/log_enum.py index 8160263d..b5babfa9 100644 --- a/handler/analyzer/log_parser/log_enum.py +++ b/handler/analyzer/log_parser/log_enum.py @@ -19,8 +19,8 @@ class OBLogRoleConst(object): - OBLOG_ROLE_ROOTSERVICE = 'RS' - OBLOG_ROLE_OBSERVER = 'OBS' + OBLOG_ROLE_ROOTSERVICE = "RS" + OBLOG_ROLE_OBSERVER = "OBS" OBLOG_ROLE_SUPPORT_LIST = [OBLOG_ROLE_OBSERVER, OBLOG_ROLE_ROOTSERVICE] @@ -55,16 +55,16 @@ class AgentRuntimeGlobalVars(object): # dir name in pack class DirNameInPackConst(object): - TRIMMED_LOG_DIR_NAME_IN_PACK = 'trimmed_log' - OS_STATE_LOG_DIR_NAME_IN_PACK = 'os_state' + TRIMMED_LOG_DIR_NAME_IN_PACK = "trimmed_log" + OS_STATE_LOG_DIR_NAME_IN_PACK = "os_state" class PackCompressConst(object): - COMPRESS_ZIP = 'zip' - COMPRESS_ZSTD = 'zstd' + COMPRESS_ZIP = "zip" + COMPRESS_ZSTD = "zstd" CompressSuffixDict = { - COMPRESS_ZIP: '.zip', - COMPRESS_ZSTD: '.tar.zst', + COMPRESS_ZIP: ".zip", + COMPRESS_ZSTD: ".tar.zst", } COMPRESS_VALID_LIST = [COMPRESS_ZIP, COMPRESS_ZSTD] # 当前使用的压缩类型 diff --git a/handler/analyzer/log_parser/tree.py b/handler/analyzer/log_parser/tree.py index 89692745..0c009e5f 100644 --- a/handler/analyzer/log_parser/tree.py +++ b/handler/analyzer/log_parser/tree.py @@ -22,11 +22,23 @@ class Node: - __slots__ = ('id', 'value_list', 'p_node', 'c_nodes', 'tree_info', 'value', 'display_id', 'host_ip', 'host_type', 'host_info', 'index') + __slots__ = ( + "id", + "value_list", + "p_node", + "c_nodes", + "tree_info", + "value", + "display_id", + "host_ip", + "host_type", + "host_info", + "index", + ) def __init__(self, value: Union[Dict, str]): if isinstance(value, dict): - self.id = value['trace_data']['id'] + self.id = value["trace_data"]["id"] self.value = value self.value_list = [value] self.host_ip = value["host_ip"] @@ -41,7 +53,7 @@ def __init__(self, value: Union[Dict, str]): self.host_info = self.host_type + "(" + self.host_ip + ")" else: self.host_info = "" - self.tree_info = '' + self.tree_info = "" self.p_node = None self.c_nodes = {} self.index = 0 @@ -50,47 +62,55 @@ def __init__(self, value: Union[Dict, str]): @property def name(self): - return self.tree_info + self.value['trace_data']['name'] if self.value else 'TRACE' + return ( + self.tree_info + self.value["trace_data"]["name"] if self.value else "TRACE" + ) def _get_key_str(self, key): if not self.value: - return '' - v = self.value['trace_data'].get(key) - return '{}: {}'.format(key, v) if v else '' + return "" + v = self.value["trace_data"].get(key) + return "{}: {}".format(key, v) if v else "" @property def elapsed_time(self): if self.value: - return TimeUtils.trans_time(self.value['trace_data']['end_ts'] - self.value['trace_data']['start_ts']) - return '-' + return TimeUtils.trans_time( + self.value["trace_data"]["end_ts"] + - self.value["trace_data"]["start_ts"] + ) + return "-" @property def elapsed_time_us(self): if self.value: - return self.value['trace_data']['end_ts'] - self.value['trace_data']['start_ts'] + return ( + self.value["trace_data"]["end_ts"] + - self.value["trace_data"]["start_ts"] + ) return 0 @property def tags(self): - return self._get_key_str('tags') + return self._get_key_str("tags") @property def logs(self): - return self._get_key_str('logs') + return self._get_key_str("logs") def merge_key(self, key, value): - info = self.value['trace_data'].get(key, None) - info2 = value['trace_data'].get(key, None) + info = self.value["trace_data"].get(key, None) + info2 = value["trace_data"].get(key, None) if not info: - self.value['trace_data'][key] = info2 + self.value["trace_data"][key] = info2 return if not info2: return if isinstance(info, int) and isinstance(info2, int): - self.value['trace_data'][key] = max(info, info2) + self.value["trace_data"][key] = max(info, info2) elif isinstance(info, str) and isinstance(info2, str): if len(info2) > len(info): - self.value['trace_data'][key] = info2 + self.value["trace_data"][key] = info2 elif isinstance(info, list) and isinstance(info2, list): for v in info2: if v not in info: @@ -102,10 +122,14 @@ def merge_key(self, key, value): def _update_value(self, value): if self.value: - for key in ['tags', 'logs']: + for key in ["tags", "logs"]: self.merge_key(key, value) - self.value['trace_data']['start_ts'] = max(self.value['trace_data']['start_ts'], value['trace_data']['start_ts']) - self.value['trace_data']['end_ts'] = max(self.value['trace_data']['end_ts'], value['trace_data']['end_ts']) + self.value["trace_data"]["start_ts"] = max( + self.value["trace_data"]["start_ts"], value["trace_data"]["start_ts"] + ) + self.value["trace_data"]["end_ts"] = max( + self.value["trace_data"]["end_ts"], value["trace_data"]["end_ts"] + ) else: self.value = value @@ -126,19 +150,19 @@ def merge(self, node): assert self.id == node.id def __repr__(self): - return 'id:{} pid:{}'.format(self.id, self.p_node.id if self.p_node else None) + return "id:{} pid:{}".format(self.id, self.p_node.id if self.p_node else None) class TreeMeta: - TAGS = 'Tags' - LOGS = 'Logs' - SPAN_NAME = 'Span Name' - ELAPSED_TIME = 'Elapsed Time' - ID = 'ID' - HOSTS = 'HOSTS' + TAGS = "Tags" + LOGS = "Logs" + SPAN_NAME = "Span Name" + ELAPSED_TIME = "Elapsed Time" + ID = "ID" + HOSTS = "HOSTS" - detail_header = 'Tags & Logs:\n-------------------------------------' - details_data = '\nDetails:\n' + detail_header = "Tags & Logs:\n-------------------------------------" + details_data = "\nDetails:\n" def __init__(self): self.max_name = len(self.SPAN_NAME) @@ -161,18 +185,21 @@ def format_len(elem): else: keyword = elem ex = 0 - return '{: <' + str(keyword + ex) + 's}' - return '| %s|' % '| '.join(format_len(elem) for elem in self.fmt_elements).format(*args) + return "{: <" + str(keyword + ex) + "s}" + + return "| %s|" % "| ".join( + format_len(elem) for elem in self.fmt_elements + ).format(*args) @property def sep_line(self): li = [] for char in self.header: - if char == '|': - li.append('+') + if char == "|": + li.append("+") else: - li.append('-') - return ''.join(li) + li.append("-") + return "".join(li) @property def header(self): @@ -193,24 +220,37 @@ def body(self, index, node: Node): return self.format(str(index), node.name, node.elapsed_time, node.host_info) def topN_body(self, index, node: Node): - topN_name = str(node.name).replace('├', '').replace('│', '').replace('└', '').replace('─', '').replace(' ', '') + topN_name = ( + str(node.name) + .replace("├", "") + .replace("│", "") + .replace("└", "") + .replace("─", "") + .replace(" ", "") + ) return self.format(str(index), topN_name, node.elapsed_time, node.host_info) def detail(self, index, node: Node): tags = node.tags logs = node.logs hosts = node.host_info - st = node.value['trace_data'].get('start_ts') if node.value else None - et = node.value['trace_data'].get('end_ts') if node.value else None + st = node.value["trace_data"].get("start_ts") if node.value else None + et = node.value["trace_data"].get("end_ts") if node.value else None if st and et: - time_str = 'Elapsed: {}'.format(TimeUtils.trans_time(et - st)) + time_str = "Elapsed: {}".format(TimeUtils.trans_time(et - st)) else: - time_str = '' - return '{} - {} {} {}' \ - '{}{}{}{}{}'.format(index, node.value['trace_data']['name'] if node.value else '', time_str, - ('\n' + ' ' * (3 + len(str(index)))) if hosts else '', hosts, - ('\n' + ' ' * (3 + len(str(index)))) if tags else '', tags, - ('\n' + ' ' * (3 + len(str(index)))) if logs else '', logs) + time_str = "" + return "{} - {} {} {}" "{}{}{}{}{}".format( + index, + node.value["trace_data"]["name"] if node.value else "", + time_str, + ("\n" + " " * (3 + len(str(index)))) if hosts else "", + hosts, + ("\n" + " " * (3 + len(str(index)))) if tags else "", + tags, + ("\n" + " " * (3 + len(str(index)))) if logs else "", + logs, + ) def record_node_info(self, node: Node): self.counter += 1 @@ -219,13 +259,15 @@ def record_node_info(self, node: Node): self.max_hosts = max(self.max_hosts, len(node.host_info)) def __str__(self): - return 'TreeMeta: counter {} max_name_len {}'.format(self.counter, self.max_name) + return "TreeMeta: counter {} max_name_len {}".format( + self.counter, self.max_name + ) class Tree: - root_id = '00000000-0000-0000-0000-000000000000' + root_id = "00000000-0000-0000-0000-000000000000" - __slots__ = ('nodes', 'meta', 'leaf_childs') + __slots__ = ("nodes", "meta", "leaf_childs") def __init__(self): self.nodes: Dict[str, Node] = {} @@ -236,8 +278,8 @@ def __len__(self): return len(self.nodes) def insert(self, di: Dict) -> None: - id = di['trace_data']['id'] - parent_id = di['trace_data']['parent_id'] + id = di["trace_data"]["id"] + parent_id = di["trace_data"]["parent_id"] if id not in self.nodes: self.nodes[id] = Node(di) else: @@ -259,26 +301,32 @@ def no_parent_node(self) -> List[Node]: if node.p_node is None: yield node - def record_meta(self, root_node, max_recursion, order_by='start_ts'): - node_chars = '└─├─' - child_chars = ' │ ' + def record_meta(self, root_node, max_recursion, order_by="start_ts"): + node_chars = "└─├─" + child_chars = " │ " meta_data = TreeMeta() def recurse(node: Node, parent_info): if max_recursion != -1 and len(parent_info) / 2 > max_recursion: return if parent_info: - node.tree_info = parent_info[:-1].replace(node_chars[0:2], child_chars[0:2]). \ - replace(node_chars[2:4], child_chars[2:4]) + parent_info[-1] + node.tree_info = ( + parent_info[:-1] + .replace(node_chars[0:2], child_chars[0:2]) + .replace(node_chars[2:4], child_chars[2:4]) + + parent_info[-1] + ) else: - node.tree_info = '' + node.tree_info = "" meta_data.record_node_info(node) node.set_index(meta_data.counter) if len(node.c_nodes) == 0: self.leaf_childs[node.id] = node yield node - ordered_list = sorted(node.c_nodes.values(), key=lambda x: x.value['trace_data'][order_by]) + ordered_list = sorted( + node.c_nodes.values(), key=lambda x: x.value["trace_data"][order_by] + ) for index, c_node in enumerate(ordered_list): total = len(node.c_nodes) if total == 1 or index == total - 1: @@ -288,7 +336,7 @@ def recurse(node: Node, parent_info): new_parent_info = parent_info + char yield from recurse(c_node, new_parent_info) - yield from recurse(root_node, '') + yield from recurse(root_node, "") self.meta[root_node] = meta_data def _traverse(self, root_node: Node, max_recursion=3, top_n=5): @@ -301,22 +349,36 @@ def _traverse(self, root_node: Node, max_recursion=3, top_n=5): topN = heapq.nlargest(top_n, leaf_nodes.items(), lambda x: x[1].elapsed_time_us) topN_meta = TreeMeta() topN_counter = 0 - table = PrettyTable(['ID','Leaf Span Name','Elapsed Time','HOSTS']) - table.align = 'l' + table = PrettyTable(["ID", "Leaf Span Name", "Elapsed Time", "HOSTS"]) + table.align = "l" while topN: topN_counter += 1 element = heapq.heappop(topN) - topN_name = str(element[1].name).replace('├', '').replace('│', '').replace('└', '').replace('─', '').replace(' ', '') - table.add_row([element[1].index, topN_name, element[1].elapsed_time, element[1].host_info]) + topN_name = ( + str(element[1].name) + .replace("├", "") + .replace("│", "") + .replace("└", "") + .replace("─", "") + .replace(" ", "") + ) + table.add_row( + [ + element[1].index, + topN_name, + element[1].elapsed_time, + element[1].host_info, + ] + ) topN_meta.record_node_info(element[1]) topN_li.append(element[1]) if topN_counter > 0: - yield '\nTop time-consuming leaf span:\n' + yield "\nTop time-consuming leaf span:\n" for line in str(table).splitlines(): yield str(line) yield topN_meta.detail_header for node in topN_li: - yield topN_meta.detail(node.index,node) + yield topN_meta.detail(node.index, node) meta = self.meta[root_node] yield meta.details_data @@ -332,11 +394,11 @@ def _traverse(self, root_node: Node, max_recursion=3, top_n=5): def traverse(self, max_recursion, topN): for node in self.no_parent_node(): - yield 'root node id: {}'.format(node.id) + yield "root node id: {}".format(node.id) self.leaf_childs = {} for item in self._traverse(node, max_recursion, topN): yield item - yield '\n\n' + yield "\n\n" def build(self, data: list): for item in data: @@ -362,7 +424,7 @@ def find_parents(node: Node): def search_name(self, name: str) -> list: nodes = set() for node in self.nodes.values(): - if node.value['trace_data']['name'] == name: + if node.value["trace_data"]["name"] == name: parents = self.find_parents(node) nodes.update(parents) data = [] diff --git a/handler/checker/__init__.py b/handler/checker/__init__.py index 681bdff5..70691bef 100644 --- a/handler/checker/__init__.py +++ b/handler/checker/__init__.py @@ -15,4 +15,3 @@ @file: __init__.py @desc: """ - diff --git a/handler/checker/check_exception.py b/handler/checker/check_exception.py index b1019974..1c9f8889 100644 --- a/handler/checker/check_exception.py +++ b/handler/checker/check_exception.py @@ -23,7 +23,10 @@ def __init__(self, msg=None, obj=None): self.msg, self.obj = msg, obj def __repr__(self): - return '%s %s' % (self.msg, self.obj is not None and pprint.pformat(self.obj) or '') + return "%s %s" % ( + self.msg, + self.obj is not None and pprint.pformat(self.obj) or "", + ) def __str__(self): return repr(self) @@ -52,24 +55,24 @@ def __init__(self, msg=None, obj=None): class ResultFalseException(CheckException): def __init__(self, msg=None, obj=None): - super(ResultFalseException,self).__init__(msg, obj) + super(ResultFalseException, self).__init__(msg, obj) class ResultFailException(CheckException): def __init__(self, msg=None, obj=None): - super(ResultFailException,self).__init__(msg, obj) + super(ResultFailException, self).__init__(msg, obj) class VerifyFalseException(CheckException): def __init__(self, msg=None, obj=None): - super(VerifyFalseException,self).__init__(msg, obj) + super(VerifyFalseException, self).__init__(msg, obj) class VerifyFailException(CheckException): def __init__(self, msg=None, obj=None): - super(VerifyFailException,self).__init__(msg, obj) + super(VerifyFailException, self).__init__(msg, obj) class TaskException(CheckException): def __init__(self, msg=None, obj=None): - super(TaskException,self).__init__(msg, obj) + super(TaskException, self).__init__(msg, obj) diff --git a/handler/checker/check_handler.py b/handler/checker/check_handler.py index f225832c..5040db3d 100644 --- a/handler/checker/check_handler.py +++ b/handler/checker/check_handler.py @@ -33,49 +33,66 @@ from common.tool import YamlUtils from common.tool import StringUtils + class CheckHandler: - def __init__(self, context, check_target_type="observer"): + def __init__(self, context, check_target_type="observer"): self.context = context self.stdio = context.stdio # init input parameters self.report = None self.tasks = None - self.work_path = os.path.expanduser(self.context.inner_config["check"]["work_path"] or "~/.obdiag/check") - self.export_report_path=os.path.expanduser(self.context.inner_config["check"]["report"]["report_path"] or "./check_report/") - self.export_report_type = self.context.inner_config["check"]["report"]["export_type"] or "table" - self.ignore_version = self.context.inner_config["check"]["ignore_version"] or False + self.work_path = os.path.expanduser( + self.context.inner_config["check"]["work_path"] or "~/.obdiag/check" + ) + self.export_report_path = os.path.expanduser( + self.context.inner_config["check"]["report"]["report_path"] + or "./check_report/" + ) + self.export_report_type = ( + self.context.inner_config["check"]["report"]["export_type"] or "table" + ) + self.ignore_version = ( + self.context.inner_config["check"]["ignore_version"] or False + ) self.cluster = self.context.cluster_config - if check_target_type=="observer": - self.nodes =self.context.cluster_config.get("servers") + if check_target_type == "observer": + self.nodes = self.context.cluster_config.get("servers") if check_target_type == "obproxy": self.nodes = self.context.obproxy_config.get("servers") self.tasks_base_path = os.path.expanduser(self.work_path + "/tasks/") self.check_target_type = check_target_type - self.stdio.verbose("CheckHandler input. ignore_version is {0} , cluster is {1} , nodes is {2}, " - "export_report_path is {3}, export_report_type is {4} , check_target_type is {5}, " - " tasks_base_path is {6}.".format(self.ignore_version, - self.cluster.get( - "ob_cluster_name") or self.cluster.get( - "obproxy_cluster_name"), - StringUtils.node_cut_passwd_for_log(self.nodes), - self.export_report_path, - self.export_report_type, - self.check_target_type, - self.tasks_base_path)) + self.stdio.verbose( + "CheckHandler input. ignore_version is {0} , cluster is {1} , nodes is {2}, " + "export_report_path is {3}, export_report_type is {4} , check_target_type is {5}, " + " tasks_base_path is {6}.".format( + self.ignore_version, + self.cluster.get("ob_cluster_name") + or self.cluster.get("obproxy_cluster_name"), + StringUtils.node_cut_passwd_for_log(self.nodes), + self.export_report_path, + self.export_report_type, + self.check_target_type, + self.tasks_base_path, + ) + ) # case_package_file # build case_package_file if check_target_type is not None: - case_package_file = self.work_path + "/" + check_target_type + "_check_package.yaml" + case_package_file = ( + self.work_path + "/" + check_target_type + "_check_package.yaml" + ) else: raise CheckException("check_target_type is null. Please check the conf") case_package_file = os.path.expanduser(case_package_file) if os.path.exists(case_package_file): self.package_file_name = case_package_file else: - raise CheckException("case_package_file {0} is not exist".format(case_package_file)) + raise CheckException( + "case_package_file {0} is not exist".format(case_package_file) + ) self.stdio.verbose("case_package_file is " + self.package_file_name) # checker tasks_base_path # build tasks_base_path @@ -87,54 +104,72 @@ def __init__(self, context, check_target_type="observer"): if os.path.exists(tasks_base_path): self.tasks_base_path = tasks_base_path else: - raise CheckException("tasks_base_path {0} is not exist".format(tasks_base_path)) + raise CheckException( + "tasks_base_path {0} is not exist".format(tasks_base_path) + ) self.stdio.verbose("tasks_base_path is " + self.tasks_base_path) # input_param - self.options=self.context.options + self.options = self.context.options # add ssher - new_node=[] + new_node = [] for node in self.nodes: # add ssher ssher = None try: - ssher = SshHelper(True, node.get("ip"), - node.get("ssh_username"), - node.get("ssh_password"), - node.get("ssh_port"), - node.get("ssh_key_file"), - node) + ssher = SshHelper( + True, + node.get("ip"), + node.get("ssh_username"), + node.get("ssh_password"), + node.get("ssh_port"), + node.get("ssh_key_file"), + node, + ) except Exception as e: - self.stdio.warn("StepBase get SshHelper fail on{0} ,Exception: {1}".format(node.get("ip"), e)) + self.stdio.warn( + "StepBase get SshHelper fail on{0} ,Exception: {1}".format( + node.get("ip"), e + ) + ) node["ssher"] = ssher new_node.append(node) - self.nodes=new_node - self.version=get_version(self.nodes, self.check_target_type,self.cluster, self.stdio) + self.nodes = new_node + self.version = get_version( + self.nodes, self.check_target_type, self.cluster, self.stdio + ) # add OBConnectorPool try: - obConnectorPool=checkOBConnectorPool(context,3,self.cluster) + obConnectorPool = checkOBConnectorPool(context, 3, self.cluster) except Exception as e: self.stdio.warn("obConnector init error. Error info is {0}".format(e)) finally: - self.context.set_variable('check_obConnector_pool', obConnectorPool) - + self.context.set_variable("check_obConnector_pool", obConnectorPool) def handle(self): try: package_name = None - if self.check_target_type == "obproxy" and Util.get_option(self.options, 'obproxy_cases'): - package_name = Util.get_option(self.options, 'obproxy_cases') + if self.check_target_type == "obproxy" and Util.get_option( + self.options, "obproxy_cases" + ): + package_name = Util.get_option(self.options, "obproxy_cases") - if self.check_target_type == "observer" and Util.get_option(self.options, 'cases'): - package_name = Util.get_option(self.options, 'cases') - if Util.get_option(self.options, 'store_dir'): - self.export_report_path = Util.get_option(self.options, 'store_dir') - self.stdio.verbose("export_report_path change to " + self.export_report_path) + if self.check_target_type == "observer" and Util.get_option( + self.options, "cases" + ): + package_name = Util.get_option(self.options, "cases") + if Util.get_option(self.options, "store_dir"): + self.export_report_path = Util.get_option(self.options, "store_dir") + self.stdio.verbose( + "export_report_path change to " + self.export_report_path + ) self.export_report_path = os.path.expanduser(self.export_report_path) if not os.path.exists(self.export_report_path): - self.stdio.warn("{0} not exists. mkdir it!".format(self.export_report_path)) + self.stdio.warn( + "{0} not exists. mkdir it!".format(self.export_report_path) + ) os.mkdir(self.export_report_path) self.stdio.verbose("export_report_path is " + self.export_report_path) @@ -157,8 +192,11 @@ def handle(self): self.get_all_tasks() filter_tasks = self.get_package_tasks("filter") if len(filter_tasks) > 0: - self.tasks = {key: value for key, value in self.tasks.items() if key not in - filter_tasks} + self.tasks = { + key: value + for key, value in self.tasks.items() + if key not in filter_tasks + } new_tasks = {} for filter_task in filter_tasks: for task_name, task_value in self.tasks.items(): @@ -175,9 +213,9 @@ def get_all_tasks(self): tasks = {} for root, dirs, files in os.walk(current_path): for file in files: - if file.endswith('.yaml'): + if file.endswith(".yaml"): folder_name = os.path.basename(root) - task_name = "{}.{}".format(folder_name, file.split('.')[0]) + task_name = "{}.{}".format(folder_name, file.split(".")[0]) task_data = YamlUtils.read_yaml_data(os.path.join(root, file)) tasks[task_name] = task_data if len(tasks) == 0: @@ -187,7 +225,7 @@ def get_all_tasks(self): # need package_name def get_package_tasks(self, package_name): # Obtain information within the package file - with open(self.package_file_name, 'r') as file: + with open(self.package_file_name, "r") as file: package_file_data = yaml.safe_load(file) packege_tasks = package_file_data if package_name not in packege_tasks: @@ -195,7 +233,11 @@ def get_package_tasks(self, package_name): return [] else: raise CheckException("no cases name is {0}".format(package_name)) - self.stdio.verbose("by cases name: {0} , get cases: {1}".format(package_name, packege_tasks[package_name])) + self.stdio.verbose( + "by cases name: {0} , get cases: {1}".format( + package_name, packege_tasks[package_name] + ) + ) if packege_tasks[package_name].get("tasks") is None: return [] return packege_tasks[package_name].get("tasks") @@ -205,13 +247,21 @@ def execute_one(self, task_name): try: self.stdio.verbose("execute tasks is {0}".format(task_name)) # Verify if the version is within a reasonable range - report = TaskReport(self.context,task_name) + report = TaskReport(self.context, task_name) if not self.ignore_version: version = self.version if version: self.cluster["version"] = version - self.stdio.verbose("cluster.version is {0}".format(self.cluster["version"])) - task = TaskBase(self.context, self.tasks[task_name]["task"], self.nodes, self.cluster, report) + self.stdio.verbose( + "cluster.version is {0}".format(self.cluster["version"]) + ) + task = TaskBase( + self.context, + self.tasks[task_name]["task"], + self.nodes, + self.cluster, + report, + ) self.stdio.verbose("{0} execute!".format(task_name)) task.execute() self.stdio.verbose("execute tasks end : {0}".format(task_name)) @@ -227,11 +277,16 @@ def execute_one(self, task_name): def execute(self): try: self.stdio.verbose( - "execute_all_tasks. the number of tasks is {0} ,tasks is {1}".format(len(self.tasks.keys()), - self.tasks.keys())) - self.report = CheckReport(self.context, export_report_path=self.export_report_path, - export_report_type=self.export_report_type, - report_target=self.check_target_type) + "execute_all_tasks. the number of tasks is {0} ,tasks is {1}".format( + len(self.tasks.keys()), self.tasks.keys() + ) + ) + self.report = CheckReport( + self.context, + export_report_path=self.export_report_path, + export_report_type=self.export_report_type, + report_target=self.check_target_type, + ) # one of tasks to execute for task in self.tasks: t_report = self.execute_one(task) @@ -242,12 +297,13 @@ def execute(self): except Exception as e: self.stdio.error("Internal error :{0}".format(e)) + class checkOBConnectorPool: - def __init__(self,context, max_size, cluster): + def __init__(self, context, max_size, cluster): self.max_size = max_size - self.cluster=cluster + self.cluster = cluster self.connections = queue.Queue(maxsize=max_size) - self.stdio=context.stdio + self.stdio = context.stdio self.stdio.verbose("obConnectorPool init success!") try: for i in range(max_size): @@ -257,14 +313,13 @@ def __init__(self,context, max_size, cluster): username=self.cluster.get("tenant_sys").get("user"), password=self.cluster.get("tenant_sys").get("password"), stdio=self.stdio, - timeout=10000 + timeout=10000, ) self.connections.put(conn) self.stdio.verbose("obConnectorPool init success!") except Exception as e: self.stdio.error("obConnectorPool init fail! err:".format(e)) - def get_connection(self): try: return self.connections.get() @@ -277,4 +332,3 @@ def release_connection(self, conn): if conn is not None: self.connections.put(conn) return - diff --git a/handler/checker/check_list.py b/handler/checker/check_list.py index f885fe74..71aa4c9d 100644 --- a/handler/checker/check_list.py +++ b/handler/checker/check_list.py @@ -26,7 +26,9 @@ class CheckListHandler: def __init__(self, context): self.context = context self.stdio = context.stdio - self.work_path = os.path.expanduser(self.context.inner_config["check"]["work_path"] or "~/.obdiag/check") + self.work_path = os.path.expanduser( + self.context.inner_config["check"]["work_path"] or "~/.obdiag/check" + ) def handle(self): self.stdio.verbose("list check cases") @@ -34,22 +36,29 @@ def handle(self): files = [f for f in entries if os.path.isfile(os.path.join(self.work_path, f))] for file in files: if "check_package" in file: - cases_map = {"all": {"name": "all", "command": "obdiag check", - "info_en": "default check all task without filter", - "info_cn": "默认执行除filter组里的所有巡检项"}} + cases_map = { + "all": { + "name": "all", + "command": "obdiag check", + "info_en": "default check all task without filter", + "info_cn": "默认执行除filter组里的所有巡检项", + } + } # Obtain which files match and corresponding header files # Using string segmentation methods - parts = file.split('_') + parts = file.split("_") if len(parts) < 1: self.stdio.warn( "invalid check package name :{0} , Please don't add file, which 'check_package' in the name".format( - file)) + file + ) + ) continue target = parts[0] file = "{0}/{1}".format(self.work_path, file) package_file_data = None # read yaml file - with open(file, 'r') as f: + with open(file, "r") as f: package_file_data = yaml.safe_load(f) if not package_file_data or len(package_file_data) == 0: self.stdio.warn("No data check package data :{0} ".format(file)) @@ -63,10 +72,15 @@ def handle(self): else: package_target = "{0}_cases".format(target) - cases_map[package_data] = {"name": package_data, - "command": "obdiag check --{0}={1}".format(package_target, - package_data), - "info_en": package_file_data[package_data].get("info_en") or "", - "info_cn": package_file_data[package_data].get("info_cn") or ""} + cases_map[package_data] = { + "name": package_data, + "command": "obdiag check --{0}={1}".format( + package_target, package_data + ), + "info_en": package_file_data[package_data].get("info_en") + or "", + "info_cn": package_file_data[package_data].get("info_cn") + or "", + } Util.print_title("check cases about {0}".format(target)) Util.print_scene(cases_map) diff --git a/handler/checker/check_report.py b/handler/checker/check_report.py index c0790061..50721452 100644 --- a/handler/checker/check_report.py +++ b/handler/checker/check_report.py @@ -34,8 +34,13 @@ class CheckReport: - def __init__(self, context, report_target="observer", export_report_path="./check_report/", - export_report_type="table"): + def __init__( + self, + context, + report_target="observer", + export_report_path="./check_report/", + export_report_type="table", + ): self.context = context self.stdio = context.stdio self.tasks = [] @@ -63,7 +68,10 @@ def add_task_report(self, task_report): def export_report(self): self.stdio.verbose( - "export report to {0}.{1}, export type is {1}".format(self.report_path, self.export_report_type)) + "export report to {0}.{1}, export type is {1}".format( + self.report_path, self.export_report_type + ) + ) try: if self.export_report_type == "table": self.export_report_table() @@ -74,8 +82,14 @@ def export_report(self): elif self.export_report_type == "yaml": self.export_report_yaml() else: - raise CheckrReportException("export_report_type: {0} is not support".format(self.export_report_type)) - self.export_report_path = self.export_report_path + "." + self.export_report_type + raise CheckrReportException( + "export_report_type: {0} is not support".format( + self.export_report_type + ) + ) + self.export_report_path = ( + self.export_report_path + "." + self.export_report_type + ) except Exception as e: self.stdio.error("export_report Exception : {0}".format(e)) raise CheckrReportException(e) @@ -85,7 +99,7 @@ def get_report_path(self): def export_report_xml(self): allMap = self.report_tobeMap() - with open(self.report_path + ".xml", 'w', encoding="utf8") as f: + with open(self.report_path + ".xml", "w", encoding="utf8") as f: allreport = {} allreport["report"] = allMap json_str = json.dumps(allreport) @@ -95,13 +109,13 @@ def export_report_xml(self): def export_report_yaml(self): allMap = self.report_tobeMap() - with open(self.report_path + ".yaml", 'w', encoding="utf8") as f: + with open(self.report_path + ".yaml", "w", encoding="utf8") as f: yaml.dump(allMap, f) def export_report_json(self): allMap = self.report_tobeMap() self.stdio.verbose("export_report_json allMap: {0}".format(allMap)) - with open(self.report_path + ".json", 'w', encoding="utf8") as f: + with open(self.report_path + ".json", "w", encoding="utf8") as f: # for python2 and python3 try: json.dump(allMap, f, ensure_ascii=False) @@ -128,9 +142,14 @@ def report_tobeMap(self): allMap["critical"] = criticalMap allMap["warning"] = warningMap allMap["all"] = allInfoMap - telemetry.push_check_info(self.report_target, - {"fail_cases": list(failMap), "critical_cases": list(criticalMap), - "warning_cases": list(warningMap)}) + telemetry.push_check_info( + self.report_target, + { + "fail_cases": list(failMap), + "critical_cases": list(criticalMap), + "warning_cases": list(warningMap), + }, + ) return allMap def export_report_table(self): @@ -157,23 +176,36 @@ def export_report_table(self): for task in self.tasks: if len(task.all_fail()) != 0: - report_fail_tb.add_row([task.name, '\n'.join(task.all_fail())]) + report_fail_tb.add_row([task.name, "\n".join(task.all_fail())]) failMap.append(task.name) if len(task.all_critical()) != 0: - report_critical_tb.add_row([task.name, '\n'.join(task.all_critical())]) + report_critical_tb.add_row( + [task.name, "\n".join(task.all_critical())] + ) criticalMap.append(task.name) if len(task.all_warning()) != 0: - report_warning_tb.add_row([task.name, '\n'.join(task.all_warning())]) + report_warning_tb.add_row( + [task.name, "\n".join(task.all_warning())] + ) warningMap.append(task.name) if len(task.all()) != 0: - report_all_tb.add_row([task.name, '\n'.join(task.all())]) - if len(task.all_fail()) == 0 and len(task.all_critical()) == 0 and len(task.all_warning()) == 0: + report_all_tb.add_row([task.name, "\n".join(task.all())]) + if ( + len(task.all_fail()) == 0 + and len(task.all_critical()) == 0 + and len(task.all_warning()) == 0 + ): report_all_tb.add_row([task.name, "all pass"]) - telemetry.push_check_info(self.report_target, - {"fail_cases": list(set(failMap)), "critical_cases": list(set(criticalMap)), - "warning_cases": list(set(warningMap))}) - - fp = open(self.report_path + ".table", 'a+', encoding="utf8") + telemetry.push_check_info( + self.report_target, + { + "fail_cases": list(set(failMap)), + "critical_cases": list(set(criticalMap)), + "warning_cases": list(set(warningMap)), + }, + ) + + fp = open(self.report_path + ".table", "a+", encoding="utf8") if len(report_fail_tb._rows) != 0: self.stdio.verbose(report_fail_tb) diff --git a/handler/checker/check_task.py b/handler/checker/check_task.py index 7e634e0a..25cea59b 100644 --- a/handler/checker/check_task.py +++ b/handler/checker/check_task.py @@ -18,8 +18,12 @@ import threading from common.ob_connector import OBConnector -from handler.checker.check_exception import StepResultFailException, \ - StepExecuteFailException, StepResultFalseException, TaskException +from handler.checker.check_exception import ( + StepResultFailException, + StepExecuteFailException, + StepResultFalseException, + TaskException, +) from handler.checker.step.stepbase import StepBase from common.tool import StringUtils from common.scene import filter_by_version @@ -51,39 +55,57 @@ def execute(self): # TODO: 这里的逻辑需要优化，如果一个节点执行失败了，那么后续的步骤就不会被执行了。 work_threads = [] for node in self.nodes: - t = threading.Thread(target=self.execute_one_node, args=(steps_nu,node)) + t = threading.Thread(target=self.execute_one_node, args=(steps_nu, node)) work_threads.append(t) t.start() for t in work_threads: t.join() self.stdio.verbose("task execute end") - def execute_one_node(self,steps_nu,node): + + def execute_one_node(self, steps_nu, node): try: - self.stdio.verbose("run task in node: {0}".format(StringUtils.node_cut_passwd_for_log(node))) + self.stdio.verbose( + "run task in node: {0}".format( + StringUtils.node_cut_passwd_for_log(node) + ) + ) steps = self.task[steps_nu] nu = 1 - task_variable_dict={} + task_variable_dict = {} for step in steps["steps"]: try: self.stdio.verbose("step nu: {0}".format(nu)) if len(self.cluster) == 0: raise Exception("cluster is not exist") - step_run = StepBase(self.context, step, node, self.cluster, task_variable_dict) + step_run = StepBase( + self.context, step, node, self.cluster, task_variable_dict + ) self.stdio.verbose("step nu: {0} initted, to execute".format(nu)) step_run.execute(self.report) - task_variable_dict=step_run.update_task_variable_dict() - if "report_type" in step["result"] and step["result"]["report_type"] == "execution": + task_variable_dict = step_run.update_task_variable_dict() + if ( + "report_type" in step["result"] + and step["result"]["report_type"] == "execution" + ): self.stdio.verbose("report_type stop this step") return except StepExecuteFailException as e: - self.stdio.error("TaskBase execute CheckStepFailException: {0} . Do Next Task".format(e)) + self.stdio.error( + "TaskBase execute CheckStepFailException: {0} . Do Next Task".format( + e + ) + ) return except StepResultFalseException as e: - self.stdio.warn("TaskBase execute StepResultFalseException: {0} .".format(e)) + self.stdio.warn( + "TaskBase execute StepResultFalseException: {0} .".format(e) + ) continue except StepResultFailException as e: - self.stdio.warn("TaskBase execute StepResultFailException: {0}".format(e)) + self.stdio.warn( + "TaskBase execute StepResultFailException: {0}".format(e) + ) return except Exception as e: self.stdio.error("TaskBase execute Exception: {0}".format(e)) @@ -94,5 +116,3 @@ def execute_one_node(self,steps_nu,node): except Exception as e: self.stdio.error("TaskBase execute Exception: {0}".format(e)) raise e - - diff --git a/handler/checker/result/__init__.py b/handler/checker/result/__init__.py index 681bdff5..70691bef 100644 --- a/handler/checker/result/__init__.py +++ b/handler/checker/result/__init__.py @@ -15,4 +15,3 @@ @file: __init__.py @desc: """ - diff --git a/handler/checker/result/result.py b/handler/checker/result/result.py index 9e505699..5e9a8d0c 100644 --- a/handler/checker/result/result.py +++ b/handler/checker/result/result.py @@ -15,7 +15,11 @@ @file: result.py @desc: """ -from handler.checker.check_exception import ResultFalseException, ResultFailException, VerifyFailException +from handler.checker.check_exception import ( + ResultFalseException, + ResultFailException, + VerifyFailException, +) from handler.checker.result.verify import VerifyResult import re @@ -24,8 +28,9 @@ # validation process, handle it as fail); VerifyException (verification failed, report needs to be combined with # report_type) + class CheckResult: - def __init__(self,context, step_result_info, variable_dict): + def __init__(self, context, step_result_info, variable_dict): self.context = context self.stdio = context.stdio self.step_result_info = step_result_info @@ -42,25 +47,35 @@ def execute(self): # if verify in step.result[] if "verify" in self.step_result_info: try: - verify = VerifyResult(self.context,self.step_result_info["verify"], - self.variable_dict, self.step_result_info["set_value"], verify_type) + verify = VerifyResult( + self.context, + self.step_result_info["verify"], + self.variable_dict, + self.step_result_info["set_value"], + verify_type, + ) result = verify.execute() - self.stdio.verbose("verify.execute end. and result is {0}".format(result)) + self.stdio.verbose( + "verify.execute end. and result is {0}".format(result) + ) except Exception as e: - self.stdio.error("check_result execute VerifyFailException :{0}".format(e)) + self.stdio.error( + "check_result execute VerifyFailException :{0}".format(e) + ) raise ResultFailException(e) if not result: err_msg = self.build_msg() self.stdio.verbose( - "verify.execute end. and result is false return ResultFalseException err_msg:{0}".format(err_msg)) + "verify.execute end. and result is false return ResultFalseException err_msg:{0}".format( + err_msg + ) + ) raise ResultFalseException(err_msg) - - def build_msg(self): s = "the step is not pass" - if 'err_msg' in self.step_result_info: + if "err_msg" in self.step_result_info: s = self.step_result_info["err_msg"] d = self.variable_dict @@ -68,6 +83,4 @@ def replacer(match): key = match.group(1) return str(d.get(key, match.group(0))) - return re.sub(r'#\{(\w+)\}', replacer, s) - - + return re.sub(r"#\{(\w+)\}", replacer, s) diff --git a/handler/checker/result/verify.py b/handler/checker/result/verify.py index 19f470d4..72d88906 100644 --- a/handler/checker/result/verify.py +++ b/handler/checker/result/verify.py @@ -27,7 +27,9 @@ class VerifyResult(object): # There are three types of validation results: pass; VerifyFailException (if an exception occurs during the # validation process, handle it as fail); VerifyException (verification failed, report needs to be combined with # report_type) - def __init__(self, context, expr, env_dict, now_step_set_value_name, verify_type="base"): + def __init__( + self, context, expr, env_dict, now_step_set_value_name, verify_type="base" + ): self.context = context self.stdio = context.stdio self.expr = expr @@ -40,7 +42,9 @@ def execute(self): self.stdio.verbose("verify_type input is {0}".format(self.verify_type)) if self.verify_type is None or self.verify_type == "": self.verify_type = "base" - self.stdio.verbose("verify_type input is {0}, to set base".format(self.verify_type)) + self.stdio.verbose( + "verify_type input is {0}, to set base".format(self.verify_type) + ) if self.verify_type == "base": return self._verify_base() elif self.verify_type == "between": @@ -56,9 +60,13 @@ def execute(self): def _verify_between(self): try: - result = StringUtils.parse_range_string(self.expr, self.env_dict[self.now_step_set_value_name]) + result = StringUtils.parse_range_string( + self.expr, self.env_dict[self.now_step_set_value_name] + ) except Exception as e: - self.stdio.error("parse_range_string error: " + self.expr + "->" + e.__str__()) + self.stdio.error( + "parse_range_string error: " + self.expr + "->" + e.__str__() + ) raise VerifyFailException(e) return result @@ -66,38 +74,46 @@ def _verify_base(self): check_verify_shell = GlobalCheckMeta().get_value(key="check_verify_shell") try: self.stdio.verbose("the result verify is {0}".format(self.expr)) - real_shell = re.sub(r'\$\{([^}]+)\}', self.expr, check_verify_shell) + real_shell = re.sub(r"\$\{([^}]+)\}", self.expr, check_verify_shell) for env in self.env_dict: - self.stdio.verbose("add env: {0} ,the value:{1} , the type:{2}".format(env, self.env_dict[env], - type(self.env_dict[env]))) + self.stdio.verbose( + "add env: {0} ,the value:{1} , the type:{2}".format( + env, self.env_dict[env], type(self.env_dict[env]) + ) + ) if isinstance(self.env_dict[env], int): - real_shell = env + '=' + str(self.env_dict[env]) + '\n' + real_shell + real_shell = env + "=" + str(self.env_dict[env]) + "\n" + real_shell else: - real_shell = env + '="' + str(self.env_dict[env]) + '"\n' + real_shell + real_shell = ( + env + '="' + str(self.env_dict[env]) + '"\n' + real_shell + ) self.stdio.verbose("real_shell: {0}".format(real_shell)) process = subprocess.Popen(real_shell, shell=True, stdout=subprocess.PIPE) out, err = process.communicate() process.stdout.close() - result = out[:-1].decode('utf-8') + result = out[:-1].decode("utf-8") self.stdio.verbose("_verify_base result: {0}".format(result)) return result == "true" except Exception as e: self.stdio.error("_verify_base error: {0} -> {1}".format(str(self.expr), e)) - raise VerifyFailException("_verify_base error: " + self.expr + "->" + e.__str__()) + raise VerifyFailException( + "_verify_base error: " + self.expr + "->" + e.__str__() + ) def _verify_max(self): try: the_num = self.env_dict[self.now_step_set_value_name] if isinstance(the_num, decimal.Decimal): - the_num= int(self.env_dict[self.now_step_set_value_name]) + the_num = int(self.env_dict[self.now_step_set_value_name]) if not isinstance(the_num, (int, float, decimal.Decimal)): self.stdio.warn( - "{0} is {1} and the type is {2}, not int or float or decimal ! set it to 0.".format(self.now_step_set_value_name, - self.env_dict[ - self.now_step_set_value_name], - type(self.env_dict[ - self.now_step_set_value_name]))) - the_num=0 + "{0} is {1} and the type is {2}, not int or float or decimal ! set it to 0.".format( + self.now_step_set_value_name, + self.env_dict[self.now_step_set_value_name], + type(self.env_dict[self.now_step_set_value_name]), + ) + ) + the_num = 0 range_str = self.expr return int(the_num) < int(range_str) except Exception as e: @@ -108,15 +124,16 @@ def _verify_min(self): try: the_num = self.env_dict[self.now_step_set_value_name] if isinstance(the_num, decimal.Decimal): - the_num= int(self.env_dict[self.now_step_set_value_name]) + the_num = int(self.env_dict[self.now_step_set_value_name]) if not isinstance(the_num, (int, float, decimal.Decimal)): self.stdio.warn( - "{0} is {1} and the type is {2}, not int or float or decimal ! set it to 0.".format(self.now_step_set_value_name, - self.env_dict[ - self.now_step_set_value_name], - type(self.env_dict[ - self.now_step_set_value_name]))) - the_num=0 + "{0} is {1} and the type is {2}, not int or float or decimal ! set it to 0.".format( + self.now_step_set_value_name, + self.env_dict[self.now_step_set_value_name], + type(self.env_dict[self.now_step_set_value_name]), + ) + ) + the_num = 0 range_str = self.expr return int(the_num) > int(range_str) except Exception as e: @@ -127,17 +144,20 @@ def _verify_equal(self): try: the_num = self.env_dict[self.now_step_set_value_name] if isinstance(the_num, decimal.Decimal): - the_num= int(self.env_dict[self.now_step_set_value_name]) + the_num = int(self.env_dict[self.now_step_set_value_name]) if not isinstance(the_num, (int, float, decimal.Decimal)): self.stdio.warn( - "{0} is {1} and the type is {2}, not int or float or decimal ! set it to 0.".format(self.now_step_set_value_name, - self.env_dict[ - self.now_step_set_value_name], - type(self.env_dict[ - self.now_step_set_value_name]))) - the_num=0 + "{0} is {1} and the type is {2}, not int or float or decimal ! set it to 0.".format( + self.now_step_set_value_name, + self.env_dict[self.now_step_set_value_name], + type(self.env_dict[self.now_step_set_value_name]), + ) + ) + the_num = 0 range_str = self.expr return int(self.env_dict[self.now_step_set_value_name]) == int(range_str) except Exception as e: - self.stdio.error("_verify_equal error: {0} -> {1}".format(str(self.expr), e)) + self.stdio.error( + "_verify_equal error: {0} -> {1}".format(str(self.expr), e) + ) raise VerifyFailException(e) diff --git a/handler/checker/step/__init__.py b/handler/checker/step/__init__.py index 681bdff5..70691bef 100644 --- a/handler/checker/step/__init__.py +++ b/handler/checker/step/__init__.py @@ -15,4 +15,3 @@ @file: __init__.py @desc: """ - diff --git a/handler/checker/step/data_size.py b/handler/checker/step/data_size.py index b8d0ff0c..721e9d9a 100644 --- a/handler/checker/step/data_size.py +++ b/handler/checker/step/data_size.py @@ -23,7 +23,7 @@ class DataSizeHandler: - def __init__(self,context, step, node, task_variable_dict): + def __init__(self, context, step, node, task_variable_dict): self.context = context self.stdio = context.stdio self.stdio.verbose("init DataSizeHandler") @@ -34,14 +34,20 @@ def __init__(self,context, step, node, task_variable_dict): self.task_variable_dict = task_variable_dict try: - self.ssh_helper=self.node["ssher"] + self.ssh_helper = self.node["ssher"] if self.ssh_helper is None: raise Exception("self.ssh_helper is None.") except Exception as e: self.stdio.error( - "DataSizeHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) + "DataSizeHandler ssh init fail . Please check the NODES conf Exception : {0} .".format( + e + ) + ) raise Exception( - "DataSizeHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) + "DataSizeHandler ssh init fail . Please check the NODES conf Exception : {0} .".format( + e + ) + ) # step report self.parameter = [] @@ -51,15 +57,21 @@ def execute(self): try: if "key" not in self.step: - raise StepExecuteFailException("DataSizeHandler execute parameter's 'key' is not set") + raise StepExecuteFailException( + "DataSizeHandler execute parameter's 'key' is not set" + ) self.stdio.verbose("DataSizeHandler execute: {0}".format(self.step["key"])) s = self.step["key"] value = self.task_variable_dict[s] - self.task_variable_dict[s]=Capacity(value).btyes() - self.stdio.verbose("DataSizeHandler set {0} = {1}".format(s,self.task_variable_dict[s])) + self.task_variable_dict[s] = Capacity(value).btyes() + self.stdio.verbose( + "DataSizeHandler set {0} = {1}".format(s, self.task_variable_dict[s]) + ) except Exception as e: self.stdio.error("DataSizeHandler execute Exception: {0}".format(e).strip()) - raise StepExecuteFailException("DataSizeHandler execute Exception: {0}".format(e).strip()) + raise StepExecuteFailException( + "DataSizeHandler execute Exception: {0}".format(e).strip() + ) def get_report(self): return self.report diff --git a/handler/checker/step/get_system_parameter.py b/handler/checker/step/get_system_parameter.py index af3341c1..1fb4ad99 100644 --- a/handler/checker/step/get_system_parameter.py +++ b/handler/checker/step/get_system_parameter.py @@ -23,7 +23,7 @@ class GetSystemParameterHandler: - def __init__(self,context, step, node, task_variable_dict): + def __init__(self, context, step, node, task_variable_dict): self.context = context self.stdio = context.stdio self.stdio.verbose("init GetSystemParameterHandler") @@ -34,14 +34,20 @@ def __init__(self,context, step, node, task_variable_dict): self.task_variable_dict = task_variable_dict try: - self.ssh_helper=self.node["ssher"] + self.ssh_helper = self.node["ssher"] if self.ssh_helper is None: raise Exception("self.ssh_helper is None.") except Exception as e: self.stdio.error( - "GetSystemParameterHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) + "GetSystemParameterHandler ssh init fail . Please check the NODES conf Exception : {0} .".format( + e + ) + ) raise Exception( - "GetSystemParameterHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) + "GetSystemParameterHandler ssh init fail . Please check the NODES conf Exception : {0} .".format( + e + ) + ) # step report self.parameter = [] @@ -50,11 +56,16 @@ def __init__(self,context, step, node, task_variable_dict): def get_parameter(self, parameter_name): try: parameter_name = parameter_name.replace(".", "/") - parameter_value = self.ssh_helper.ssh_exec_cmd("cat /proc/sys/" + parameter_name).strip() + parameter_value = self.ssh_helper.ssh_exec_cmd( + "cat /proc/sys/" + parameter_name + ).strip() self.ssh_helper.ssh_close() except Exception as e: self.stdio.warn( - "get {0} fail:{1} .please check, the parameter_value will be set -1".format(parameter_name, e)) + "get {0} fail:{1} .please check, the parameter_value will be set -1".format( + parameter_name, e + ) + ) parameter_value = str("-1") return parameter_value @@ -62,16 +73,23 @@ def execute(self): try: if "parameter" not in self.step: - raise StepExecuteFailException("GetSystemParameterHandler execute parameter is not set") - self.stdio.verbose("GetSystemParameterHandler execute: {0}".format(self.step["parameter"])) + raise StepExecuteFailException( + "GetSystemParameterHandler execute parameter is not set" + ) + self.stdio.verbose( + "GetSystemParameterHandler execute: {0}".format(self.step["parameter"]) + ) s = self.step["parameter"] - if '.' in s: - last_substring = s.rsplit('.', 1) + if "." in s: + last_substring = s.rsplit(".", 1) s = last_substring[len(last_substring) - 1] else: s = self.step["parameter"] # SystemParameter exist? - if self.ssh_helper.ssh_exec_cmd('find /proc/sys/ -name "{0}"'.format(s)) == "": + if ( + self.ssh_helper.ssh_exec_cmd('find /proc/sys/ -name "{0}"'.format(s)) + == "" + ): self.stdio.warn("{0} is not exist".format(self.step["parameter"])) if "result" in self.step and "set_value" in self.step["result"]: self.task_variable_dict[self.step["result"]["set_value"]] = "" @@ -81,11 +99,17 @@ def execute(self): if "result" in self.step and "set_value" in self.step["result"]: if len(parameter_value) > 0: parameter_value = parameter_value.strip() - self.stdio.verbose("GetSystemParameterHandler get value : {0}".format(parameter_value)) - self.task_variable_dict[self.step["result"]["set_value"]] = Util.convert_to_number(parameter_value) + self.stdio.verbose( + "GetSystemParameterHandler get value : {0}".format(parameter_value) + ) + self.task_variable_dict[self.step["result"]["set_value"]] = ( + Util.convert_to_number(parameter_value) + ) except Exception as e: self.stdio.error("get_parameter execute: {0}".format(e).strip()) - raise StepExecuteFailException("get_parameter execute: {0}".format(e).strip()) + raise StepExecuteFailException( + "get_parameter execute: {0}".format(e).strip() + ) def get_report(self): return self.report diff --git a/handler/checker/step/sql.py b/handler/checker/step/sql.py index b572c201..9f9406bd 100644 --- a/handler/checker/step/sql.py +++ b/handler/checker/step/sql.py @@ -23,7 +23,7 @@ class StepSQLHandler: - def __init__(self,context, step, task_variable_dict): + def __init__(self, context, step, task_variable_dict): try: self.context = context self.stdio = context.stdio @@ -32,14 +32,24 @@ def __init__(self,context, step, task_variable_dict): self.tenant_mode = None self.sys_database = None self.database = None - self.ob_connector_pool=self.context.get_variable('check_obConnector_pool',None) + self.ob_connector_pool = self.context.get_variable( + "check_obConnector_pool", None + ) if self.ob_connector_pool is not None: - self.ob_connector=self.ob_connector_pool.get_connection() + self.ob_connector = self.ob_connector_pool.get_connection() if self.ob_connector is None: raise Exception("self.ob_connector is None.") except Exception as e: - self.stdio.error("StepSQLHandler init fail. Please check the OBCLUSTER conf. Exception : {0} .".format(e)) - raise Exception("StepSQLHandler init fail. Please check the OBCLUSTER conf. Exception : {0} .".format(e)) + self.stdio.error( + "StepSQLHandler init fail. Please check the OBCLUSTER conf. Exception : {0} .".format( + e + ) + ) + raise Exception( + "StepSQLHandler init fail. Please check the OBCLUSTER conf. Exception : {0} .".format( + e + ) + ) self.task_variable_dict = task_variable_dict self.enable_dump_db = False self.trace_id = None @@ -55,23 +65,35 @@ def execute(self): try: if "sql" not in self.step: raise StepExecuteFailException("StepSQLHandler execute sql is not set") - sql = StringUtils.build_str_on_expr_by_dict(self.step["sql"], self.task_variable_dict) + sql = StringUtils.build_str_on_expr_by_dict( + self.step["sql"], self.task_variable_dict + ) self.stdio.verbose("StepSQLHandler execute: {0}".format(sql)) data = self.ob_connector.execute_sql(sql) self.stdio.verbose("execute_sql result:{0}".format(data)) - if data is None or len(data) == 0: - data="" + if data is None or len(data) == 0: + data = "" else: data = data[0][0] if data is None: data = "" - self.stdio.verbose("sql result:{0}".format(Util.convert_to_number(str(data)))) + self.stdio.verbose( + "sql result:{0}".format(Util.convert_to_number(str(data))) + ) if "result" in self.step and "set_value" in self.step["result"]: - self.stdio.verbose("sql execute update task_variable_dict: {0} = {1}".format(self.step["result"]["set_value"], Util.convert_to_number(data))) - self.task_variable_dict[self.step["result"]["set_value"]] = Util.convert_to_number(data) + self.stdio.verbose( + "sql execute update task_variable_dict: {0} = {1}".format( + self.step["result"]["set_value"], Util.convert_to_number(data) + ) + ) + self.task_variable_dict[self.step["result"]["set_value"]] = ( + Util.convert_to_number(data) + ) except Exception as e: self.stdio.error("StepSQLHandler execute Exception: {0}".format(e)) - raise StepExecuteFailException("StepSQLHandler execute Exception: {0}".format(e)) + raise StepExecuteFailException( + "StepSQLHandler execute Exception: {0}".format(e) + ) finally: self.ob_connector_pool.release_connection(self.ob_connector) diff --git a/handler/checker/step/ssh.py b/handler/checker/step/ssh.py index 282477e2..baad64e7 100644 --- a/handler/checker/step/ssh.py +++ b/handler/checker/step/ssh.py @@ -23,7 +23,7 @@ class SshHandler: - def __init__(self,context, step, node, task_variable_dict): + def __init__(self, context, step, node, task_variable_dict): self.context = context self.stdio = context.stdio self.ssh_report_value = None @@ -31,14 +31,20 @@ def __init__(self,context, step, node, task_variable_dict): self.step = step self.node = node try: - self.ssh_helper=self.node["ssher"] + self.ssh_helper = self.node["ssher"] if self.ssh_helper is None: raise Exception("self.ssh_helper is None.") except Exception as e: self.stdio.error( - "SshHandler init fail. Please check the NODES conf. node: {0}. Exception : {1} .".format(node, e)) + "SshHandler init fail. Please check the NODES conf. node: {0}. Exception : {1} .".format( + node, e + ) + ) raise Exception( - "SshHandler init fail. Please check the NODES conf node: {0} Exception : {1} .".format(node, e)) + "SshHandler init fail. Please check the NODES conf node: {0} Exception : {1} .".format( + node, e + ) + ) self.task_variable_dict = task_variable_dict self.parameter = [] self.report = TaskReport @@ -47,24 +53,38 @@ def execute(self): try: if "ssh" not in self.step: raise StepExecuteFailException("SshHandler execute ssh is not set") - ssh_cmd = StringUtils.build_str_on_expr_by_dict(self.step["ssh"], self.task_variable_dict) + ssh_cmd = StringUtils.build_str_on_expr_by_dict( + self.step["ssh"], self.task_variable_dict + ) self.stdio.verbose("step SshHandler execute :{0} ".format(ssh_cmd)) ssh_report_value = self.ssh_helper.ssh_exec_cmd(ssh_cmd) if ssh_report_value is None: ssh_report_value = "" if len(ssh_report_value) > 0: ssh_report_value = ssh_report_value.strip() - self.stdio.verbose("ssh result:{0}".format(Util.convert_to_number(ssh_report_value))) + self.stdio.verbose( + "ssh result:{0}".format(Util.convert_to_number(ssh_report_value)) + ) if "result" in self.step and "set_value" in self.step["result"]: - self.stdio.verbose("ssh result set {0}".format(self.step["result"]["set_value"], - Util.convert_to_number(ssh_report_value))) - self.task_variable_dict[self.step["result"]["set_value"]] = Util.convert_to_number(ssh_report_value) + self.stdio.verbose( + "ssh result set {0}".format( + self.step["result"]["set_value"], + Util.convert_to_number(ssh_report_value), + ) + ) + self.task_variable_dict[self.step["result"]["set_value"]] = ( + Util.convert_to_number(ssh_report_value) + ) except Exception as e: self.stdio.error("ssh execute Exception:{0}".format(e).strip()) - raise StepExecuteFailException("ssh execute Exception:{0}".format(e).strip()) + raise StepExecuteFailException( + "ssh execute Exception:{0}".format(e).strip() + ) finally: self.ssh_helper.ssh_close() - self.stdio.verbose("step SshHandler ssh_report_value:{0}".format(ssh_report_value)) + self.stdio.verbose( + "step SshHandler ssh_report_value:{0}".format(ssh_report_value) + ) def update_step_variable_dict(self): return self.task_variable_dict diff --git a/handler/checker/step/stepbase.py b/handler/checker/step/stepbase.py index c3afe63c..60a18687 100644 --- a/handler/checker/step/stepbase.py +++ b/handler/checker/step/stepbase.py @@ -16,8 +16,13 @@ @desc: """ -from handler.checker.check_exception import StepResultFailException, StepExecuteFailException, \ - ResultFalseException, ResultFailException, StepResultFalseException +from handler.checker.check_exception import ( + StepResultFailException, + StepExecuteFailException, + ResultFalseException, + ResultFailException, + StepResultFalseException, +) from handler.checker.step.data_size import DataSizeHandler from handler.checker.step.get_system_parameter import GetSystemParameterHandler from handler.checker.result.result import CheckResult @@ -45,48 +50,87 @@ def execute(self, report): self.task_variable_dict["remote_ip"] = self.node["ip"] elif "ssh_type" in self.node and self.node["ssh_type"] == "docker": self.stdio.verbose("execute ssh_type is docker") - self.task_variable_dict["remote_ip"] = \ - docker.from_env().containers.get(self.node["container_name"]).attrs['NetworkSettings']['Networks'][ - 'bridge']["IPAddress"] + self.task_variable_dict["remote_ip"] = ( + docker.from_env() + .containers.get(self.node["container_name"]) + .attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] + ) for node in self.node: self.task_variable_dict["remote_{0}".format(node)] = self.node[node] if "type" not in self.step: raise StepExecuteFailException("Missing field :type") if self.step["type"] == "get_system_parameter": - handler = GetSystemParameterHandler(self.context, self.step, self.node, self.task_variable_dict) + handler = GetSystemParameterHandler( + self.context, self.step, self.node, self.task_variable_dict + ) elif self.step["type"] == "ssh": - handler = SshHandler(self.context, self.step, self.node, self.task_variable_dict) + handler = SshHandler( + self.context, self.step, self.node, self.task_variable_dict + ) elif self.step["type"] == "sql": - handler = StepSQLHandler(self.context, self.step, task_variable_dict=self.task_variable_dict) + handler = StepSQLHandler( + self.context, self.step, task_variable_dict=self.task_variable_dict + ) elif self.step["type"] == "data_size": - handler = DataSizeHandler(self.context, self.step, self.cluster, self.task_variable_dict) + handler = DataSizeHandler( + self.context, self.step, self.cluster, self.task_variable_dict + ) else: - raise StepExecuteFailException("the type not support: {0}".format(self.step["type"])) + raise StepExecuteFailException( + "the type not support: {0}".format(self.step["type"]) + ) self.stdio.verbose("task execute and result") handler.execute() except Exception as e: self.stdio.error("StepBase handler.execute fail {0}".format(e)) if self.step["type"] == "sql": - report.add("[cluster:{0}] {1}".format(self.cluster.get("ob_cluster_name") or self.cluster.get( - "obproxy_cluster_name") or no_cluster_name_msg, e), "fail") + report.add( + "[cluster:{0}] {1}".format( + self.cluster.get("ob_cluster_name") + or self.cluster.get("obproxy_cluster_name") + or no_cluster_name_msg, + e, + ), + "fail", + ) else: - report.add("[{0}:{1}] {2}".format(self.node.get("ssh_type") or "", - self.node.get("container_name") or self.task_variable_dict.get( - "remote_ip") or "", e), "fail") - raise StepExecuteFailException("StepBase handler.execute fail {0}".format(e)) + report.add( + "[{0}:{1}] {2}".format( + self.node.get("ssh_type") or "", + self.node.get("container_name") + or self.task_variable_dict.get("remote_ip") + or "", + e, + ), + "fail", + ) + raise StepExecuteFailException( + "StepBase handler.execute fail {0}".format(e) + ) try: self.task_variable_dict = handler.update_step_variable_dict() - self.stdio.verbose("self.task_variable_dict: {0}".format(self.task_variable_dict)) - if self.step["type"] == "get_system_parameter" and "result" in self.step and "set_value" in self.step[ - "result"] and self.task_variable_dict[self.step["result"]["set_value"]] == "": + self.stdio.verbose( + "self.task_variable_dict: {0}".format(self.task_variable_dict) + ) + if ( + self.step["type"] == "get_system_parameter" + and "result" in self.step + and "set_value" in self.step["result"] + and self.task_variable_dict[self.step["result"]["set_value"]] == "" + ): return if "result" in self.step: self.stdio.verbose("result execute ") - result = CheckResult(self.context, self.step["result"], self.task_variable_dict) + result = CheckResult( + self.context, self.step["result"], self.task_variable_dict + ) result.execute() - if "report_type" in self.step["result"] and self.step["result"]["report_type"] == "execution": + if ( + "report_type" in self.step["result"] + and self.step["result"]["report_type"] == "execution" + ): self.stdio.verbose("report_type stop this step") return @@ -96,37 +140,77 @@ def execute(self, report): # When result.type is execution, if this step is executed successfully, subsequent steps will not be # executed. - self.stdio.warn("step_base ResultFalseException:{0}".format(resultException)) + self.stdio.warn( + "step_base ResultFalseException:{0}".format(resultException) + ) level = "critical" - self.stdio.verbose("step_base ResultFalseException self.step.result:{0}".format(self.step["result"])) + self.stdio.verbose( + "step_base ResultFalseException self.step.result:{0}".format( + self.step["result"] + ) + ) if "result" in self.step: if "report_type" in self.step["result"]: - self.stdio.verbose("report_type use is {0}".format(self.step["result"]["report_type"])) + self.stdio.verbose( + "report_type use is {0}".format( + self.step["result"]["report_type"] + ) + ) level = self.step["result"]["report_type"] if level == "execution": level = "warning" if self.step["type"] == "sql": - report.add("[cluster:{0}] {1}".format(self.cluster.get("ob_cluster_name") or self.cluster.get( - "obproxy_cluster_name") or no_cluster_name_msg, resultException), level) + report.add( + "[cluster:{0}] {1}".format( + self.cluster.get("ob_cluster_name") + or self.cluster.get("obproxy_cluster_name") + or no_cluster_name_msg, + resultException, + ), + level, + ) else: - report.add("[{0}:{1}] {2}".format(self.node.get("ssh_type") or "", - self.node.get("container_name") or self.task_variable_dict.get( - "remote_ip") or "", resultException), level) + report.add( + "[{0}:{1}] {2}".format( + self.node.get("ssh_type") or "", + self.node.get("container_name") + or self.task_variable_dict.get("remote_ip") + or "", + resultException, + ), + level, + ) if level == "critical": raise StepResultFailException(resultException) raise StepResultFalseException(resultException) except ResultFailException as resultFailException: # 验证失败，属于fail类型，一般是verify阶段出现异常,需要马上修正 - self.stdio.error("step_base ResultFailException:{0}".format(resultFailException)) + self.stdio.error( + "step_base ResultFailException:{0}".format(resultFailException) + ) if self.step["type"] == "sql": - report.add("[cluster:{0}] {1}".format(self.cluster.get("ob_cluster_name") or self.cluster.get( - "obproxy_cluster_name") or no_cluster_name_msg, resultFailException), "fail") + report.add( + "[cluster:{0}] {1}".format( + self.cluster.get("ob_cluster_name") + or self.cluster.get("obproxy_cluster_name") + or no_cluster_name_msg, + resultFailException, + ), + "fail", + ) else: - report.add("[{0}:{1}] {2}".format(self.node.get("ssh_type") or "", - self.node.get("container_name") or self.task_variable_dict.get( - "remote_ip") or "", resultFailException), "fail") + report.add( + "[{0}:{1}] {2}".format( + self.node.get("ssh_type") or "", + self.node.get("container_name") + or self.task_variable_dict.get("remote_ip") + or "", + resultFailException, + ), + "fail", + ) raise StepResultFailException(resultFailException) except Exception as e: diff --git a/handler/gather/__init__.py b/handler/gather/__init__.py index 29f4a072..d3a64b03 100644 --- a/handler/gather/__init__.py +++ b/handler/gather/__init__.py @@ -14,4 +14,4 @@ @time: 2023/9/20 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/gather/gather_ash_report.py b/handler/gather/gather_ash_report.py index c445f263..386bd746 100644 --- a/handler/gather/gather_ash_report.py +++ b/handler/gather/gather_ash_report.py @@ -28,7 +28,7 @@ class GatherAshReportHandler(SafeStdio): - def __init__(self, context, gather_pack_dir='./'): + def __init__(self, context, gather_pack_dir="./"): super().__init__() self.result_summary_file_name = None self.report_type = None @@ -58,7 +58,7 @@ def __init__(self, context, gather_pack_dir='./'): password=self.cluster.get("tenant_sys").get("password"), stdio=self.stdio, timeout=10000, - database="oceanbase" + database="oceanbase", ) except Exception as e: self.stdio.error("Failed to connect to database: {0}".format(e)) @@ -66,37 +66,64 @@ def __init__(self, context, gather_pack_dir='./'): def handle(self): if not self.version_check(): - self.stdio.error('version check failed') + self.stdio.error("version check failed") return False if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False self.__init_report_path() self.execute() self.__print_result() + def version_check(self): observer_version = "" try: observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) except Exception as e: if len(self.observer_nodes) > 0: - ssher=SshHelper(self.observer_nodes[0]["ip"], self.observer_nodes[0]["ssh_port"], self.observer_nodes[0]["ssh_username"], self.observer_nodes[0]["ssh_password"]) - observer_version = get_observer_version(True, ssher, - self.observer_nodes[0]["home_path"],self.stdio) + ssher = SshHelper( + self.observer_nodes[0]["ip"], + self.observer_nodes[0]["ssh_port"], + self.observer_nodes[0]["ssh_username"], + self.observer_nodes[0]["ssh_password"], + ) + observer_version = get_observer_version( + True, ssher, self.observer_nodes[0]["home_path"], self.stdio + ) else: - self.stdio.warn("RCAHandler Failed to get observer version:{0}".format(e)) - self.stdio.verbose("RCAHandler.init get observer version: {0}".format(observer_version)) + self.stdio.warn( + "RCAHandler Failed to get observer version:{0}".format(e) + ) + self.stdio.verbose( + "RCAHandler.init get observer version: {0}".format(observer_version) + ) - if not (observer_version == "4.0.0.0" or StringUtils.compare_versions_greater(observer_version, "4.0.0.0")): - self.stdio.error("observer version: {0}, must greater than 4.0.0.0".format(observer_version)) + if not ( + observer_version == "4.0.0.0" + or StringUtils.compare_versions_greater(observer_version, "4.0.0.0") + ): + self.stdio.error( + "observer version: {0}, must greater than 4.0.0.0".format( + observer_version + ) + ) return False return True def execute(self): try: - ash_report_arg = (self.from_time_str, self.to_time_str, self.sql_id, self.trace_id, self.wait_class, self.report_type) + ash_report_arg = ( + self.from_time_str, + self.to_time_str, + self.sql_id, + self.trace_id, + self.wait_class, + self.report_type, + ) self.stdio.verbose("ash report arg: {0}".format(ash_report_arg)) - ash_report_data = self.obconn.callproc("DBMS_WORKLOAD_REPOSITORY.ASH_REPORT", args=ash_report_arg) + ash_report_data = self.obconn.callproc( + "DBMS_WORKLOAD_REPOSITORY.ASH_REPORT", args=ash_report_arg + ) if not ash_report_data or len(ash_report_data) == 0: self.stdio.error("ash report data is empty") raise OBDIAGException("ash report data is empty") @@ -108,14 +135,24 @@ def execute(self): # save ash_report_data self.ash_report_file_name = "ash_report_{0}.txt".format( - TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) - self.ash_report_file_name=os.path.join(self.report_path, self.ash_report_file_name) + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ) + self.ash_report_file_name = os.path.join( + self.report_path, self.ash_report_file_name + ) - with open(self.ash_report_file_name, 'w+') as f: + with open(self.ash_report_file_name, "w+") as f: f.write(ash_report) - self.stdio.print("save ash report file name: "+ Fore.YELLOW +"{0}".format(self.ash_report_file_name)+Style.RESET_ALL) - self.result_summary_file_name = os.path.join(self.report_path, "result_summary.txt") - with open(self.result_summary_file_name, 'w+') as f: + self.stdio.print( + "save ash report file name: " + + Fore.YELLOW + + "{0}".format(self.ash_report_file_name) + + Style.RESET_ALL + ) + self.result_summary_file_name = os.path.join( + self.report_path, "result_summary.txt" + ) + with open(self.result_summary_file_name, "w+") as f: f.write(self.ash_report_file_name) except Exception as e: @@ -123,23 +160,27 @@ def execute(self): def __init_report_path(self): try: - self.report_path = os.path.join(self.gather_pack_dir, "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp), self.stdio)) + self.report_path = os.path.join( + self.gather_pack_dir, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp), + self.stdio, + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(self.report_path)) DirectoryUtil.mkdir(path=self.report_path, stdio=self.stdio) except Exception as e: self.stdio.error("init_report_path failed, error:{0}".format(e)) - - def init_option(self): options = self.context.options - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - trace_id_option = Util.get_option(options, 'trace_id') - sql_id_option = Util.get_option(options, 'sql_id') - report_type_option = Util.get_option(options, 'report_type') - wait_class_option = Util.get_option(options, 'wait_class') - store_dir_option = Util.get_option(options, 'store_dir' ) + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + trace_id_option = Util.get_option(options, "trace_id") + sql_id_option = Util.get_option(options, "sql_id") + report_type_option = Util.get_option(options, "report_type") + wait_class_option = Util.get_option(options, "wait_class") + store_dir_option = Util.get_option(options, "store_dir") since_option = "30m" if from_option is not None and to_option is not None: @@ -150,28 +191,55 @@ def init_option(self): self.to_time_str = to_option except OBDIAGFormatException: self.stdio.exception( - 'Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}'.format( - from_option, to_option)) + "Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}".format( + from_option, to_option + ) + ) return False if to_timestamp <= from_timestamp: - self.stdio.exception('Error: from datetime is larger than to datetime, please check.') + self.stdio.exception( + "Error: from datetime is larger than to datetime, please check." + ) return False - elif (from_option is None or to_option is None): + elif from_option is None or to_option is None: now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=0)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta( - seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=0)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 30 minutes') + self.stdio.warn( + "No time option provided, default processing is based on the last 30 minutes" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) if store_dir_option: if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format( - os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) if sql_id_option: @@ -186,7 +254,9 @@ def init_option(self): if report_type_option: self.report_type = report_type_option.strip() if report_type_option.upper() != "TEXT": - self.stdio.error("Invalid argument for report type, Now just support TEXT") + self.stdio.error( + "Invalid argument for report type, Now just support TEXT" + ) return False else: self.report_type = None @@ -198,11 +268,26 @@ def init_option(self): self.gather_pack_dir = store_dir_option else: self.gather_pack_dir = "./" - self.stdio.print("from_time: {0}, to_time: {1}, sql_id: {2}, trace_id: {3}, report_type: {4}, wait_class: {5}, store_dir: {6}".format(self.from_time_str, self.to_time_str, self.sql_id, self.trace_id, self.report_type, self.wait_class,self.gather_pack_dir)) + self.stdio.print( + "from_time: {0}, to_time: {1}, sql_id: {2}, trace_id: {3}, report_type: {4}, wait_class: {5}, store_dir: {6}".format( + self.from_time_str, + self.to_time_str, + self.sql_id, + self.trace_id, + self.report_type, + self.wait_class, + self.gather_pack_dir, + ) + ) return True def __print_result(self): - self.stdio.print(Fore.YELLOW + "\nGather ash_report results stored in this directory: {0}".format( - self.report_path) + Style.RESET_ALL) - self.stdio.print("") \ No newline at end of file + self.stdio.print( + Fore.YELLOW + + "\nGather ash_report results stored in this directory: {0}".format( + self.report_path + ) + + Style.RESET_ALL + ) + self.stdio.print("") diff --git a/handler/gather/gather_awr.py b/handler/gather/gather_awr.py index 1e64111a..fc8ddc5a 100644 --- a/handler/gather/gather_awr.py +++ b/handler/gather/gather_awr.py @@ -30,14 +30,14 @@ class GatherAwrHandler(object): - def __init__(self, context, gather_pack_dir='./'): + def __init__(self, context, gather_pack_dir="./"): self.context = context self.stdio = context.stdio self.gather_pack_dir = gather_pack_dir self.cluster_name = None self.cluster_id = None - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() @@ -51,15 +51,18 @@ def init_config(self): def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False # example of the format of pack dir for this command: (gather_pack_dir)/gather_pack_20190610123344 - pack_dir_this_command = os.path.join(self.gather_pack_dir, - "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time( - self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.gather_pack_dir, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) gather_tuples = [] @@ -73,12 +76,12 @@ def handle_awr_from_ocp(ocp_url, cluster_name): """ st = time.time() # step 1: generate awr report - self.stdio.start_loading('generate awr report') + self.stdio.start_loading("generate awr report") report_name = self.__generate_awr_report() if report_name: - self.stdio.stop_loading('generate awr report sucess') + self.stdio.stop_loading("generate awr report sucess") else: - self.stdio.stop_loading('generate awr failed') + self.stdio.stop_loading("generate awr failed") return # step 2: get awr report_id @@ -89,22 +92,43 @@ def handle_awr_from_ocp(ocp_url, cluster_name): if resp["skip"]: return if resp["error"]: - gather_tuples.append((ocp_url, True, - resp["error_msg"], 0, int(time.time() - st), - "Error:{0}".format(resp["error_msg"]), "")) + gather_tuples.append( + ( + ocp_url, + True, + resp["error_msg"], + 0, + int(time.time() - st), + "Error:{0}".format(resp["error_msg"]), + "", + ) + ) return gather_pack_path_dict[(cluster_name, ocp_url)] = resp["gather_pack_path"] - gather_tuples.append((cluster_name, False, "", - os.path.getsize(resp["gather_pack_path"]), - int(time.time() - st), resp["gather_pack_path"])) + gather_tuples.append( + ( + cluster_name, + False, + "", + os.path.getsize(resp["gather_pack_path"]), + int(time.time() - st), + resp["gather_pack_path"], + ) + ) - ocp_threads = [threading.Thread(None, handle_awr_from_ocp(self.ocp_url, self.cluster_name), args=())] + ocp_threads = [ + threading.Thread( + None, handle_awr_from_ocp(self.ocp_url, self.cluster_name), args=() + ) + ] list(map(lambda x: x.start(), ocp_threads)) list(map(lambda x: x.join(), ocp_threads)) summary_tuples = self.__get_overall_summary(gather_tuples) self.stdio.print(summary_tuples) # 将汇总结果持久化记录到文件中 - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) return gather_tuples, gather_pack_path_dict @@ -121,15 +145,20 @@ def __download_report(self, store_path, name, report_id): } self.stdio.verbose( - "Sending Status Request to cluster {0} ...".format(self.cluster_name)) + "Sending Status Request to cluster {0} ...".format(self.cluster_name) + ) - path = ocp_api.cluster + "/%s/performance/workload/reports/%s" % (self.cluster_id, report_id) + path = ocp_api.cluster + "/%s/performance/workload/reports/%s" % ( + self.cluster_id, + report_id, + ) save_path = os.path.join(store_path, name + ".html") - self.stdio.start_loading('download AWR report') + self.stdio.start_loading("download AWR report") pack_path = self.download(self.ocp_url + path, save_path, self.auth) - self.stdio.stop_loading('download AWR report') + self.stdio.stop_loading("download AWR report") self.stdio.verbose( - "cluster {0} response. analysing...".format(self.cluster_name)) + "cluster {0} response. analysing...".format(self.cluster_name) + ) resp["gather_pack_path"] = pack_path if resp["error"]: @@ -149,7 +178,10 @@ def __generate_awr_report(self): """ snapshot_list = self.__get_snapshot_list() if len(snapshot_list) <= 1: - self.stdio.warn("AWR report at least need 2 snapshot, cluster now only have %s, please adjusted to --from/to or --since", len(snapshot_list)) + self.stdio.warn( + "AWR report at least need 2 snapshot, cluster now only have %s, please adjusted to --from/to or --since", + len(snapshot_list), + ) return None else: start_sid, start_time = snapshot_list[0] @@ -157,15 +189,23 @@ def __generate_awr_report(self): path = ocp_api.cluster + "/%s/performance/workload/reports" % self.cluster_id - start_time = datetime.datetime.strptime(TimeUtils.trans_datetime_utc_to_local(start_time.split(".")[0]), - "%Y-%m-%d %H:%M:%S") - end_time = datetime.datetime.strptime(TimeUtils.trans_datetime_utc_to_local(end_time.split(".")[0]), - "%Y-%m-%d %H:%M:%S") + start_time = datetime.datetime.strptime( + TimeUtils.trans_datetime_utc_to_local(start_time.split(".")[0]), + "%Y-%m-%d %H:%M:%S", + ) + end_time = datetime.datetime.strptime( + TimeUtils.trans_datetime_utc_to_local(end_time.split(".")[0]), + "%Y-%m-%d %H:%M:%S", + ) params = { - "name": "OBAWR_obcluster_%s_%s_%s" % ( - self.cluster_name, start_time.strftime("%Y%m%d%H%M%S"), end_time.strftime("%Y%m%d%H%M%S")), + "name": "OBAWR_obcluster_%s_%s_%s" + % ( + self.cluster_name, + start_time.strftime("%Y%m%d%H%M%S"), + end_time.strftime("%Y%m%d%H%M%S"), + ), "startSnapshotId": start_sid, - "endSnapshotId": end_sid + "endSnapshotId": end_sid, } response = requests.post(self.ocp_url + path, auth=self.auth, data=params) @@ -187,7 +227,9 @@ def __get_snapshot_list(self): response = requests.get(self.ocp_url + path, auth=self.auth) # Validate the response status code response.raise_for_status() - from_datetime_timestamp = TimeUtils.datetime_to_timestamp(self.from_time_str) + from_datetime_timestamp = TimeUtils.datetime_to_timestamp( + self.from_time_str + ) to_datetime_timestamp = TimeUtils.datetime_to_timestamp(self.to_time_str) # If the user-specified time interval is less than one hour, @@ -195,21 +237,42 @@ def __get_snapshot_list(self): if from_datetime_timestamp + 3 * 3600000000 >= to_datetime_timestamp: # Round the start time to the nearest hour from_datetime_timestamp = TimeUtils.datetime_to_timestamp( - TimeUtils.get_time_rounding(dt=TimeUtils.parse_time_str(self.from_time_str), step=0, rounding_level="hour")) + TimeUtils.get_time_rounding( + dt=TimeUtils.parse_time_str(self.from_time_str), + step=0, + rounding_level="hour", + ) + ) # Set the end time to one hour and three minutes after the rounded start time # (the three-minute offset ensures snapshots can be obtained) - to_datetime_timestamp = from_datetime_timestamp + 3 * 3600000000 + 3 * 60000000 + to_datetime_timestamp = ( + from_datetime_timestamp + 3 * 3600000000 + 3 * 60000000 + ) - if "data" not in response.json() or "contents" not in response.json()["data"]: - raise ValueError("Invalid response structure. Missing 'data' or 'contents' key.") + if ( + "data" not in response.json() + or "contents" not in response.json()["data"] + ): + raise ValueError( + "Invalid response structure. Missing 'data' or 'contents' key." + ) for info in response.json()["data"]["contents"]: try: snapshot_time = TimeUtils.datetime_to_timestamp( - TimeUtils.trans_datetime_utc_to_local(str(info["snapshotTime"]).split(".")[0])) - if from_datetime_timestamp <= snapshot_time <= to_datetime_timestamp: - snapshot_id_list.append((info["snapshotId"], info["snapshotTime"])) + TimeUtils.trans_datetime_utc_to_local( + str(info["snapshotTime"]).split(".")[0] + ) + ) + if ( + from_datetime_timestamp + <= snapshot_time + <= to_datetime_timestamp + ): + snapshot_id_list.append( + (info["snapshotId"], info["snapshotTime"]) + ) except KeyError: self.stdio.error(f"Malformed snapshot data: {info}") continue @@ -236,17 +299,17 @@ def __get_awr_report_id(self, report_name): def init_option(self): options = self.context.options - store_dir_option = Util.get_option(options, 'store_dir') - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - since_option = Util.get_option(options, 'since') - cluster_name_option = Util.get_option(options, 'cluster_name') + store_dir_option = Util.get_option(options, "store_dir") + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + since_option = Util.get_option(options, "since") + cluster_name_option = Util.get_option(options, "cluster_name") if cluster_name_option: self.cluster_name = cluster_name_option else: self.stdio.error("--cluster_name option need provided") return False - cluster_id_option = Util.get_option(options, 'cluster_id') + cluster_id_option = Util.get_option(options, "cluster_id") if cluster_id_option: self.cluster_id = cluster_id_option else: @@ -259,13 +322,21 @@ def init_option(self): from_timestamp = TimeUtils.datetime_to_timestamp(from_option) to_timestamp = TimeUtils.datetime_to_timestamp(to_option) except OBDIAGFormatException: - self.stdio.error("Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. " \ - "from_datetime={0}, to_datetime={1}".format(from_option, to_option)) + self.stdio.error( + "Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. " + "from_datetime={0}, to_datetime={1}".format(from_option, to_option) + ) return False if to_timestamp <= from_timestamp: - self.stdio.error("Error: from datetime is larger than to datetime, please check.") + self.stdio.error( + "Error: from datetime is larger than to datetime, please check." + ) return False - self.stdio.print('gather log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.stdio.print( + "gather log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) elif (from_option is None or to_option is None) and since_option is not None: # the format of since must be 'n' try: @@ -274,21 +345,45 @@ def init_option(self): self.stdio.error("Error: the format of since must be 'n'") return False now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) if since_to_seconds < 3 * 3600: - self.stdio.warn('The --since requires a value greater than 3h. your provided is less than 3h, adjusted to 3h.') + self.stdio.warn( + "The --since requires a value greater than 3h. your provided is less than 3h, adjusted to 3h." + ) since_to_seconds = 3 * 3600 - self.from_time_str = (now_time - datetime.timedelta(seconds=since_to_seconds)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.from_time_str = ( + now_time - datetime.timedelta(seconds=since_to_seconds) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 3 h') + self.stdio.warn( + "No time option provided, default processing is based on the last 3 h" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(seconds=3 * 3600)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time - datetime.timedelta(seconds=3 * 3600) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) return True @@ -309,7 +404,15 @@ def __get_overall_summary(node_summary_tuple): consume_time = tup[4] pack_path = tup[5] format_file_size = FileUtil.size_format(num=file_size, output_str=True) - summary_tab.append((cluster, "Error" if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) - return "\nGather AWR Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + summary_tab.append( + ( + cluster, + "Error" if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather AWR Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 6003fc74..9a7f8d4d 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -22,7 +22,17 @@ from handler.base_shell_handler import BaseShellHandler from common.obdiag_exception import OBDIAGFormatException from common.constant import const -from common.command import get_file_size, download_file, is_empty_dir, rm_rf_file, get_logfile_name_list, mkdir, delete_empty_file, zip_encrypt_dir, zip_dir +from common.command import ( + get_file_size, + download_file, + is_empty_dir, + rm_rf_file, + get_logfile_name_list, + mkdir, + delete_empty_file, + zip_encrypt_dir, + zip_dir, +) from common.ssh import SshHelper from common.command import SshClient, LocalClient from common.tool import TimeUtils @@ -33,7 +43,7 @@ class GatherLogHandler(BaseShellHandler): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): super(GatherLogHandler, self).__init__() self.pack_dir_this_command = "" self.context = context @@ -49,13 +59,13 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.zip_encrypt = False self.is_scene = is_scene self.config_path = const.DEFAULT_CONFIG_PATH - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.nodes, self.stdio) if new_nodes: self.nodes = new_nodes @@ -64,33 +74,33 @@ def init_config(self): self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] return True def init_option(self): options = self.context.options - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - since_option = Util.get_option(options, 'since') - store_dir_option = Util.get_option(options, 'store_dir') - grep_option = Util.get_option(options, 'grep') - scope_option = Util.get_option(options, 'scope') - encrypt_option = Util.get_option(options, 'encrypt') - if self.context.get_variable("gather_from",None) : - from_option=self.context.get_variable("gather_from") - if self.context.get_variable("gather_to",None) : - to_option=self.context.get_variable("gather_to") - if self.context.get_variable("gather_since",None) : - since_option=self.context.get_variable("gather_since") - if self.context.get_variable("store_dir",None) : - store_dir_option=self.context.get_variable("store_dir") - if self.context.get_variable("gather_scope",None) : - scope_option=self.context.get_variable("gather_scope") - if self.context.get_variable("gather_grep",None) : - grep_option=self.context.get_variable("gather_grep") + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + since_option = Util.get_option(options, "since") + store_dir_option = Util.get_option(options, "store_dir") + grep_option = Util.get_option(options, "grep") + scope_option = Util.get_option(options, "scope") + encrypt_option = Util.get_option(options, "encrypt") + if self.context.get_variable("gather_from", None): + from_option = self.context.get_variable("gather_from") + if self.context.get_variable("gather_to", None): + to_option = self.context.get_variable("gather_to") + if self.context.get_variable("gather_since", None): + since_option = self.context.get_variable("gather_since") + if self.context.get_variable("store_dir", None): + store_dir_option = self.context.get_variable("store_dir") + if self.context.get_variable("gather_scope", None): + scope_option = self.context.get_variable("gather_scope") + if self.context.get_variable("gather_grep", None): + grep_option = self.context.get_variable("gather_grep") if from_option is not None and to_option is not None: try: from_timestamp = TimeUtils.parse_time_str(from_option) @@ -98,28 +108,64 @@ def init_option(self): self.from_time_str = from_option self.to_time_str = to_option except OBDIAGFormatException: - self.stdio.exception('Error: Datetime is invalid. Must be in format "yyyy-mm-dd hh:mm:ss". from_datetime={0}, to_datetime={1}'.format(from_option, to_option)) + self.stdio.exception( + 'Error: Datetime is invalid. Must be in format "yyyy-mm-dd hh:mm:ss". from_datetime={0}, to_datetime={1}'.format( + from_option, to_option + ) + ) return False if to_timestamp <= from_timestamp: - self.stdio.exception('Error: from datetime is larger than to datetime, please check.') + self.stdio.exception( + "Error: from datetime is larger than to datetime, please check." + ) return False elif (from_option is None or to_option is None) and since_option is not None: now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 30 minutes') + self.stdio.warn( + "No time option provided, default processing is based on the last 30 minutes" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) if since_option: - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") else: - self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather log from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) - if store_dir_option is not None and store_dir_option != './': + self.from_time_str = ( + now_time - datetime.timedelta(minutes=30) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather log from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) + if store_dir_option is not None and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) if scope_option: @@ -132,17 +178,22 @@ def init_option(self): def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False if self.is_scene: pack_dir_this_command = self.gather_pack_dir else: - pack_dir_this_command = os.path.join(self.gather_pack_dir, "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.gather_pack_dir, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) - self.stdio.verbose('Use {0} as pack dir.'.format(pack_dir_this_command)) + self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] def handle_from_node(node): @@ -151,11 +202,17 @@ def handle_from_node(node): file_size = "" if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], - file_size, - resp["zip_password"], - int(time.time() - st), - resp["gather_pack_path"])) + gather_tuples.append( + ( + node.get("ip"), + False, + resp["error"], + file_size, + resp["zip_password"], + int(time.time() - st), + resp["gather_pack_path"], + ) + ) if self.is_ssh: for node in self.nodes: @@ -169,19 +226,20 @@ def handle_from_node(node): summary_tuples = self.__get_overall_summary(gather_tuples, self.zip_encrypt) self.stdio.print(summary_tuples) - self.pack_dir_this_command=pack_dir_this_command + self.pack_dir_this_command = pack_dir_this_command # Persist the summary results to a file - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) - last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) + last_info = ( + "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + os.path.join(pack_dir_this_command, "result_summary.txt") + ) + ) return True def __handle_from_node(self, pack_dir_this_command, node): - resp = { - "skip": False, - "error": "", - "zip_password": "", - "gather_pack_path": "" - } + resp = {"skip": False, "error": "", "zip_password": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") @@ -189,27 +247,47 @@ def __handle_from_node(self, pack_dir_this_command, node): remote_private_key = node.get("ssh_key_file") remote_home_path = node.get("home_path") ssh_failed = False - self.stdio.verbose('Sending Collect Shell Command to node {0} ...'.format(remote_ip)) + self.stdio.verbose( + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) if "ssh_type" in node and node["ssh_type"] == "docker": - local_store_dir = "{0}/docker_{1}".format(pack_dir_this_command, node["container_name"]) + local_store_dir = "{0}/docker_{1}".format( + pack_dir_this_command, node["container_name"] + ) else: local_store_dir = "{0}/{1}".format(pack_dir_this_command, remote_ip) try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.exception('ssh {0}@{1}: failed, Please check the {2}'.format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.exception( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) - if not ssh_failed: + if not ssh_failed: # transform timestamp(in us) to yyyymmddhhmmss (filename_time style) - from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) - to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) - gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp) + from_datetime_timestamp = TimeUtils.timestamp_to_filename_time( + TimeUtils.datetime_to_timestamp(self.from_time_str) + ) + to_datetime_timestamp = TimeUtils.timestamp_to_filename_time( + TimeUtils.datetime_to_timestamp(self.to_time_str) + ) + gather_dir_name = "ob_log_{0}_{1}_{2}".format( + ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp + ) gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) @@ -217,10 +295,20 @@ def __handle_from_node(self, pack_dir_this_command, node): if resp["skip"]: return resp if self.context.get_variable("gather_mode") == "trace_id_log": - self.__grep_log_until_empty(ssh_helper=ssh, home_path=remote_home_path, log_list=log_list, gather_path=gather_dir_full_path) + self.__grep_log_until_empty( + ssh_helper=ssh, + home_path=remote_home_path, + log_list=log_list, + gather_path=gather_dir_full_path, + ) else: for log_name in log_list: - self.__pharse_log(ssh_helper=ssh, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) + self.__pharse_log( + ssh_helper=ssh, + log_name=log_name, + home_path=remote_home_path, + gather_path=gather_dir_full_path, + ) delete_empty_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) is_empty = is_empty_dir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) @@ -229,36 +317,58 @@ def __handle_from_node(self, pack_dir_this_command, node): resp["zip_password"] = "" rm_rf_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) else: - self.__handle_zip_file(node.get("ip"), ssh, resp, gather_dir_name, pack_dir_this_command) + self.__handle_zip_file( + node.get("ip"), ssh, resp, gather_dir_name, pack_dir_this_command + ) ssh.ssh_close() return resp - + def __grep_log_until_empty(self, ssh_helper, home_path, log_list, gather_path): """ 按时间顺序排序日志，从最新的时间（或者从设置的时间）开始往前找日志，直到grep的结果不为空，再直到grep的结果为空，则停止 :param ssh_helper, home_path, log_list, gather_path :return: """ - log_type_list = ['observer', 'election', 'rootservice'] + log_type_list = ["observer", "election", "rootservice"] # 理论上只有上述三种日志，other_log_list应该为空 - other_log_list = [log_name for log_name in log_list if not any(log_name.startswith(prefix) for prefix in log_type_list)] + other_log_list = [ + log_name + for log_name in log_list + if not any(log_name.startswith(prefix) for prefix in log_type_list) + ] for log_name in other_log_list: - self.__pharse_log(ssh_helper=ssh_helper, log_name=log_name, home_path=home_path, gather_path=gather_path) + self.__pharse_log( + ssh_helper=ssh_helper, + log_name=log_name, + home_path=home_path, + gather_path=gather_path, + ) # wf结尾的日志非全量日志，不排查 # 形如observer.log等日志不方便排序，暂时删除，在后续重新加上 - log_list = [log_name for log_name in log_list if (log_name not in other_log_list) and log_name[-1].isdigit()] + log_list = [ + log_name + for log_name in log_list + if (log_name not in other_log_list) and log_name[-1].isdigit() + ] for log_type in log_type_list: - cur_type_log_list = [log_name for log_name in log_list if log_name.startswith(log_type)] + cur_type_log_list = [ + log_name for log_name in log_list if log_name.startswith(log_type) + ] # 按照时间从最新的到最旧的 cur_type_log_list.sort(reverse=True) # 没有时间后缀的是最新日志，插入到首部 - cur_type_log_list.insert(0, f'{log_type}.log') + cur_type_log_list.insert(0, f"{log_type}.log") has_res = False for log_name in cur_type_log_list: - is_empty = self.__grep_log(ssh_helper=ssh_helper, log_name=log_name, home_path=home_path, gather_path=gather_path) + is_empty = self.__grep_log( + ssh_helper=ssh_helper, + log_name=log_name, + home_path=home_path, + gather_path=gather_path, + ) if not is_empty: has_res = True elif has_res: @@ -277,14 +387,25 @@ def __grep_log(self, ssh_helper, home_path, log_name, gather_path): grep_args=self.grep_options, gather_path=gather_path, log_name=log_name, - log_dir=log_path) - find_file_cmd = "find {gather_path} -type f -name {log_name} ! -empty".format( - gather_path=gather_path, - log_name=log_name) + log_dir=log_path, + ) + find_file_cmd = ( + "find {gather_path} -type f -name {log_name} ! -empty".format( + gather_path=gather_path, log_name=log_name + ) + ) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) self.stdio.verbose("grep files, run cmd = [{0}]".format(find_file_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) - find_file_res = SshClient(self.stdio).run(ssh_helper, find_file_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ( + SshClient(self.stdio).run(ssh_helper, grep_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(grep_cmd) + ) + find_file_res = ( + SshClient(self.stdio).run(ssh_helper, find_file_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(grep_cmd) + ) return find_file_res == "" else: raise Exception("grep arg is none.") @@ -293,15 +414,25 @@ def __handle_log_list(self, ssh, node, resp): log_list = self.__get_log_name(ssh, node) ip = node.get("ip") if len(log_list) > self.file_number_limit: - self.stdio.warn('{0} The number of log files is {1}, out of range (0,{2}], ' - "Please adjust the query limit".format(ip, len(log_list), self.file_number_limit)) - resp["skip"] = True, - resp["error"] = "Too many files {0} > {1}".format(len(log_list), self.file_number_limit) + self.stdio.warn( + "{0} The number of log files is {1}, out of range (0,{2}], " + "Please adjust the query limit".format( + ip, len(log_list), self.file_number_limit + ) + ) + resp["skip"] = (True,) + resp["error"] = "Too many files {0} > {1}".format( + len(log_list), self.file_number_limit + ) return log_list, resp elif len(log_list) <= 0: - self.stdio.warn('{0} The number of log files is {1}, The time range for file gather from {2} to {3}, and no eligible files were found.' - " Please adjust the query time limit.".format(ip, len(log_list), self.from_time_str, self.to_time_str)) - resp["skip"] = True, + self.stdio.warn( + "{0} The number of log files is {1}, The time range for file gather from {2} to {3}, and no eligible files were found." + " Please adjust the query time limit.".format( + ip, len(log_list), self.from_time_str, self.to_time_str + ) + ) + resp["skip"] = (True,) resp["error"] = "No files found" return log_list, resp return log_list, resp @@ -314,17 +445,40 @@ def __get_log_name(self, ssh_helper, node): """ home_path = node.get("home_path") log_path = os.path.join(home_path, "log") - if self.scope == "observer" or self.scope == "rootservice" or self.scope == "election": - get_oblog = "ls -1 -F %s/*%s.log* | awk -F '/' '{print $NF}'" % (log_path, self.scope) + if ( + self.scope == "observer" + or self.scope == "rootservice" + or self.scope == "election" + ): + get_oblog = "ls -1 -F %s/*%s.log* | awk -F '/' '{print $NF}'" % ( + log_path, + self.scope, + ) else: - get_oblog = "ls -1 -F %s/observer.log* %s/rootservice.log* %s/election.log* | awk -F '/' '{print $NF}'" % \ - (log_path, log_path, log_path) + get_oblog = ( + "ls -1 -F %s/observer.log* %s/rootservice.log* %s/election.log* | awk -F '/' '{print $NF}'" + % (log_path, log_path, log_path) + ) log_name_list = [] - log_files = SshClient(self.stdio).run(ssh_helper, get_oblog) if self.is_ssh else LocalClient(self.stdio).run(get_oblog) + log_files = ( + SshClient(self.stdio).run(ssh_helper, get_oblog) + if self.is_ssh + else LocalClient(self.stdio).run(get_oblog) + ) if log_files: - log_name_list = get_logfile_name_list(self.is_ssh, ssh_helper, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) + log_name_list = get_logfile_name_list( + self.is_ssh, + ssh_helper, + self.from_time_str, + self.to_time_str, + log_path, + log_files, + self.stdio, + ) else: - self.stdio.error('Unable to find the log file. Please provide the correct home_path, the default is [/root/observer]') + self.stdio.error( + "Unable to find the log file. Please provide the correct home_path, the default is [/root/observer]" + ) return log_name_list def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): @@ -335,56 +489,96 @@ def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): """ log_path = os.path.join(home_path, "log") if self.grep_options is not None: - grep_cmd="" + grep_cmd = "" if type(self.grep_options) == str: grep_cmd = "grep -e '{grep_options}' {log_dir}/{log_name} >> {gather_path}/{log_name} ".format( grep_options=self.grep_options, gather_path=gather_path, log_name=log_name, - log_dir=log_path) - elif type(self.grep_options) == list and len(self.grep_options)>0: - grep_litter_cmd="" + log_dir=log_path, + ) + elif type(self.grep_options) == list and len(self.grep_options) > 0: + grep_litter_cmd = "" for grep_option in self.grep_options: - if type(grep_option)!=str: - self.stdio.error('The grep args must be string or list of strings, but got {0}'.format(type(grep_option))) - raise Exception('The grep args must be string or list of strings, but got {0}'.format(type(grep_option))) - elif grep_option == "": - self.stdio.warn('The grep args must be string or list of strings, but got ""') + if type(grep_option) != str: + self.stdio.error( + "The grep args must be string or list of strings, but got {0}".format( + type(grep_option) + ) + ) + raise Exception( + "The grep args must be string or list of strings, but got {0}".format( + type(grep_option) + ) + ) + elif grep_option == "": + self.stdio.warn( + 'The grep args must be string or list of strings, but got ""' + ) continue grep_litter_cmd += "| grep -e '{0}'".format(grep_option) grep_cmd = "cat {log_dir}/{log_name} {grep_options} >> {gather_path}/{log_name} ".format( grep_options=grep_litter_cmd, gather_path=gather_path, log_name=log_name, - log_dir=log_path) - self.stdio.verbose('grep files, run cmd = [{0}]'.format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + log_dir=log_path, + ) + self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) + ( + SshClient(self.stdio).run(ssh_helper, grep_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(grep_cmd) + ) else: cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format( - gather_path=gather_path, - log_name=log_name, - log_dir=log_path) - self.stdio.verbose('copy files, run cmd = [{0}]'.format(cp_cmd)) - SshClient(self.stdio).run(ssh_helper, cp_cmd) if self.is_ssh else LocalClient(self.stdio).run(cp_cmd) + gather_path=gather_path, log_name=log_name, log_dir=log_path + ) + self.stdio.verbose("copy files, run cmd = [{0}]".format(cp_cmd)) + ( + SshClient(self.stdio).run(ssh_helper, cp_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cp_cmd) + ) def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_command): zip_password = "" - gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name) - self.stdio.start_loading('[ip: {0}] zip observer log start'.format(ip)) + gather_dir_full_path = "{0}/{1}".format( + self.gather_ob_log_temporary_dir, gather_dir_name + ) + self.stdio.start_loading("[ip: {0}] zip observer log start".format(ip)) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(self.is_ssh, ssh, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + zip_encrypt_dir( + self.is_ssh, + ssh, + zip_password, + self.gather_ob_log_temporary_dir, + gather_dir_name, + self.stdio, + ) else: - zip_dir(self.is_ssh, ssh, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) - self.stdio.stop_loading('[ip: {0}] zip observer log end'.format(ip)) + zip_dir( + self.is_ssh, + ssh, + self.gather_ob_log_temporary_dir, + gather_dir_name, + self.stdio, + ) + self.stdio.stop_loading("[ip: {0}] zip observer log end".format(ip)) gather_package_dir = "{0}.zip".format(gather_dir_full_path) - gather_log_file_size = get_file_size(self.is_ssh, ssh, gather_package_dir, self.stdio) + gather_log_file_size = get_file_size( + self.is_ssh, ssh, gather_package_dir, self.stdio + ) self.stdio.print(FileUtil.show_file_size_tabulate(ip, gather_log_file_size)) local_store_path = "" if int(gather_log_file_size) < self.file_size_limit: - local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) - download_file(self.is_ssh, ssh, gather_package_dir, local_store_path, self.stdio) + local_store_path = pack_dir_this_command + "/{0}.zip".format( + gather_dir_name + ) + download_file( + self.is_ssh, ssh, gather_package_dir, local_store_path, self.stdio + ) resp["error"] = "" resp["zip_password"] = zip_password else: @@ -392,7 +586,11 @@ def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_comman resp["zip_password"] = "" rm_rf_file(self.is_ssh, ssh, gather_package_dir, self.stdio) resp["gather_pack_path"] = local_store_path - self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ip, gather_package_dir)) + self.stdio.verbose( + "Collect pack gathered from node {0}: stored in {1}".format( + ip, gather_package_dir + ) + ) return resp @staticmethod @@ -419,10 +617,26 @@ def __get_overall_summary(node_summary_tuple, is_zip_encrypt): except: format_file_size = FileUtil.size_format(num=0, output_str=True) if is_zip_encrypt: - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, tup[4], "{0} s".format(int(consume_time)), pack_path)) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + tup[4], + "{0} s".format(int(consume_time)), + pack_path, + ) + ) else: - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) - return "\nGather Ob Log Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather Ob Log Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/gather_obadmin.py b/handler/gather/gather_obadmin.py index 41e50d0a..302f688d 100644 --- a/handler/gather/gather_obadmin.py +++ b/handler/gather/gather_obadmin.py @@ -24,7 +24,14 @@ from common.constant import const from common.command import LocalClient, SshClient, is_empty_dir from handler.base_shell_handler import BaseShellHandler -from common.command import download_file, rm_rf_file, get_file_size, zip_encrypt_dir, zip_dir, get_observer_version +from common.command import ( + download_file, + rm_rf_file, + get_file_size, + zip_encrypt_dir, + zip_dir, + get_observer_version, +) from common.ssh import SshHelper from common.tool import TimeUtils from common.tool import StringUtils @@ -35,7 +42,7 @@ class GatherObAdminHandler(BaseShellHandler): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): super(GatherObAdminHandler, self).__init__() self.context = context self.stdio = context.stdio @@ -48,37 +55,37 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.grep_args = None self.zip_encrypt = False self.config_path = const.DEFAULT_CONFIG_PATH - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.nodes, self.stdio) if new_nodes: self.nodes = new_nodes self.inner_config = self.context.inner_config - self.ob_admin_mode = 'clog' - if self.context.get_variable("gather_obadmin_mode", None) : + self.ob_admin_mode = "clog" + if self.context.get_variable("gather_obadmin_mode", None): self.ob_admin_mode = self.context.get_variable("gather_obadmin_mode") if self.inner_config is None: self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] return True def init_option(self): options = self.context.options - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - since_option = Util.get_option(options, 'since') - store_dir_option = Util.get_option(options, 'store_dir') - encrypt_option = Util.get_option(options, 'encrypt') + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + since_option = Util.get_option(options, "since") + store_dir_option = Util.get_option(options, "store_dir") + encrypt_option = Util.get_option(options, "encrypt") if from_option is not None and to_option is not None: try: from_timestamp = TimeUtils.parse_time_str(from_option) @@ -86,28 +93,66 @@ def init_option(self): self.from_time_str = from_option self.to_time_str = to_option except OBDIAGFormatException: - self.stdio.exception('Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}'.format(from_option, to_option)) + self.stdio.exception( + "Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}".format( + from_option, to_option + ) + ) return False if to_timestamp <= from_timestamp: - self.stdio.exception('Error: from datetime is larger than to datetime, please check.') + self.stdio.exception( + "Error: from datetime is larger than to datetime, please check." + ) return False elif (from_option is None or to_option is None) and since_option is not None: now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option, self.stdio))).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option, self.stdio) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 30 minutes') + self.stdio.warn( + "No time option provided, default processing is based on the last 30 minutes" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) if since_option is not None: - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option, self.stdio))).strftime('%Y-%m-%d %H:%M:%S') + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec( + since_option, self.stdio + ) + ) + ).strftime("%Y-%m-%d %H:%M:%S") else: - self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) - if store_dir_option and store_dir_option != './': + self.from_time_str = ( + now_time - datetime.timedelta(minutes=30) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) + if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('Error: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "Error: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.local_stored_path = os.path.abspath(store_dir_option) if encrypt_option == "true": @@ -116,12 +161,17 @@ def init_option(self): def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False - pack_dir_this_command = os.path.join(self.local_stored_path, "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.local_stored_path, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] @@ -131,11 +181,17 @@ def handle_from_node(node): file_size = "" if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], - file_size, - resp["zip_password"], - int(time.time() - st), - resp["gather_pack_path"])) + gather_tuples.append( + ( + node.get("ip"), + False, + resp["error"], + file_size, + resp["zip_password"], + int(time.time() - st), + resp["gather_pack_path"], + ) + ) if self.is_ssh: for node in self.nodes: @@ -151,28 +207,33 @@ def handle_from_node(node): mode = "slog" else: mode = "clog" - summary_tuples = self.__get_overall_summary(gather_tuples, mode, self.zip_encrypt) + summary_tuples = self.__get_overall_summary( + gather_tuples, mode, self.zip_encrypt + ) self.stdio.print(summary_tuples) # Persist the summary results to a file - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) - last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + last_info = ( + "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + os.path.join(pack_dir_this_command, "result_summary.txt") + ) + ) def __handle_from_node(self, local_stored_path, node): - resp = { - "skip": False, - "error": "", - "gather_pack_path": "" - } + resp = {"skip": False, "error": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") self.stdio.verbose( - "Sending Collect Shell Command to node {0} ...".format(remote_ip)) + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=local_stored_path, stdio=self.stdio) - now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") if self.ob_admin_mode == "slog": remote_dir_name = "slog_{0}_{1}".format(remote_ip, now_time) else: @@ -180,22 +241,46 @@ def __handle_from_node(self, local_stored_path, node): remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_helper = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.error("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.error( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) if not ssh_failed: mkdir_cmd = "mkdir -p {0}".format(remote_dir_full_path) - SshClient(self.stdio).run(ssh_helper, mkdir_cmd) if self.is_ssh else LocalClient(self.stdio).run(mkdir_cmd) - ob_version = get_observer_version(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) - if (ob_version != "" and not StringUtils.compare_versions_lower(ob_version, const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN, self.stdio)) or ob_version == "": - self.stdio.verbose("This version {0} does not support gather clog/slog . The max supported version less than {1}". - format(ob_version, const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN)) + ( + SshClient(self.stdio).run(ssh_helper, mkdir_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(mkdir_cmd) + ) + ob_version = get_observer_version( + self.is_ssh, ssh_helper, node.get("home_path"), self.stdio + ) + if ( + ob_version != "" + and not StringUtils.compare_versions_lower( + ob_version, const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN, self.stdio + ) + ) or ob_version == "": + self.stdio.verbose( + "This version {0} does not support gather clog/slog . The max supported version less than {1}".format( + ob_version, const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN + ) + ) resp["error"] = "{0} not support gather clog/slog".format(ob_version) resp["gather_pack_path"] = "{0}".format(local_stored_path) resp["zip_password"] = "" @@ -205,11 +290,15 @@ def __handle_from_node(self, local_stored_path, node): self.__gather_log_info(ssh_helper, node, slog, remote_dir_full_path) self.__mv_log(ssh_helper, remote_dir_full_path) - if is_empty_dir(self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio): + if is_empty_dir( + self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio + ): resp["error"] = "gather failed, folder is empty" resp["zip_password"] = "" else: - resp = self.__handle_zip_file(remote_ip, ssh_helper, resp, remote_dir_name, local_stored_path) + resp = self.__handle_zip_file( + remote_ip, ssh_helper, resp, remote_dir_name, local_stored_path + ) rm_rf_file(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) return resp @@ -218,33 +307,56 @@ def __handle_log_list(self, ssh, node, resp): if len(log_list) > 20: self.stdio.warn( "{0} The number of log files is {1}, out of range (0,20], " - "Please adjust the query limit".format(node.get("ip"), len(log_list))) - resp["skip"] = True, + "Please adjust the query limit".format(node.get("ip"), len(log_list)) + ) + resp["skip"] = (True,) resp["error"] = "Too many files {0} > 20".format(len(log_list)) return log_list, resp elif len(log_list) <= 0: self.stdio.warn( "{0} The number of log files is {1}, out of range (0,20], " - "Please adjust the query limit".format(node.get("ip"), len(log_list))) - resp["skip"] = True, + "Please adjust the query limit".format(node.get("ip"), len(log_list)) + ) + resp["skip"] = (True,) resp["error"] = "No files found" return log_list, resp return log_list, resp def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_command): zip_password = "" - gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name) + gather_dir_full_path = "{0}/{1}".format( + self.gather_ob_log_temporary_dir, gather_dir_name + ) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(self.is_ssh, ssh, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + zip_encrypt_dir( + self.is_ssh, + ssh, + zip_password, + self.gather_ob_log_temporary_dir, + gather_dir_name, + self.stdio, + ) else: - zip_dir(self.is_ssh, ssh, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + zip_dir( + self.is_ssh, + ssh, + self.gather_ob_log_temporary_dir, + gather_dir_name, + self.stdio, + ) gather_package_dir = "{0}.zip".format(gather_dir_full_path) - gather_log_file_size = get_file_size(self.is_ssh, ssh, gather_package_dir, self.stdio) - self.stdio.print(FileUtil.show_file_size_tabulate(ip, gather_log_file_size, self.stdio)) + gather_log_file_size = get_file_size( + self.is_ssh, ssh, gather_package_dir, self.stdio + ) + self.stdio.print( + FileUtil.show_file_size_tabulate(ip, gather_log_file_size, self.stdio) + ) local_path = "" if int(gather_log_file_size) < self.file_size_limit: - local_path = download_file(self.is_ssh, ssh, gather_package_dir, pack_dir_this_command, self.stdio) + local_path = download_file( + self.is_ssh, ssh, gather_package_dir, pack_dir_this_command, self.stdio + ) resp["error"] = "" resp["zip_password"] = zip_password else: @@ -254,7 +366,10 @@ def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_comman resp["gather_pack_path"] = local_path self.stdio.verbose( - "Collect pack gathered from node {0}: stored in {1}".format(ip, gather_package_dir)) + "Collect pack gathered from node {0}: stored in {1}".format( + ip, gather_package_dir + ) + ) return resp def __get_log_name(self, ssh_helper, node): @@ -266,27 +381,50 @@ def __get_log_name(self, ssh_helper, node): slog_dir = os.path.join(node.get("data_dir"), "/slog") clog_dir = os.path.join(node.get("data_dir"), "/clog") if self.ob_admin_mode == "slog": - get_log = "ls -l SLOG_DIR --time-style '+.%Y%m%d%H%M%S' | awk '{print $7,$6}'".replace("SLOG_DIR", slog_dir) + get_log = "ls -l SLOG_DIR --time-style '+.%Y%m%d%H%M%S' | awk '{print $7,$6}'".replace( + "SLOG_DIR", slog_dir + ) else: - get_log = "ls -l CLOG_DIR --time-style '+.%Y%m%d%H%M%S' | awk '{print $7,$6}'".replace("CLOG_DIR", clog_dir) - log_files = SshClient(self.stdio).run(ssh_helper, get_log) if self.is_ssh else LocalClient(self.stdio).run(get_log) + get_log = "ls -l CLOG_DIR --time-style '+.%Y%m%d%H%M%S' | awk '{print $7,$6}'".replace( + "CLOG_DIR", clog_dir + ) + log_files = ( + SshClient(self.stdio).run(ssh_helper, get_log) + if self.is_ssh + else LocalClient(self.stdio).run(get_log) + ) log_name_list = [] - for file_name in log_files.split('\n'): + for file_name in log_files.split("\n"): if file_name == "": self.stdio.verbose("existing file name is empty") continue log_name_fields = file_name.split(".") - if bytes.isdigit(log_name_fields[-1].encode("utf-8")) and len(log_name_fields[-1]) == 14: - log_time = datetime.datetime.strptime(log_name_fields[-1], "%Y%m%d%H%M%S") - from_time = datetime.datetime.strptime(self.from_time_str, "%Y-%m-%d %H:%M:%S") - to_time = datetime.datetime.strptime(self.to_time_str, "%Y-%m-%d %H:%M:%S") + if ( + bytes.isdigit(log_name_fields[-1].encode("utf-8")) + and len(log_name_fields[-1]) == 14 + ): + log_time = datetime.datetime.strptime( + log_name_fields[-1], "%Y%m%d%H%M%S" + ) + from_time = datetime.datetime.strptime( + self.from_time_str, "%Y-%m-%d %H:%M:%S" + ) + to_time = datetime.datetime.strptime( + self.to_time_str, "%Y-%m-%d %H:%M:%S" + ) if (log_time > from_time) and (log_time < to_time): log_name_list.append(str(log_name_fields[0]).rstrip()) if len(log_name_list): - self.stdio.verbose("Find the qualified log file {0} on Server [{1}], " - "wait for the next step".format(log_name_list, ssh_helper.get_name())) + self.stdio.verbose( + "Find the qualified log file {0} on Server [{1}], " + "wait for the next step".format(log_name_list, ssh_helper.get_name()) + ) else: - self.stdio.warn("No found the qualified log file on Server [{0}]".format(ssh_helper.get_name())) + self.stdio.warn( + "No found the qualified log file on Server [{0}]".format( + ssh_helper.get_name() + ) + ) return log_name_list def __gather_log_info(self, ssh_helper, node, log_name, remote_dir): @@ -297,24 +435,37 @@ def __gather_log_info(self, ssh_helper, node, log_name, remote_dir): ob_install_dir=home_path, store_dir=remote_dir, obadmin_dir=obadmin_install_dir, - slog_name=log_name) + slog_name=log_name, + ) else: cmd = "export LD_LIBRARY_PATH={ob_install_dir}/lib && cd {store_dir} && {obadmin_dir}/ob_admin clog_tool dump_all {clog_name}".format( ob_install_dir=home_path, store_dir=remote_dir, obadmin_dir=obadmin_install_dir, clog_name=log_name, - ) + ) self.stdio.verbose("gather obadmin info, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ( + SshClient(self.stdio).run(ssh_helper, cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cmd) + ) def __mv_log(self, ssh_helper, remote_dir): if self.ob_admin_mode == "slog": - cmd = "cd {remote_dir} && mv ob_admin.log ob_admin_slog.log".format(remote_dir=remote_dir) + cmd = "cd {remote_dir} && mv ob_admin.log ob_admin_slog.log".format( + remote_dir=remote_dir + ) else: - cmd = "cd {remote_dir} && mv ob_admin.log ob_admin_clog.log".format(remote_dir=remote_dir) + cmd = "cd {remote_dir} && mv ob_admin.log ob_admin_clog.log".format( + remote_dir=remote_dir + ) self.stdio.verbose("mv log info, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ( + SshClient(self.stdio).run(ssh_helper, cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cmd) + ) @staticmethod def __get_overall_summary(node_summary_tuple, mode, is_zip_encrypt): @@ -335,14 +486,31 @@ def __get_overall_summary(node_summary_tuple, mode, is_zip_encrypt): except: format_file_size = FileUtil.size_format(num=0, output_str=True) if is_zip_encrypt: - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, tup[4], "{0} s".format(int(consume_time)), pack_path)) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + tup[4], + "{0} s".format(int(consume_time)), + pack_path, + ) + ) else: - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) if mode == "slog": - return "\nGather slog Summary:\n" + tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", - showindex=False) + return "\nGather slog Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) else: - return "\nGather clog Summary:\n" + tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", - showindex=False) + return "\nGather clog Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index ccb2e3e7..831ce167 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -25,8 +25,17 @@ from common.obdiag_exception import OBDIAGFormatException from common.command import LocalClient, SshClient from common.constant import const -from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, \ - rm_rf_file, zip_encrypt_dir, zip_dir +from common.command import ( + get_file_size, + download_file, + is_empty_dir, + get_logfile_name_list, + mkdir, + delete_empty_file, + rm_rf_file, + zip_encrypt_dir, + zip_dir, +) from common.ssh import SshHelper from common.tool import Util from common.tool import DirectoryUtil @@ -36,7 +45,7 @@ class GatherObProxyLogHandler(BaseShellHandler): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): super(GatherObProxyLogHandler, self).__init__() self.pack_dir_this_command = "" self.context = context @@ -52,13 +61,13 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.zip_encrypt = False self.is_scene = is_scene self.config_path = const.DEFAULT_CONFIG_PATH - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.obproxy_config['servers'] + self.nodes = self.context.obproxy_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.nodes, self.stdio) if new_nodes: self.nodes = new_nodes @@ -67,30 +76,30 @@ def init_config(self): self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] return True def init_option(self): options = self.context.options - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - since_option = Util.get_option(options, 'since') - store_dir_option = Util.get_option(options, 'store_dir') - grep_option = Util.get_option(options, 'grep') - encrypt_option = Util.get_option(options, 'encrypt') - scope_option = Util.get_option(options, 'scope') - if self.context.get_variable("gather_from",None) : - from_option=self.context.get_variable("gather_from") - if self.context.get_variable("gather_to",None) : - to_option=self.context.get_variable("gather_to") - if self.context.get_variable("gather_since",None) : - since_option=self.context.get_variable("gather_since") - if self.context.get_variable("store_dir",None) : - store_dir_option=self.context.get_variable("store_dir") - if self.context.get_variable("gather_scope",None) : + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + since_option = Util.get_option(options, "since") + store_dir_option = Util.get_option(options, "store_dir") + grep_option = Util.get_option(options, "grep") + encrypt_option = Util.get_option(options, "encrypt") + scope_option = Util.get_option(options, "scope") + if self.context.get_variable("gather_from", None): + from_option = self.context.get_variable("gather_from") + if self.context.get_variable("gather_to", None): + to_option = self.context.get_variable("gather_to") + if self.context.get_variable("gather_since", None): + since_option = self.context.get_variable("gather_since") + if self.context.get_variable("store_dir", None): + store_dir_option = self.context.get_variable("store_dir") + if self.context.get_variable("gather_scope", None): scope_option = self.context.get_variable("gather_scope") if self.context.get_variable("gather_grep", None): grep_option = self.context.get_variable("gather_grep") @@ -102,28 +111,64 @@ def init_option(self): self.from_time_str = from_option self.to_time_str = to_option except OBDIAGFormatException: - self.stdio.exception('Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}'.format(from_option, to_option)) + self.stdio.exception( + "Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}".format( + from_option, to_option + ) + ) return False if to_timestamp <= from_timestamp: - self.stdio.exception('Error: from datetime is larger than to datetime, please check.') + self.stdio.exception( + "Error: from datetime is larger than to datetime, please check." + ) return False elif (from_option is None or to_option is None) and since_option is not None: now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 30 minutes') + self.stdio.warn( + "No time option provided, default processing is based on the last 30 minutes" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) if since_option is not None: - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") else: - self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) - if store_dir_option and store_dir_option != './': + self.from_time_str = ( + now_time - datetime.timedelta(minutes=30) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) + if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) if scope_option: @@ -137,15 +182,20 @@ def init_option(self): def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False if self.is_scene: pack_dir_this_command = self.gather_pack_dir else: - pack_dir_this_command = os.path.join(self.gather_pack_dir, "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.gather_pack_dir, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] @@ -156,7 +206,17 @@ def handle_from_node(node): file_size = "" if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) + gather_tuples.append( + ( + node.get("ip"), + False, + resp["error"], + file_size, + resp["zip_password"], + int(time.time() - st), + resp["gather_pack_path"], + ) + ) if self.is_ssh: for node in self.nodes: @@ -170,18 +230,19 @@ def handle_from_node(node): summary_tuples = self.__get_overall_summary(gather_tuples, self.zip_encrypt) self.stdio.print(summary_tuples) - self.pack_dir_this_command=pack_dir_this_command - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) - last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + self.pack_dir_this_command = pack_dir_this_command + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) + last_info = ( + "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + os.path.join(pack_dir_this_command, "result_summary.txt") + ) + ) return True def __handle_from_node(self, node, pack_dir_this_command): - resp = { - "skip": False, - "error": "", - "zip_password": "", - "gather_pack_path": "" - } + resp = {"skip": False, "error": "", "zip_password": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") @@ -189,22 +250,40 @@ def __handle_from_node(self, node, pack_dir_this_command): remote_private_key = node.get("ssh_key_file") remote_home_path = node.get("home_path") ssh_failed = False - self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) + self.stdio.verbose( + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key,node, self.stdio) + ssh = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.exception( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) if not ssh_failed: - from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) - to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) - gather_dir_name = "obproxy_log_{0}_{1}_{2}".format(ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp) + from_datetime_timestamp = TimeUtils.timestamp_to_filename_time( + TimeUtils.datetime_to_timestamp(self.from_time_str) + ) + to_datetime_timestamp = TimeUtils.timestamp_to_filename_time( + TimeUtils.datetime_to_timestamp(self.to_time_str) + ) + gather_dir_name = "obproxy_log_{0}_{1}_{2}".format( + ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp + ) gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) @@ -212,7 +291,12 @@ def __handle_from_node(self, node, pack_dir_this_command): if resp["skip"]: return resp for log_name in log_list: - self.__pharse_log(ssh_helper=ssh, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) + self.__pharse_log( + ssh_helper=ssh, + log_name=log_name, + home_path=remote_home_path, + gather_path=gather_dir_full_path, + ) delete_empty_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) is_empty = is_empty_dir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) @@ -221,7 +305,9 @@ def __handle_from_node(self, node, pack_dir_this_command): resp["zip_password"] = "" rm_rf_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) else: - self.__handle_zip_file(remote_ip, ssh, resp, gather_dir_name, pack_dir_this_command) + self.__handle_zip_file( + remote_ip, ssh, resp, gather_dir_name, pack_dir_this_command + ) ssh.ssh_close() return resp @@ -229,15 +315,25 @@ def __handle_log_list(self, ssh, node, resp): log_list = self.__get_log_name(ssh, node) ip = node.get("ip") if len(log_list) > self.file_number_limit: - self.stdio.warn("{0} The number of log files is {1}, out of range (0,{2}], " - "Please adjust the query limit".format(ip, len(log_list), self.file_number_limit)) - resp["skip"] = True, - resp["error"] = "Too many files {0} > {1}".format(len(log_list), self.file_number_limit) + self.stdio.warn( + "{0} The number of log files is {1}, out of range (0,{2}], " + "Please adjust the query limit".format( + ip, len(log_list), self.file_number_limit + ) + ) + resp["skip"] = (True,) + resp["error"] = "Too many files {0} > {1}".format( + len(log_list), self.file_number_limit + ) return log_list, resp elif len(log_list) <= 0: - self.stdio.warn("{0} The number of log files is {1}, The time range for file gather from {2} to {3}, and no eligible files were found." - " Please adjust the query time limit.".format(ip, len(log_list), self.from_time_str, self.to_time_str)) - resp["skip"] = True, + self.stdio.warn( + "{0} The number of log files is {1}, The time range for file gather from {2} to {3}, and no eligible files were found." + " Please adjust the query time limit.".format( + ip, len(log_list), self.from_time_str, self.to_time_str + ) + ) + resp["skip"] = (True,) resp["error"] = "No files found" return log_list, resp return log_list, resp @@ -245,20 +341,45 @@ def __handle_log_list(self, ssh, node, resp): def __get_log_name(self, ssh_helper, node): home_path = node.get("home_path") log_path = os.path.join(home_path, "log") - if self.scope == "obproxy" or self.scope == "obproxy_stat" or self.scope == "obproxy_digest" or \ - self.scope == "obproxy_limit" or self.scope == "obproxy_slow" or self.scope == "obproxy_diagnosis" or self.scope == "obproxy_error": - get_obproxy_log = "ls -1 -F %s/*%s.*log* | awk -F '/' '{print $NF}'" % (log_path, self.scope) + if ( + self.scope == "obproxy" + or self.scope == "obproxy_stat" + or self.scope == "obproxy_digest" + or self.scope == "obproxy_limit" + or self.scope == "obproxy_slow" + or self.scope == "obproxy_diagnosis" + or self.scope == "obproxy_error" + ): + get_obproxy_log = "ls -1 -F %s/*%s.*log* | awk -F '/' '{print $NF}'" % ( + log_path, + self.scope, + ) else: - get_obproxy_log = "ls -1 -F %s/obproxy.*log* %s/obproxy_error.*log* %s/obproxy_stat.*log* %s/obproxy_digest.*log* %s/obproxy_limit.*log* %s/obproxy_slow.*log* %s/obproxy_diagnosis.*log*| awk -F '/' '{print $NF}'" % (log_path, log_path, log_path, log_path, log_path, log_path,log_path) + get_obproxy_log = ( + "ls -1 -F %s/obproxy.*log* %s/obproxy_error.*log* %s/obproxy_stat.*log* %s/obproxy_digest.*log* %s/obproxy_limit.*log* %s/obproxy_slow.*log* %s/obproxy_diagnosis.*log*| awk -F '/' '{print $NF}'" + % (log_path, log_path, log_path, log_path, log_path, log_path, log_path) + ) if self.is_ssh: log_files = SshClient(self.stdio).run(ssh_helper, get_obproxy_log).strip() else: log_files = LocalClient(self.stdio).run(get_obproxy_log).strip() log_name_list = [] if log_files: - log_name_list = get_logfile_name_list(self.is_ssh, ssh_helper, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) + log_name_list = get_logfile_name_list( + self.is_ssh, + ssh_helper, + self.from_time_str, + self.to_time_str, + log_path, + log_files, + self.stdio, + ) else: - self.stdio.error("Unable to find the log file. Please provide the correct home_path config and check obproxy {0} log exist".format(log_path)) + self.stdio.error( + "Unable to find the log file. Please provide the correct home_path config and check obproxy {0} log exist".format( + log_path + ) + ) return log_name_list def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): @@ -269,21 +390,32 @@ def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): """ log_path = os.path.join(home_path, "log") if self.grep_args is not None: - grep_cmd="" + grep_cmd = "" if type(self.grep_args) == str: grep_cmd = "grep -e '{grep_args}' {log_dir}/{log_name} >> {gather_path}/{log_name} ".format( grep_args=self.grep_args, gather_path=gather_path, log_name=log_name, - log_dir=log_path) - elif type(self.grep_args) == list and len(self.grep_args)>0: - grep_litter_cmd="" + log_dir=log_path, + ) + elif type(self.grep_args) == list and len(self.grep_args) > 0: + grep_litter_cmd = "" for grep_arg in self.grep_args: - if type(grep_arg)!=str: - self.stdio.error('The grep args must be string or list of strings, but got {0}'.format(type(grep_arg))) - raise Exception('The grep args must be string or list of strings, but got {0}'.format(type(grep_arg))) - elif grep_arg == "": - self.stdio.warn('The grep args must be string or list of strings, but got ""') + if type(grep_arg) != str: + self.stdio.error( + "The grep args must be string or list of strings, but got {0}".format( + type(grep_arg) + ) + ) + raise Exception( + "The grep args must be string or list of strings, but got {0}".format( + type(grep_arg) + ) + ) + elif grep_arg == "": + self.stdio.warn( + 'The grep args must be string or list of strings, but got ""' + ) continue grep_litter_cmd += "| grep -e '{0}'".format(grep_arg) @@ -291,33 +423,62 @@ def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): grep_args=grep_litter_cmd, gather_path=gather_path, log_name=log_name, - log_dir=log_path) + log_dir=log_path, + ) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ( + SshClient(self.stdio).run(ssh_helper, grep_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(grep_cmd) + ) else: cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format( - gather_path=gather_path, - log_name=log_name, - log_dir=log_path) + gather_path=gather_path, log_name=log_name, log_dir=log_path + ) self.stdio.verbose("copy files, run cmd = [{0}]".format(cp_cmd)) - SshClient(self.stdio).run(ssh_helper, cp_cmd) if self.is_ssh else LocalClient(self.stdio).run(cp_cmd) + ( + SshClient(self.stdio).run(ssh_helper, cp_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cp_cmd) + ) def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_command): zip_password = "" - gather_dir_full_path = "{0}/{1}".format(self.gather_log_temporary_dir, gather_dir_name) + gather_dir_full_path = "{0}/{1}".format( + self.gather_log_temporary_dir, gather_dir_name + ) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(self.is_ssh, ssh, zip_password, self.gather_log_temporary_dir, gather_dir_name, self.stdio) + zip_encrypt_dir( + self.is_ssh, + ssh, + zip_password, + self.gather_log_temporary_dir, + gather_dir_name, + self.stdio, + ) else: - zip_dir(self.is_ssh, ssh, self.gather_log_temporary_dir, gather_dir_name, self.stdio) + zip_dir( + self.is_ssh, + ssh, + self.gather_log_temporary_dir, + gather_dir_name, + self.stdio, + ) gather_package_dir = "{0}.zip".format(gather_dir_full_path) - gather_log_file_size = get_file_size(self.is_ssh, ssh, gather_package_dir, self.stdio) + gather_log_file_size = get_file_size( + self.is_ssh, ssh, gather_package_dir, self.stdio + ) self.stdio.print(FileUtil.show_file_size_tabulate(ip, gather_log_file_size)) local_path = "" if int(gather_log_file_size) < self.file_size_limit: - local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) - local_path = download_file(self.is_ssh, ssh, gather_package_dir, local_store_path, self.stdio) + local_store_path = pack_dir_this_command + "/{0}.zip".format( + gather_dir_name + ) + local_path = download_file( + self.is_ssh, ssh, gather_package_dir, local_store_path, self.stdio + ) resp["error"] = "" resp["zip_password"] = zip_password else: @@ -325,7 +486,11 @@ def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_comman resp["zip_password"] = "" rm_rf_file(self.is_ssh, ssh, gather_package_dir, self.stdio) resp["gather_pack_path"] = local_path - self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ip, gather_package_dir)) + self.stdio.verbose( + "Collect pack gathered from node {0}: stored in {1}".format( + ip, gather_package_dir + ) + ) return resp @staticmethod @@ -352,10 +517,26 @@ def __get_overall_summary(node_summary_tuple, is_zip_encrypt): except: format_file_size = FileUtil.size_format(num=0, output_str=True) if is_zip_encrypt: - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, tup[4], "{0} s".format(int(consume_time)), pack_path)) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + tup[4], + "{0} s".format(int(consume_time)), + pack_path, + ) + ) else: - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) - return "\nGather ObProxy Log Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather ObProxy Log Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/gather_obstack2.py b/handler/gather/gather_obstack2.py index a8804ad7..44687623 100644 --- a/handler/gather/gather_obstack2.py +++ b/handler/gather/gather_obstack2.py @@ -22,7 +22,19 @@ import tabulate -from common.command import download_file, is_empty_dir, is_support_arch, get_observer_version, get_observer_pid, mkdir, zip_dir, get_file_size, delete_file_force, is_empty_file, upload_file +from common.command import ( + download_file, + is_empty_dir, + is_support_arch, + get_observer_version, + get_observer_pid, + mkdir, + zip_dir, + get_file_size, + delete_file_force, + is_empty_file, + upload_file, +) from common.constant import const from common.command import LocalClient, SshClient from handler.base_shell_handler import BaseShellHandler @@ -37,7 +49,7 @@ class GatherObstack2Handler(BaseShellHandler): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): super(GatherObstack2Handler, self).__init__() self.context = context self.stdio = context.stdio @@ -46,13 +58,13 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.remote_stored_path = None self.is_scene = is_scene self.config_path = const.DEFAULT_CONFIG_PATH - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.nodes, self.stdio) if new_nodes: self.nodes = new_nodes @@ -61,45 +73,61 @@ def init_config(self): self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] return True def init_option(self): options = self.context.options - store_dir_option = Util.get_option(options, 'store_dir') - if store_dir_option and store_dir_option != './': + store_dir_option = Util.get_option(options, "store_dir") + if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.local_stored_path = os.path.abspath(store_dir_option) return True def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False if self.is_scene: pack_dir_this_command = self.local_stored_path else: - pack_dir_this_command = os.path.join(self.local_stored_path, "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.local_stored_path, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] + def handle_from_node(node): st = time.time() resp = self.__handle_from_node(pack_dir_this_command, node) file_size = "" if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], - file_size, - int(time.time() - st), - resp["gather_pack_path"])) + gather_tuples.append( + ( + node.get("ip"), + False, + resp["error"], + file_size, + int(time.time() - st), + resp["gather_pack_path"], + ) + ) if self.is_ssh: for node in self.nodes: @@ -114,152 +142,277 @@ def handle_from_node(node): summary_tuples = self.__get_overall_summary(gather_tuples) self.stdio.print(summary_tuples) # Persist the summary results to a file - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) - last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) + last_info = ( + "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + os.path.join(pack_dir_this_command, "result_summary.txt") + ) + ) def __handle_from_node(self, local_stored_path, node): - resp = { - "skip": False, - "error": "", - "gather_pack_path": "" - } + resp = {"skip": False, "error": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") self.stdio.verbose( - "Sending Collect Shell Command to node {0} ...".format(remote_ip)) + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=local_stored_path, stdio=self.stdio) - now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") remote_dir_name = "obstack2_{0}_{1}".format(remote_ip, now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key,node, self.stdio) + ssh_helper = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.exception( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) if not ssh_failed: if not is_support_arch(self.is_ssh, ssh_helper, self.stdio): - resp["error"] = "remote server {0} arch not support gather obstack".format(ssh_helper.get_name()) + resp["error"] = ( + "remote server {0} arch not support gather obstack".format( + ssh_helper.get_name() + ) + ) return resp mkdir(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) # install and chmod obstack2 - ob_version = get_observer_version(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) - if not StringUtils.compare_versions_greater(ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK): - self.stdio.verbose("This version {0} does not support gather obstack . The minimum supported version is {1}". - format(ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK)) + ob_version = get_observer_version( + self.is_ssh, ssh_helper, node.get("home_path"), self.stdio + ) + if not StringUtils.compare_versions_greater( + ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK + ): + self.stdio.verbose( + "This version {0} does not support gather obstack . The minimum supported version is {1}".format( + ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK + ) + ) resp["error"] = "{0} not support gather obstack".format(ob_version) resp["gather_pack_path"] = "{0}".format(local_stored_path) return resp is_need_install_obstack = self.__is_obstack_exists(self.is_ssh, ssh_helper) if is_need_install_obstack: - self.stdio.verbose("There is no obstack2 on the host {0}. It needs to be installed. " - "Please wait a moment ...".format(remote_ip)) - if getattr(sys, 'frozen', False): + self.stdio.verbose( + "There is no obstack2 on the host {0}. It needs to be installed. " + "Please wait a moment ...".format(remote_ip) + ) + if getattr(sys, "frozen", False): absPath = os.path.dirname(sys.executable) else: - absPath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - obstack2_local_stored_full_path = os.path.join(absPath, const.OBSTACK2_LOCAL_STORED_PATH) - upload_file(self.is_ssh, ssh_helper, obstack2_local_stored_full_path, const.OBSTACK2_DEFAULT_INSTALL_PATH, self.stdio) - self.stdio.verbose("Installation of obstack2 is completed and gather begins ...") - + absPath = os.path.dirname( + os.path.dirname(os.path.dirname(__file__)) + ) + obstack2_local_stored_full_path = os.path.join( + absPath, const.OBSTACK2_LOCAL_STORED_PATH + ) + upload_file( + self.is_ssh, + ssh_helper, + obstack2_local_stored_full_path, + const.OBSTACK2_DEFAULT_INSTALL_PATH, + self.stdio, + ) + self.stdio.verbose( + "Installation of obstack2 is completed and gather begins ..." + ) + self.__chmod_obstack2(self.is_ssh, ssh_helper) # get observer_pid - observer_pid_list = get_observer_pid(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) + observer_pid_list = get_observer_pid( + self.is_ssh, ssh_helper, node.get("home_path"), self.stdio + ) # gather obstack2 info for observer_pid in observer_pid_list: user = self.__get_observer_execute_user(ssh_helper, observer_pid) - self.__gather_obstack2_info(self.is_ssh, ssh_helper, user, observer_pid, remote_dir_name,node) + self.__gather_obstack2_info( + self.is_ssh, ssh_helper, user, observer_pid, remote_dir_name, node + ) try: - self.stdio.start_loading('gather obstack info') + self.stdio.start_loading("gather obstack info") self.is_ready(ssh_helper, observer_pid, remote_dir_name) - self.stdio.stop_loading('gather obstack info sucess') + self.stdio.stop_loading("gather obstack info sucess") except: - self.stdio.stop_loading('gather info failed') - self.stdio.error("Gather obstack info on the host {0} observer pid {1}".format(remote_ip, observer_pid)) - delete_file_force(self.is_ssh, ssh_helper, "/tmp/{dir_name}/observer_{pid}_obstack.txt" - .format(dir_name=remote_dir_name, pid=observer_pid), self.stdio) + self.stdio.stop_loading("gather info failed") + self.stdio.error( + "Gather obstack info on the host {0} observer pid {1}".format( + remote_ip, observer_pid + ) + ) + delete_file_force( + self.is_ssh, + ssh_helper, + "/tmp/{dir_name}/observer_{pid}_obstack.txt".format( + dir_name=remote_dir_name, pid=observer_pid + ), + self.stdio, + ) pass - if is_empty_dir(self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio): + if is_empty_dir( + self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio + ): resp["error"] = "gather failed, folder is empty" return resp - + zip_dir(self.is_ssh, ssh_helper, "/tmp", remote_dir_name, self.stdio) remote_zip_file_path = "{0}.zip".format(remote_dir_full_path) - file_size = get_file_size(self.is_ssh, ssh_helper, remote_zip_file_path, self.stdio) + file_size = get_file_size( + self.is_ssh, ssh_helper, remote_zip_file_path, self.stdio + ) remote_file_full_path = "{0}.zip".format(remote_dir_full_path) if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - download_file(self.is_ssh, ssh_helper, remote_file_full_path, local_file_path, self.stdio) + local_file_path = "{0}/{1}.zip".format( + local_stored_path, remote_dir_name + ) + download_file( + self.is_ssh, + ssh_helper, + remote_file_full_path, + local_file_path, + self.stdio, + ) resp["error"] = "" else: resp["error"] = "File too large" - delete_file_force(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) + delete_file_force( + self.is_ssh, ssh_helper, remote_file_full_path, self.stdio + ) ssh_helper.ssh_close() - resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) + resp["gather_pack_path"] = "{0}/{1}.zip".format( + local_stored_path, remote_dir_name + ) return resp @Util.retry(5, 2) def is_ready(self, ssh_helper, pid, remote_dir_name): try: - self.stdio.verbose("Check whether the directory /tmp/{dir_name} or " - "file /tmp/{dir_name}/observer_{pid}_obstack.txt is empty" - .format(dir_name=remote_dir_name, pid=pid)) - is_empty_dir_res = is_empty_dir(self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio) - is_empty_file_res = is_empty_file(self.is_ssh, ssh_helper, "/tmp/{dir_name}/observer_{pid}_obstack.txt" - .format(dir_name=remote_dir_name, pid=pid), self.stdio) + self.stdio.verbose( + "Check whether the directory /tmp/{dir_name} or " + "file /tmp/{dir_name}/observer_{pid}_obstack.txt is empty".format( + dir_name=remote_dir_name, pid=pid + ) + ) + is_empty_dir_res = is_empty_dir( + self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio + ) + is_empty_file_res = is_empty_file( + self.is_ssh, + ssh_helper, + "/tmp/{dir_name}/observer_{pid}_obstack.txt".format( + dir_name=remote_dir_name, pid=pid + ), + self.stdio, + ) if is_empty_dir_res or is_empty_file_res: self.stdio.verbose( "The server {host_ip} directory /tmp/{dir_name} or file /tmp/{dir_name}/observer_{pid}_obstack.txt" - " is empty, waiting for the collection to complete" - .format(host_ip=ssh_helper.get_name() if self.is_ssh else NetUtils.get_inner_ip(self.stdio), dir_name=remote_dir_name, pid=pid)) + " is empty, waiting for the collection to complete".format( + host_ip=( + ssh_helper.get_name() + if self.is_ssh + else NetUtils.get_inner_ip(self.stdio) + ), + dir_name=remote_dir_name, + pid=pid, + ) + ) raise except Exception as e: raise e def __chmod_obstack2(self, is_ssh, ssh_helper): cmd = "chmod a+x {file}".format(file=const.OBSTACK2_DEFAULT_INSTALL_PATH) - SshClient(self.stdio).run(ssh_helper, cmd) if is_ssh else LocalClient(self.stdio).run(cmd) + ( + SshClient(self.stdio).run(ssh_helper, cmd) + if is_ssh + else LocalClient(self.stdio).run(cmd) + ) def __is_obstack_exists(self, is_ssh, ssh_helper): - cmd = "test -e {file} && echo exists".format(file=const.OBSTACK2_DEFAULT_INSTALL_PATH) - stdout = SshClient(self.stdio).run(ssh_helper, cmd) if is_ssh else LocalClient(self.stdio).run(cmd) - if stdout == 'exists': + cmd = "test -e {file} && echo exists".format( + file=const.OBSTACK2_DEFAULT_INSTALL_PATH + ) + stdout = ( + SshClient(self.stdio).run(ssh_helper, cmd) + if is_ssh + else LocalClient(self.stdio).run(cmd) + ) + if stdout == "exists": return False else: return True def __get_observer_execute_user(self, ssh_helper, pid): - cmd = "ps -o ruser=userForLongName -e -o pid,ppid,c,stime,tty,time,cmd | grep observer | grep {0} | awk {1}".format(pid, "'{print $1}'") - stdout = SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + cmd = "ps -o ruser=userForLongName -e -o pid,ppid,c,stime,tty,time,cmd | grep observer | grep {0} | awk {1}".format( + pid, "'{print $1}'" + ) + stdout = ( + SshClient(self.stdio).run(ssh_helper, cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cmd) + ) user = stdout.splitlines()[0] - self.stdio.verbose("get observer execute user, run cmd = [{0}], result:{1} ".format(cmd, user)) + self.stdio.verbose( + "get observer execute user, run cmd = [{0}], result:{1} ".format(cmd, user) + ) return user - def __gather_obstack2_info(self, is_ssh, ssh_helper, user, observer_pid, remote_gather_dir,node): + def __gather_obstack2_info( + self, is_ssh, ssh_helper, user, observer_pid, remote_gather_dir, node + ): cmd = "{obstack} {pid} > /tmp/{gather_dir}/observer_{pid}_obstack.txt".format( obstack=const.OBSTACK2_DEFAULT_INSTALL_PATH, pid=observer_pid, - gather_dir=remote_gather_dir) + gather_dir=remote_gather_dir, + ) if is_ssh: if user == ssh_helper.username: - self.stdio.verbose("gather obstack info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), cmd)) + self.stdio.verbose( + "gather obstack info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), cmd + ) + ) SshClient(self.stdio).run_ignore_err(ssh_helper, cmd) else: - ssh_helper_new = SshHelper(ssh_helper.host_ip, ssh_helper.username, ssh_helper.password, ssh_helper.ssh_port, ssh_helper.key_file,node) - chown_cmd = "chown {user} /tmp/{gather_dir}/".format(user=user,gather_dir=remote_gather_dir) + ssh_helper_new = SshHelper( + ssh_helper.host_ip, + ssh_helper.username, + ssh_helper.password, + ssh_helper.ssh_port, + ssh_helper.key_file, + node, + ) + chown_cmd = "chown {user} /tmp/{gather_dir}/".format( + user=user, gather_dir=remote_gather_dir + ) SshClient(self.stdio).run(ssh_helper_new, chown_cmd) - self.stdio.verbose("gather obstack info on server {0}, run cmd = [su {1}, {2}]".format(ssh_helper.get_name(), user, cmd)) + self.stdio.verbose( + "gather obstack info on server {0}, run cmd = [su {1}, {2}]".format( + ssh_helper.get_name(), user, cmd + ) + ) ssh_helper_new.ssh_invoke_shell_switch_user(user, cmd, 10) else: LocalClient(self.stdio).run(cmd) @@ -278,7 +431,15 @@ def __get_overall_summary(node_summary_tuple): format_file_size = FileUtil.size_format(num=file_size, output_str=True) except: format_file_size = FileUtil.size_format(num=0, output_str=True) - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) - return "\nGather Ob stack Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather Ob stack Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/gather_perf.py b/handler/gather/gather_perf.py index 79e7a996..a8b8c563 100644 --- a/handler/gather/gather_perf.py +++ b/handler/gather/gather_perf.py @@ -21,7 +21,14 @@ import tabulate -from common.command import get_observer_pid, mkdir, zip_dir, get_file_size, download_file, delete_file_force +from common.command import ( + get_observer_pid, + mkdir, + zip_dir, + get_file_size, + download_file, + delete_file_force, +) from common.command import LocalClient, SshClient from common.constant import const from handler.base_shell_handler import BaseShellHandler @@ -34,7 +41,7 @@ class GatherPerfHandler(BaseShellHandler): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): super(GatherPerfHandler, self).__init__() self.context = context self.stdio = context.stdio @@ -45,13 +52,13 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.is_scene = is_scene self.scope = "all" self.config_path = const.DEFAULT_CONFIG_PATH - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.nodes, self.stdio) if new_nodes: self.nodes = new_nodes @@ -60,34 +67,43 @@ def init_config(self): self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] return True def init_option(self): options = self.context.options - store_dir_option = Util.get_option(options, 'store_dir') - if store_dir_option and store_dir_option != './': + store_dir_option = Util.get_option(options, "store_dir") + if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.local_stored_path = os.path.abspath(store_dir_option) - self.scope_option = Util.get_option(options, 'scope') + self.scope_option = Util.get_option(options, "scope") return True def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False if self.is_scene: pack_dir_this_command = self.local_stored_path else: - pack_dir_this_command = os.path.join(self.local_stored_path,"gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.local_stored_path, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] @@ -97,10 +113,16 @@ def handle_from_node(node): file_size = "" if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], - file_size, - int(time.time() - st), - resp["gather_pack_path"])) + gather_tuples.append( + ( + node.get("ip"), + False, + resp["error"], + file_size, + int(time.time() - st), + resp["gather_pack_path"], + ) + ) if self.is_ssh: for node in self.nodes: @@ -115,104 +137,182 @@ def handle_from_node(node): summary_tuples = self.__get_overall_summary(gather_tuples) self.stdio.print(summary_tuples) # Persist the summary results to a file - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) - last_info = "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(pack_dir_this_command, "result_summary.txt")) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) + last_info = ( + "For result details, please run cmd \033[32m' cat {0} '\033[0m\n".format( + os.path.join(pack_dir_this_command, "result_summary.txt") + ) + ) def __handle_from_node(self, node, local_stored_path): - resp = { - "skip": False, - "error": "", - "gather_pack_path": "" - } + resp = {"skip": False, "error": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip(self.stdio) remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") - self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) + self.stdio.verbose( + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=local_stored_path, stdio=self.stdio) - now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") remote_dir_name = "perf_{0}_{1}".format(node.get("ip"), now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, - remote_private_key, node, self.stdio) + ssh_helper = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.exception( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) if not ssh_failed: mkdir(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) - pid_observer_list = get_observer_pid(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) + pid_observer_list = get_observer_pid( + self.is_ssh, ssh_helper, node.get("home_path"), self.stdio + ) if len(pid_observer_list) == 0: resp["error"] = "can't find observer" return resp for pid_observer in pid_observer_list: if self.scope == "sample": - self.__gather_perf_sample(ssh_helper, remote_dir_full_path, pid_observer) + self.__gather_perf_sample( + ssh_helper, remote_dir_full_path, pid_observer + ) elif self.scope == "flame": - self.__gather_perf_flame(ssh_helper, remote_dir_full_path, pid_observer) + self.__gather_perf_flame( + ssh_helper, remote_dir_full_path, pid_observer + ) else: - self.__gather_perf_sample(ssh_helper, remote_dir_full_path, pid_observer) - self.__gather_perf_flame(ssh_helper, remote_dir_full_path, pid_observer) + self.__gather_perf_sample( + ssh_helper, remote_dir_full_path, pid_observer + ) + self.__gather_perf_flame( + ssh_helper, remote_dir_full_path, pid_observer + ) self.__gather_top(ssh_helper, remote_dir_full_path, pid_observer) zip_dir(self.is_ssh, ssh_helper, "/tmp", remote_dir_name, self.stdio) remote_file_full_path = "{0}.zip".format(remote_dir_full_path) - file_size = get_file_size(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) + file_size = get_file_size( + self.is_ssh, ssh_helper, remote_file_full_path, self.stdio + ) if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - download_file(self.is_ssh,ssh_helper, remote_file_full_path, local_file_path, self.stdio) + local_file_path = "{0}/{1}.zip".format( + local_stored_path, remote_dir_name + ) + download_file( + self.is_ssh, + ssh_helper, + remote_file_full_path, + local_file_path, + self.stdio, + ) resp["error"] = "" else: resp["error"] = "File too large" - delete_file_force(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) + delete_file_force( + self.is_ssh, ssh_helper, remote_file_full_path, self.stdio + ) ssh_helper.ssh_close() - resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) + resp["gather_pack_path"] = "{0}/{1}.zip".format( + local_stored_path, remote_dir_name + ) return resp def __gather_perf_sample(self, ssh_helper, gather_path, pid_observer): try: cmd = "cd {gather_path} && perf record -o sample.data -e cycles -c 100000000 -p {pid} -g -- sleep 20".format( - gather_path=gather_path, pid=pid_observer) + gather_path=gather_path, pid=pid_observer + ) self.stdio.verbose("gather perf sample, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run_ignore_err(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ( + SshClient(self.stdio).run_ignore_err(ssh_helper, cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cmd) + ) generate_data = "cd {gather_path} && perf script -i sample.data -F ip,sym -f > sample.viz".format( - gather_path=gather_path) - self.stdio.verbose("generate perf sample data, run cmd = [{0}]".format(generate_data)) - SshClient(self.stdio).run_ignore_err(ssh_helper, generate_data) if self.is_ssh else LocalClient(self.stdio).run(generate_data) + gather_path=gather_path + ) + self.stdio.verbose( + "generate perf sample data, run cmd = [{0}]".format(generate_data) + ) + ( + SshClient(self.stdio).run_ignore_err(ssh_helper, generate_data) + if self.is_ssh + else LocalClient(self.stdio).run(generate_data) + ) except: - self.stdio.error("generate perf sample data on server [{0}] failed".format(ssh_helper.get_name())) + self.stdio.error( + "generate perf sample data on server [{0}] failed".format( + ssh_helper.get_name() + ) + ) def __gather_perf_flame(self, ssh_helper, gather_path, pid_observer): try: perf_cmd = "cd {gather_path} && perf record -o flame.data -F 99 -p {pid} -g -- sleep 20".format( - gather_path=gather_path, pid=pid_observer) + gather_path=gather_path, pid=pid_observer + ) self.stdio.verbose("gather perf, run cmd = [{0}]".format(perf_cmd)) - SshClient(self.stdio).run_ignore_err(ssh_helper, perf_cmd) if self.is_ssh else LocalClient(self.stdio).run(perf_cmd) + ( + SshClient(self.stdio).run_ignore_err(ssh_helper, perf_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(perf_cmd) + ) - generate_data = "cd {gather_path} && perf script -i flame.data > flame.viz".format( - gather_path=gather_path) - self.stdio.verbose("generate perf data, run cmd = [{0}]".format(generate_data)) - SshClient(self.stdio).run_ignore_err(ssh_helper, generate_data) if self.is_ssh else LocalClient(self.stdio).run(generate_data) + generate_data = ( + "cd {gather_path} && perf script -i flame.data > flame.viz".format( + gather_path=gather_path + ) + ) + self.stdio.verbose( + "generate perf data, run cmd = [{0}]".format(generate_data) + ) + ( + SshClient(self.stdio).run_ignore_err(ssh_helper, generate_data) + if self.is_ssh + else LocalClient(self.stdio).run(generate_data) + ) except: - self.stdio.error("generate perf data on server [{0}] failed".format(ssh_helper.get_name())) + self.stdio.error( + "generate perf data on server [{0}] failed".format( + ssh_helper.get_name() + ) + ) def __gather_top(self, ssh_helper, gather_path, pid_observer): try: cmd = "cd {gather_path} && top -Hp {pid} -b -n 1 > top.txt".format( - gather_path=gather_path, pid=pid_observer) + gather_path=gather_path, pid=pid_observer + ) self.stdio.verbose("gather top, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ( + SshClient(self.stdio).run(ssh_helper, cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cmd) + ) except: - self.stdio.error("gather top on server failed [{0}]".format(ssh_helper.get_name())) + self.stdio.error( + "gather top on server failed [{0}]".format(ssh_helper.get_name()) + ) @staticmethod def __get_overall_summary(node_summary_tuple): @@ -228,7 +328,15 @@ def __get_overall_summary(node_summary_tuple): format_file_size = FileUtil.size_format(num=file_size, output_str=True) except: format_file_size = FileUtil.size_format(num=0, output_str=True) - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) - return "\nGather Perf Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather Perf Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/gather_plan_monitor.py b/handler/gather/gather_plan_monitor.py index ac7b71c2..24acd66a 100644 --- a/handler/gather/gather_plan_monitor.py +++ b/handler/gather/gather_plan_monitor.py @@ -35,7 +35,7 @@ class GatherPlanMonitorHandler(object): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): self.context = context self.stdio = context.stdio self.ob_cluster = None @@ -53,33 +53,44 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.sql_audit_name = "gv$sql_audit" self.plan_explain_name = "gv$plan_cache_plan_explain" self.is_scene = is_scene - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): ob_cluster = self.context.cluster_config self.ob_cluster = ob_cluster - self.sys_connector = OBConnector(ip=ob_cluster.get("db_host"), port=ob_cluster.get("db_port"), username=ob_cluster.get("tenant_sys").get("user"), password=ob_cluster.get("tenant_sys").get("password"), stdio=self.stdio, timeout=100) + self.sys_connector = OBConnector( + ip=ob_cluster.get("db_host"), + port=ob_cluster.get("db_port"), + username=ob_cluster.get("tenant_sys").get("user"), + password=ob_cluster.get("tenant_sys").get("password"), + stdio=self.stdio, + timeout=100, + ) self.ob_cluster_name = ob_cluster.get("ob_cluster_name") return True def init_option(self): options = self.context.options - trace_id_option = Util.get_option(options, 'trace_id') - store_dir_option = Util.get_option(options, 'store_dir') - env_option = Util.get_option(options, 'env') - if self.context.get_variable("gather_plan_monitor_trace_id", None) : - trace_id_option=self.context.get_variable("gather_plan_monitor_trace_id") + trace_id_option = Util.get_option(options, "trace_id") + store_dir_option = Util.get_option(options, "store_dir") + env_option = Util.get_option(options, "env") + if self.context.get_variable("gather_plan_monitor_trace_id", None): + trace_id_option = self.context.get_variable("gather_plan_monitor_trace_id") if trace_id_option is not None: self.trace_id = trace_id_option else: self.stdio.error("option --trace_id not found, please provide") return False - if store_dir_option and store_dir_option != './': + if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: option --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: option --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.local_stored_path = os.path.abspath(store_dir_option) if env_option is not None: @@ -89,21 +100,35 @@ def init_option(self): return self.tenant_mode_detected() def __init_db_connector(self): - self.db_connector = OBConnector(ip=self.db_conn.get("host"), port=self.db_conn.get("port"), username=self.db_conn.get("user"), password=self.db_conn.get("password"), database=self.db_conn.get("database"), stdio=self.stdio, timeout=100) + self.db_connector = OBConnector( + ip=self.db_conn.get("host"), + port=self.db_conn.get("port"), + username=self.db_conn.get("user"), + password=self.db_conn.get("password"), + database=self.db_conn.get("database"), + stdio=self.stdio, + timeout=100, + ) def handle(self): if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if self.is_scene: pack_dir_this_command = self.local_stored_path else: - pack_dir_this_command = os.path.join(self.local_stored_path, "gather_pack_{0}".format( - TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) - self.report_file_path = os.path.join(pack_dir_this_command, "sql_plan_monitor_report.html") + pack_dir_this_command = os.path.join( + self.local_stored_path, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) + self.report_file_path = os.path.join( + pack_dir_this_command, "sql_plan_monitor_report.html" + ) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) gather_tuples = [] @@ -116,7 +141,9 @@ def handle_plan_monitor_from_ob(cluster_name): """ st = time.time() resp = self.init_resp() - result_sql_audit_by_trace_id_limit1 = self.select_sql_audit_by_trace_id_limit1() + result_sql_audit_by_trace_id_limit1 = ( + self.select_sql_audit_by_trace_id_limit1() + ) if len(result_sql_audit_by_trace_id_limit1) > 0: trace = result_sql_audit_by_trace_id_limit1[0] trace_id = trace[0] @@ -135,25 +162,52 @@ def handle_plan_monitor_from_ob(cluster_name): self.stdio.verbose("PLAN_ID: %s " % plan_id) self.stdio.verbose("TENANT_ID: %s " % tenant_id) - sql_plan_monitor_svr_agg_template = self.sql_plan_monitor_svr_agg_template_sql() - sql_plan_monitor_svr_agg_v1 = str(sql_plan_monitor_svr_agg_template) \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_ORDER_BY##", "PLAN_LINE_ID ASC, MAX_CHANGE_TIME ASC, SVR_IP, SVR_PORT") - sql_plan_monitor_svr_agg_v2 = str(sql_plan_monitor_svr_agg_template) \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ + sql_plan_monitor_svr_agg_template = ( + self.sql_plan_monitor_svr_agg_template_sql() + ) + sql_plan_monitor_svr_agg_v1 = ( + str(sql_plan_monitor_svr_agg_template) + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace( + "##REPLACE_ORDER_BY##", + "PLAN_LINE_ID ASC, MAX_CHANGE_TIME ASC, SVR_IP, SVR_PORT", + ) + ) + sql_plan_monitor_svr_agg_v2 = ( + str(sql_plan_monitor_svr_agg_template) + .replace("##REPLACE_TRACE_ID##", trace_id) .replace("##REPLACE_ORDER_BY##", "SVR_IP, SVR_PORT, PLAN_LINE_ID") + ) - sql_plan_monitor_detail_template = self.sql_plan_monitor_detail_template_sql() - sql_plan_monitor_detail_v1 = str(sql_plan_monitor_detail_template) \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_ORDER_BY##", "PLAN_LINE_ID ASC, SVR_IP, SVR_PORT, CHANGE_TS, PROCESS_NAME ASC") - sql_plan_monitor_detail_v2 = str(sql_plan_monitor_detail_template) \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_ORDER_BY##", "PROCESS_NAME ASC, PLAN_LINE_ID ASC, FIRST_REFRESH_TIME ASC") + sql_plan_monitor_detail_template = ( + self.sql_plan_monitor_detail_template_sql() + ) + sql_plan_monitor_detail_v1 = ( + str(sql_plan_monitor_detail_template) + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace( + "##REPLACE_ORDER_BY##", + "PLAN_LINE_ID ASC, SVR_IP, SVR_PORT, CHANGE_TS, PROCESS_NAME ASC", + ) + ) + sql_plan_monitor_detail_v2 = ( + str(sql_plan_monitor_detail_template) + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace( + "##REPLACE_ORDER_BY##", + "PROCESS_NAME ASC, PLAN_LINE_ID ASC, FIRST_REFRESH_TIME ASC", + ) + ) - sql_plan_monitor_dfo_op = self.sql_plan_monitor_dfo_op_sql(tenant_id, plan_id, trace_id) - full_audit_sql_by_trace_id_sql = self.full_audit_sql_by_trace_id_sql(trace_id) - plan_explain_sql = self.plan_explain_sql(tenant_id, plan_id, svr_ip, svr_port) + sql_plan_monitor_dfo_op = self.sql_plan_monitor_dfo_op_sql( + tenant_id, plan_id, trace_id + ) + full_audit_sql_by_trace_id_sql = self.full_audit_sql_by_trace_id_sql( + trace_id + ) + plan_explain_sql = self.plan_explain_sql( + tenant_id, plan_id, svr_ip, svr_port + ) # 输出报告头 self.stdio.verbose("[sql plan monitor report task] report header") @@ -162,7 +216,11 @@ def handle_plan_monitor_from_ob(cluster_name): self.stdio.verbose("[sql plan monitor report task] report sql_audit") self.report_sql_audit() # 输出sql explain的信息 - self.stdio.verbose("[sql plan monitor report task] report plan explain, sql: [{0}]".format(sql)) + self.stdio.verbose( + "[sql plan monitor report task] report plan explain, sql: [{0}]".format( + sql + ) + ) self.report_plan_explain(db_name, sql) # 输出plan cache的信息 self.stdio.verbose("[sql plan monitor report task] report plan cache") @@ -172,47 +230,82 @@ def handle_plan_monitor_from_ob(cluster_name): self.report_schema(user_sql) self.init_monitor_stat() # 输出sql_audit的详细信息 - self.stdio.verbose("[sql plan monitor report task] report sql_audit details") + self.stdio.verbose( + "[sql plan monitor report task] report sql_audit details" + ) self.report_sql_audit_details(full_audit_sql_by_trace_id_sql) # 输出算子信息 表+图 - self.stdio.verbose("[sql plan monitor report task] report sql plan monitor dfo") + self.stdio.verbose( + "[sql plan monitor report task] report sql plan monitor dfo" + ) self.report_sql_plan_monitor_dfo_op(sql_plan_monitor_dfo_op) # 输出算子信息按 svr 级汇总 表+图 - self.stdio.verbose("[sql plan monitor report task] report sql plan monitor group by server") - self.report_sql_plan_monitor_svr_agg(sql_plan_monitor_svr_agg_v1, sql_plan_monitor_svr_agg_v2) + self.stdio.verbose( + "[sql plan monitor report task] report sql plan monitor group by server" + ) + self.report_sql_plan_monitor_svr_agg( + sql_plan_monitor_svr_agg_v1, sql_plan_monitor_svr_agg_v2 + ) self.report_fast_preview() # 输出算子信息按算子维度聚集 - self.stdio.verbose("[sql plan monitor report task] sql plan monitor detail operator") - self.report_sql_plan_monitor_detail_operator_priority(sql_plan_monitor_detail_v1) + self.stdio.verbose( + "[sql plan monitor report task] sql plan monitor detail operator" + ) + self.report_sql_plan_monitor_detail_operator_priority( + sql_plan_monitor_detail_v1 + ) # 输出算子信息按线程维度聚集 - self.stdio.verbose("[sql plan monitor report task] sql plan monitor group by priority") - self.reportsql_plan_monitor_detail_svr_priority(sql_plan_monitor_detail_v2) + self.stdio.verbose( + "[sql plan monitor report task] sql plan monitor group by priority" + ) + self.reportsql_plan_monitor_detail_svr_priority( + sql_plan_monitor_detail_v2 + ) # 输出本报告在租户下使用的 SQL self.__report("

本报告在租户下使用的 SQL

") - self.__report("
DFO 级
%s

" % ( - sql_plan_monitor_dfo_op)) - self.__report("
机器级
%s

" % ( - sql_plan_monitor_svr_agg_v1)) - self.__report("
线程级
%s

" % ( - sql_plan_monitor_detail_v1)) + self.__report( + "
DFO 级
%s

" + % (sql_plan_monitor_dfo_op) + ) + self.__report( + "
机器级
%s

" + % (sql_plan_monitor_svr_agg_v1) + ) + self.__report( + "
线程级
%s

" + % (sql_plan_monitor_detail_v1) + ) t = time.localtime(time.time()) - self.__report("报告生成时间： %s" % (time.strftime("%Y-%m-%d %H:%M:%S", t))) + self.__report( + "报告生成时间： %s" % (time.strftime("%Y-%m-%d %H:%M:%S", t)) + ) self.report_footer() self.stdio.verbose("report footer complete") if resp["skip"]: return if resp["error"]: - gather_tuples.append((cluster_name, True, resp["error_msg"], 0, int(time.time() - st), - "Error:{0}".format(resp["error_msg"]), "")) + gather_tuples.append( + ( + cluster_name, + True, + resp["error_msg"], + 0, + int(time.time() - st), + "Error:{0}".format(resp["error_msg"]), + "", + ) + ) return gather_pack_path_dict[cluster_name] = resp["gather_pack_path"] - gather_tuples.append((cluster_name, False, "", int(time.time() - st), pack_dir_this_command)) + gather_tuples.append( + (cluster_name, False, "", int(time.time() - st), pack_dir_this_command) + ) - if getattr(sys, 'frozen', False): - absPath = os.path.dirname(sys.executable) + if getattr(sys, "frozen", False): + absPath = os.path.dirname(sys.executable) else: absPath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) cs_resources_path = os.path.join(absPath, "resources") @@ -225,10 +318,11 @@ def handle_plan_monitor_from_ob(cluster_name): summary_tuples = self.__get_overall_summary(gather_tuples) self.stdio.print(summary_tuples) # 将汇总结果持久化记录到文件中 - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) return gather_tuples, gather_pack_path_dict - def __init_db_conn(self, env): try: env_dict = StringUtils.parse_env(env) @@ -238,11 +332,17 @@ def __init_db_conn(self, env): if StringUtils.validate_db_info(self.db_conn): self.__init_db_connector() else: - self.stdio.error("db connection information requird [db_connect = '-hxx -Pxx -uxx -pxx -Dxx'] but provided {0}, please check the --env {0}".format(env_dict)) + self.stdio.error( + "db connection information requird [db_connect = '-hxx -Pxx -uxx -pxx -Dxx'] but provided {0}, please check the --env {0}".format( + env_dict + ) + ) self.db_connector = self.sys_connector except Exception as e: self.db_connector = self.sys_connector - self.stdio.exception("init db connector, error: {0}, please check --env option ") + self.stdio.exception( + "init db connector, error: {0}, please check --env option " + ) @staticmethod def __get_overall_summary(node_summary_tuple): @@ -259,17 +359,41 @@ def __get_overall_summary(node_summary_tuple): consume_time = tup[3] pack_path = tup[4] summary_tab.append( - (cluster, "Error" if is_err else "Completed", "{0} s".format(int(consume_time)), pack_path)) - return "\nGather Sql Plan Monitor Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + ( + cluster, + "Error" if is_err else "Completed", + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather Sql Plan Monitor Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) def report_schema(self, sql): try: schemas = "" valid_words = [] if self.enable_dump_db: - words = [w.strip(',') for w in ("%s" % sql).split() if not ( - "[" in w or "=" in w or "|" in w or "(" in w or "--" in w or "]" in w or ")" in w or "*" in w or "/" in w or "%" in w or "'" in w or "-" in w or w.isdigit())] + words = [ + w.strip(",") + for w in ("%s" % sql).split() + if not ( + "[" in w + or "=" in w + or "|" in w + or "(" in w + or "--" in w + or "]" in w + or ")" in w + or "*" in w + or "/" in w + or "%" in w + or "'" in w + or "-" in w + or w.isdigit() + ) + ] for t in words: if t in valid_words: continue @@ -277,26 +401,42 @@ def report_schema(self, sql): for t in valid_words: try: data = self.db_connector.execute_sql("show create table %s" % t) - schemas = schemas + "
{0}
".format(data[0][1]) + schemas = ( + schemas + + "
{0}
".format( + data[0][1] + ) + ) self.stdio.verbose("table schema: {0}".format(schemas)) except Exception as e: pass - cursor = self.sys_connector.execute_sql_return_cursor("show variables like '%parallel%'") + cursor = self.sys_connector.execute_sql_return_cursor( + "show variables like '%parallel%'" + ) s = from_db_cursor(cursor) - s.align = 'l' - schemas = schemas + "
%s
" % s + s.align = "l" + schemas = ( + schemas + "
%s
" % s + ) cursor.execute("show variables") s = from_db_cursor(cursor) - s.align = 'l' - schemas = schemas + "
%s
" % s + s.align = "l" + schemas = ( + schemas + "
%s
" % s + ) cursor.execute("show parameters") s = from_db_cursor(cursor) - s.align = 'l' - schemas = schemas + "
%s
" % s + s.align = "l" + schemas = ( + schemas + "
%s
" % s + ) self.__report( - "

SCHEMA 信息

") + "

SCHEMA 信息

" + ) cursor.close() except Exception as e: self.stdio.exception("report table schema failed %s" % sql) @@ -304,18 +444,21 @@ def report_schema(self, sql): pass def report_pre(self, s): - pre = f'''
{s}
''' + pre = f"""
{s}
""" self.__report(pre) def report_header(self): header = GlobalHtmlMeta().get_value(key="sql_plan_monitor_report_header") - with open(self.report_file_path, 'w') as f: + with open(self.report_file_path, "w") as f: f.write(header) self.stdio.verbose("report header complete") def init_monitor_stat(self): - sql = "select ID,NAME,TYPE from " + ( - "SYS." if self.tenant_mode == "oracle" else "oceanbase.") + "v$sql_monitor_statname order by ID" + sql = ( + "select ID,NAME,TYPE from " + + ("SYS." if self.tenant_mode == "oracle" else "oceanbase.") + + "v$sql_monitor_statname order by ID" + ) data = self.sys_connector.execute_sql(sql) for item in data: self.STAT_NAME[item[0]] = {"type": item[2], "name": item[1]} @@ -330,13 +473,21 @@ def otherstat_detail_explain_item(self, item, n, v): elif self.STAT_NAME[item[n]]["type"] == 2: val = "%0.3fMB" % (item[n + 1] / 1024.0 / 1024) elif self.STAT_NAME[item[n]]["type"] == 3: - val = "%s.%06d" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(item[v] / 1000000)), - item[v] - (item[v] / 1000000) * 1000000) + val = "%s.%06d" % ( + time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(item[v] / 1000000) + ), + item[v] - (item[v] / 1000000) * 1000000, + ) else: val = str(item[v]) except Exception as e: val = str(item[v]) - return "" if item[n] == 0 else self.STAT_NAME[item[n]]["name"] + "(" + val + ");
" + return ( + "" + if item[n] == 0 + else self.STAT_NAME[item[n]]["name"] + "(" + val + ");
" + ) def otherstat_agg_explain_item(self, item, n, v): try: @@ -350,193 +501,406 @@ def otherstat_agg_explain_item(self, item, n, v): val = None except Exception as e: val = str(item[v]) - return "" if val == None else self.STAT_NAME[item[n]]["name"] + "(" + val + ");
" + return ( + "" + if val == None + else self.STAT_NAME[item[n]]["name"] + "(" + val + ");
" + ) def detail_otherstat_explain(self, item): otherstat = "" - otherstat += self.otherstat_detail_explain_item(item, "OTHERSTAT_1_ID", "OTHERSTAT_1_VALUE") - otherstat += self.otherstat_detail_explain_item(item, "OTHERSTAT_2_ID", "OTHERSTAT_2_VALUE") - otherstat += self.otherstat_detail_explain_item(item, "OTHERSTAT_3_ID", "OTHERSTAT_3_VALUE") - otherstat += self.otherstat_detail_explain_item(item, "OTHERSTAT_4_ID", "OTHERSTAT_4_VALUE") - otherstat += self.otherstat_detail_explain_item(item, "OTHERSTAT_5_ID", "OTHERSTAT_5_VALUE") - otherstat += self.otherstat_detail_explain_item(item, "OTHERSTAT_6_ID", "OTHERSTAT_6_VALUE") + otherstat += self.otherstat_detail_explain_item( + item, "OTHERSTAT_1_ID", "OTHERSTAT_1_VALUE" + ) + otherstat += self.otherstat_detail_explain_item( + item, "OTHERSTAT_2_ID", "OTHERSTAT_2_VALUE" + ) + otherstat += self.otherstat_detail_explain_item( + item, "OTHERSTAT_3_ID", "OTHERSTAT_3_VALUE" + ) + otherstat += self.otherstat_detail_explain_item( + item, "OTHERSTAT_4_ID", "OTHERSTAT_4_VALUE" + ) + otherstat += self.otherstat_detail_explain_item( + item, "OTHERSTAT_5_ID", "OTHERSTAT_5_VALUE" + ) + otherstat += self.otherstat_detail_explain_item( + item, "OTHERSTAT_6_ID", "OTHERSTAT_6_VALUE" + ) return otherstat def dfo_otherstat_explain(self, item): otherstat = "" - otherstat += self.otherstat_agg_explain_item(item, "OTHERSTAT_1_ID", "SUM_STAT_1") - otherstat += self.otherstat_agg_explain_item(item, "OTHERSTAT_2_ID", "SUM_STAT_2") - otherstat += self.otherstat_agg_explain_item(item, "OTHERSTAT_3_ID", "SUM_STAT_3") - otherstat += self.otherstat_agg_explain_item(item, "OTHERSTAT_4_ID", "SUM_STAT_4") - otherstat += self.otherstat_agg_explain_item(item, "OTHERSTAT_5_ID", "SUM_STAT_5") - otherstat += self.otherstat_agg_explain_item(item, "OTHERSTAT_6_ID", "SUM_STAT_6") + otherstat += self.otherstat_agg_explain_item( + item, "OTHERSTAT_1_ID", "SUM_STAT_1" + ) + otherstat += self.otherstat_agg_explain_item( + item, "OTHERSTAT_2_ID", "SUM_STAT_2" + ) + otherstat += self.otherstat_agg_explain_item( + item, "OTHERSTAT_3_ID", "SUM_STAT_3" + ) + otherstat += self.otherstat_agg_explain_item( + item, "OTHERSTAT_4_ID", "SUM_STAT_4" + ) + otherstat += self.otherstat_agg_explain_item( + item, "OTHERSTAT_5_ID", "SUM_STAT_5" + ) + otherstat += self.otherstat_agg_explain_item( + item, "OTHERSTAT_6_ID", "SUM_STAT_6" + ) return otherstat - def report_detail_graph_data(self, ident, cursor, title=''): + def report_detail_graph_data(self, ident, cursor, title=""): data = "" data = data + "

%s

" % (title, ident) self.__report(data) - def report_detail_graph_data_obversion4(self, ident, cursor, title=''): + def report_detail_graph_data_obversion4(self, ident, cursor, title=""): data = "" data = data + "

%s

" % (title, ident) self.__report(data) # dfo db time - def report_dfo_agg_db_time_graph_data_obversion4(self, cursor, title=''): + def report_dfo_agg_db_time_graph_data_obversion4(self, cursor, title=""): data = "

%s

" % (title); + data = ( + data + + "

%s

" % (title) + ) self.__report(data) - def report_dfo_agg_graph_data(self, cursor, title=''): + def report_dfo_agg_graph_data(self, cursor, title=""): data = "

%s

" % (title) + data = data + "

%s

" % ( + title + ) self.__report(data) - def report_dfo_agg_graph_data_obversion4(self, cursor, title=''): + def report_dfo_agg_graph_data_obversion4(self, cursor, title=""): data = "

%s

" % (title) + data = data + "

%s

" % ( + title + ) self.__report(data) - def report_dfo_sched_agg_graph_data(self, cursor, title=''): + def report_dfo_sched_agg_graph_data(self, cursor, title=""): data = "

%s

" % (title) + data = ( + data + + "

%s

" + % (title) + ) self.__report(data) - def report_dfo_sched_agg_graph_data_obversion4(self, cursor, title=''): + def report_dfo_sched_agg_graph_data_obversion4(self, cursor, title=""): data = "

%s

" % (title) + data = ( + data + + "

%s

" + % (title) + ) self.__report(data) # sqc，辅助查询协调者 - def report_svr_agg_graph_data(self, ident, cursor, title=''): + def report_svr_agg_graph_data(self, ident, cursor, title=""): data = "" data = data + "

%s

" % (title, ident) - self.stdio.verbose("report SQL_PLAN_MONITOR SQC operator priority start, DATA: %s", data) + self.stdio.verbose( + "report SQL_PLAN_MONITOR SQC operator priority start, DATA: %s", data + ) self.__report(data) - def report_svr_agg_graph_data_obversion4(self, ident, cursor, title=''): + def report_svr_agg_graph_data_obversion4(self, ident, cursor, title=""): data = "" data = data + "

%s

" % (title, ident) - self.stdio.verbose("report SQL_PLAN_MONITOR SQC operator priority start, DATA: %s", data) + self.stdio.verbose( + "report SQL_PLAN_MONITOR SQC operator priority start, DATA: %s", data + ) self.__report(data) def report_fast_preview(self): - content = ''' + content = """ - ''' + """ self.__report(content) self.stdio.verbose("report SQL_PLAN_MONITOR fast preview complete") @@ -545,54 +909,68 @@ def report_footer(self): self.__report(footer) def __report(self, s): - with open(self.report_file_path, 'a') as f: + with open(self.report_file_path, "a") as f: f.write(s) def tenant_mode_detected(self): try: # Detect MySQL mode - data = self.db_connector.execute_sql("show variables like 'version_comment'") + data = self.db_connector.execute_sql( + "show variables like 'version_comment'" + ) for row in data: ob_version = row[1] - version_pattern = r'(?:OceanBase(_CE)?\s+)?(\d+\.\d+\.\d+\.\d+)' + version_pattern = r"(?:OceanBase(_CE)?\s+)?(\d+\.\d+\.\d+\.\d+)" matched_version = re.search(version_pattern, ob_version) if matched_version: version = matched_version.group(2) - major_version = int(version.split('.')[0]) + major_version = int(version.split(".")[0]) - self.sql_audit_name = "gv$ob_sql_audit" if major_version >= 4 else "gv$sql_audit" + self.sql_audit_name = ( + "gv$ob_sql_audit" if major_version >= 4 else "gv$sql_audit" + ) self.plan_explain_name = ( - "gv$ob_plan_cache_plan_explain" if major_version >= 4 else "gv$plan_cache_plan_explain" + "gv$ob_plan_cache_plan_explain" + if major_version >= 4 + else "gv$plan_cache_plan_explain" ) self.ob_major_version = major_version self.tenant_mode = "mysql" self.sys_database = "oceanbase" - self.stdio.verbose(f"Detected MySQL mode successful, Database version: {ob_version}") + self.stdio.verbose( + f"Detected MySQL mode successful, Database version: {ob_version}" + ) return True else: raise ValueError("Failed to match MySQL version") except Exception as e: # Detect Oracle mode try: - data = self.sys_connector.execute_sql("select SUBSTR(BANNER, 11, 100) from V$VERSION;") + data = self.sys_connector.execute_sql( + "select SUBSTR(BANNER, 11, 100) from V$VERSION;" + ) banner = data[0][0] - version_pattern = r'(\d+\.\d+\.\d+\.\d+)' + version_pattern = r"(\d+\.\d+\.\d+\.\d+)" matched_version = re.search(version_pattern, banner) if matched_version: version = matched_version.group(1) - major_version = int(version.split('.')[0]) + major_version = int(version.split(".")[0]) - self.sql_audit_name = f"gv$ob_sql_audit" if major_version >= 4 else "gv$sql_audit" + self.sql_audit_name = ( + f"gv$ob_sql_audit" if major_version >= 4 else "gv$sql_audit" + ) self.ob_major_version = major_version self.tenant_mode = "oracle" self.sys_database = "SYS" - self.stdio.verbose(f"Detected Oracle mode successful, Database version: {version}") + self.stdio.verbose( + f"Detected Oracle mode successful, Database version: {version}" + ) return True else: raise ValueError("Failed to match Oracle version") @@ -605,7 +983,11 @@ def init_resp(self): :param target_ob: the agent object :return: a resp dict, indicating the information of the response """ - resp = {"skip": False, "error": False, "gather_pack_path": self.local_stored_path} + resp = { + "skip": False, + "error": False, + "gather_pack_path": self.local_stored_path, + } if resp["error"]: return resp return resp @@ -614,14 +996,20 @@ def copy_cs_resource(self, source_path, target_path): shutil.copytree(source_path, target_path) def sql_audit_by_trace_id_limit1_sql(self): - if self.tenant_mode == 'mysql': - sql = str(GlobalSqlMeta().get_value(key="sql_audit_by_trace_id_limit1_mysql")) \ - .replace("##REPLACE_TRACE_ID##", self.trace_id).replace("##REPLACE_SQL_AUDIT_TABLE_NAME##", - self.sql_audit_name) + if self.tenant_mode == "mysql": + sql = ( + str(GlobalSqlMeta().get_value(key="sql_audit_by_trace_id_limit1_mysql")) + .replace("##REPLACE_TRACE_ID##", self.trace_id) + .replace("##REPLACE_SQL_AUDIT_TABLE_NAME##", self.sql_audit_name) + ) else: - sql = str(GlobalSqlMeta().get_value(key="sql_audit_by_trace_id_limit1_oracle")).replace( - "##REPLACE_TRACE_ID##", self.trace_id).replace("##REPLACE_SQL_AUDIT_TABLE_NAME##", - self.sql_audit_name) + sql = ( + str( + GlobalSqlMeta().get_value(key="sql_audit_by_trace_id_limit1_oracle") + ) + .replace("##REPLACE_TRACE_ID##", self.trace_id) + .replace("##REPLACE_SQL_AUDIT_TABLE_NAME##", self.sql_audit_name) + ) return sql def select_sql_audit_by_trace_id_limit1(self): @@ -630,97 +1018,175 @@ def select_sql_audit_by_trace_id_limit1(self): return result def plan_explain_sql(self, tenant_id, plan_id, svr_ip, svr_port): - if self.tenant_mode == 'mysql': + if self.tenant_mode == "mysql": if self.ob_major_version >= 4: - sql = "select * from oceanbase.gv$ob_plan_cache_plan_explain where tenant_id = %s and " \ - "plan_id = %s and svr_ip = '%s' and svr_port = %s" % (tenant_id, plan_id, svr_ip, svr_port) + sql = ( + "select * from oceanbase.gv$ob_plan_cache_plan_explain where tenant_id = %s and " + "plan_id = %s and svr_ip = '%s' and svr_port = %s" + % (tenant_id, plan_id, svr_ip, svr_port) + ) else: - sql = "select * from oceanbase.gv$plan_cache_plan_explain where tenant_id = %s and " \ - "plan_id = %s and ip = '%s' and port = %s" % (tenant_id, plan_id, svr_ip, svr_port) + sql = ( + "select * from oceanbase.gv$plan_cache_plan_explain where tenant_id = %s and " + "plan_id = %s and ip = '%s' and port = %s" + % (tenant_id, plan_id, svr_ip, svr_port) + ) else: if self.ob_major_version >= 4: - sql = "select * from sys.gv$ob_plan_cache_plan_explain where tenant_id = %s and plan_id = " \ - "%s and svr_ip = '%s' and svr_port = %s" % (tenant_id, plan_id, svr_ip, svr_port) + sql = ( + "select * from sys.gv$ob_plan_cache_plan_explain where tenant_id = %s and plan_id = " + "%s and svr_ip = '%s' and svr_port = %s" + % (tenant_id, plan_id, svr_ip, svr_port) + ) else: - sql = "select * from sys.gv$plan_cache_plan_explain where tenant_id = %s and plan_id = " \ - "%s and svr_ip = '%s' and svr_port = %s" % (tenant_id, plan_id, svr_ip, svr_port) + sql = ( + "select * from sys.gv$plan_cache_plan_explain where tenant_id = %s and plan_id = " + "%s and svr_ip = '%s' and svr_port = %s" + % (tenant_id, plan_id, svr_ip, svr_port) + ) return sql def full_audit_sql_by_trace_id_sql(self, trace_id): - if self.tenant_mode == 'mysql': - if self.ob_major_version >=4: - sql = "select /*+ sql_audit */ %s from oceanbase.%s where trace_id = '%s' " \ - "AND client_ip IS NOT NULL ORDER BY QUERY_SQL ASC, REQUEST_ID" % ( - GlobalSqlMeta().get_value(key="sql_audit_item_mysql_obversion4"), self.sql_audit_name, trace_id) + if self.tenant_mode == "mysql": + if self.ob_major_version >= 4: + sql = ( + "select /*+ sql_audit */ %s from oceanbase.%s where trace_id = '%s' " + "AND client_ip IS NOT NULL ORDER BY QUERY_SQL ASC, REQUEST_ID" + % ( + GlobalSqlMeta().get_value( + key="sql_audit_item_mysql_obversion4" + ), + self.sql_audit_name, + trace_id, + ) + ) else: - sql = "select /*+ sql_audit */ %s from oceanbase.%s where trace_id = '%s' " \ - "AND client_ip IS NOT NULL ORDER BY QUERY_SQL ASC, REQUEST_ID" % ( - GlobalSqlMeta().get_value(key="sql_audit_item_mysql"), self.sql_audit_name, trace_id) + sql = ( + "select /*+ sql_audit */ %s from oceanbase.%s where trace_id = '%s' " + "AND client_ip IS NOT NULL ORDER BY QUERY_SQL ASC, REQUEST_ID" + % ( + GlobalSqlMeta().get_value(key="sql_audit_item_mysql"), + self.sql_audit_name, + trace_id, + ) + ) else: - if self.ob_major_version >=4: - sql = "select /*+ sql_audit */ %s from sys.%s where trace_id = '%s' AND " \ - "length(client_ip) > 4 ORDER BY REQUEST_ID" % ( - GlobalSqlMeta().get_value(key="sql_audit_item_oracle_obversion4"), self.sql_audit_name, trace_id) + if self.ob_major_version >= 4: + sql = ( + "select /*+ sql_audit */ %s from sys.%s where trace_id = '%s' AND " + "length(client_ip) > 4 ORDER BY REQUEST_ID" + % ( + GlobalSqlMeta().get_value( + key="sql_audit_item_oracle_obversion4" + ), + self.sql_audit_name, + trace_id, + ) + ) else: - sql = "select /*+ sql_audit */ %s from sys.%s where trace_id = '%s' AND " \ - "length(client_ip) > 4 ORDER BY REQUEST_ID" % ( - GlobalSqlMeta().get_value(key="sql_audit_item_oracle"), self.sql_audit_name, trace_id) + sql = ( + "select /*+ sql_audit */ %s from sys.%s where trace_id = '%s' AND " + "length(client_ip) > 4 ORDER BY REQUEST_ID" + % ( + GlobalSqlMeta().get_value(key="sql_audit_item_oracle"), + self.sql_audit_name, + trace_id, + ) + ) return sql def sql_plan_monitor_dfo_op_sql(self, tenant_id, plan_id, trace_id): - if self.tenant_mode == 'mysql': + if self.tenant_mode == "mysql": if self.ob_major_version >= 4: - sql = str(GlobalSqlMeta().get_value(key="sql_plan_monitor_dfo_op_mysql_obversion4")) \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_PLAN_ID##", str(plan_id)) \ - .replace("##REPLACE_TENANT_ID##", str(tenant_id)) \ - .replace("##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name) + sql = ( + str( + GlobalSqlMeta().get_value( + key="sql_plan_monitor_dfo_op_mysql_obversion4" + ) + ) + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace("##REPLACE_PLAN_ID##", str(plan_id)) + .replace("##REPLACE_TENANT_ID##", str(tenant_id)) + .replace( + "##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name + ) + ) else: - sql = str(GlobalSqlMeta().get_value(key="sql_plan_monitor_dfo_op_mysql")) \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_PLAN_ID##", str(plan_id)) \ - .replace("##REPLACE_TENANT_ID##", str(tenant_id)) \ - .replace("##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name) + sql = ( + str(GlobalSqlMeta().get_value(key="sql_plan_monitor_dfo_op_mysql")) + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace("##REPLACE_PLAN_ID##", str(plan_id)) + .replace("##REPLACE_TENANT_ID##", str(tenant_id)) + .replace( + "##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name + ) + ) else: if self.ob_major_version >= 4: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_dfo_op_oracle_obversion4") \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_PLAN_ID##", str(plan_id)) \ - .replace("##REPLACE_TENANT_ID##", str(tenant_id)) \ - .replace("##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name) + sql = ( + GlobalSqlMeta() + .get_value(key="sql_plan_monitor_dfo_op_oracle_obversion4") + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace("##REPLACE_PLAN_ID##", str(plan_id)) + .replace("##REPLACE_TENANT_ID##", str(tenant_id)) + .replace( + "##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name + ) + ) else: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_dfo_op_oracle") \ - .replace("##REPLACE_TRACE_ID##", trace_id) \ - .replace("##REPLACE_PLAN_ID##", str(plan_id)) \ - .replace("##REPLACE_TENANT_ID##", str(tenant_id)) \ - .replace("##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name) + sql = ( + GlobalSqlMeta() + .get_value(key="sql_plan_monitor_dfo_op_oracle") + .replace("##REPLACE_TRACE_ID##", trace_id) + .replace("##REPLACE_PLAN_ID##", str(plan_id)) + .replace("##REPLACE_TENANT_ID##", str(tenant_id)) + .replace( + "##REPLACE_PLAN_EXPLAIN_TABLE_NAME##", self.plan_explain_name + ) + ) return sql def sql_plan_monitor_svr_agg_template_sql(self): - if self.tenant_mode == 'mysql': + if self.tenant_mode == "mysql": if self.ob_major_version >= 4: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_svr_agg_template_mysql_obversion4") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_svr_agg_template_mysql_obversion4" + ) else: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_svr_agg_template_mysql") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_svr_agg_template_mysql" + ) else: if self.ob_major_version >= 4: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_svr_agg_template_oracle_obversion4") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_svr_agg_template_oracle_obversion4" + ) else: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_svr_agg_template_oracle") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_svr_agg_template_oracle" + ) return sql def sql_plan_monitor_detail_template_sql(self): - if self.tenant_mode == 'mysql': + if self.tenant_mode == "mysql": if self.ob_major_version >= 4: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_detail_template_mysql_obversion4") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_detail_template_mysql_obversion4" + ) else: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_detail_template_mysql") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_detail_template_mysql" + ) else: if self.ob_major_version >= 4: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_detail_template_oracle_obversion4") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_detail_template_oracle_obversion4" + ) else: - sql = GlobalSqlMeta().get_value(key="sql_plan_monitor_detail_template_oracle") + sql = GlobalSqlMeta().get_value( + key="sql_plan_monitor_detail_template_oracle" + ) return sql # sql audit 细节 @@ -728,7 +1194,10 @@ def report_sql_audit_details(self, sql): if self.enable_dump_db: full_audit_sql_result = self.sys_connector.execute_sql_pretty(sql) self.__report( - "

SQL_AUDIT 信息

") + "

SQL_AUDIT 信息

" + ) self.stdio.verbose("report full sql audit complete") # plan cache @@ -740,7 +1209,7 @@ def report_plan_cache(self, sql): self.stdio.verbose("report plan_explain_sql complete") data_plan_explain = from_db_cursor(cursor_plan_explain) - data_plan_explain.align = 'l' + data_plan_explain.align = "l" self.report_pre(data_plan_explain) self.stdio.verbose("report plan_explain complete") except Exception as e: @@ -765,7 +1234,9 @@ def report_sql_audit(self): def report_plan_explain(self, db_name, raw_sql): explain_sql = "explain %s" % raw_sql try: - sql_explain_cursor = self.db_connector.execute_sql_return_cursor(explain_sql) + sql_explain_cursor = self.db_connector.execute_sql_return_cursor( + explain_sql + ) self.stdio.verbose("execute SQL: %s", explain_sql) sql_explain_result_sql = "%s" % explain_sql sql_explain_result = from_db_cursor(sql_explain_cursor) @@ -774,7 +1245,7 @@ def report_plan_explain(self, db_name, raw_sql): self.stdio.verbose("report sql_explain_result_sql complete") self.report_pre(sql_explain_result_sql) self.stdio.verbose("report sql_explain_result_sql complete") - sql_explain_result.align = 'l' + sql_explain_result.align = "l" self.report_pre(sql_explain_result) self.stdio.verbose("report sql_explain_result complete") except Exception as e: @@ -785,64 +1256,116 @@ def report_plan_explain(self, db_name, raw_sql): def report_sql_plan_monitor_dfo_op(self, sql): data_sql_plan_monitor_dfo_op = self.sys_connector.execute_sql_pretty(sql) self.__report( - "

SQL_PLAN_MONITOR DFO 级调度时序汇总

") + "

SQL_PLAN_MONITOR DFO 级调度时序汇总

" + ) self.stdio.verbose("report SQL_PLAN_MONITOR DFO complete") - cursor_sql_plan_monitor_dfo_op = self.sys_connector.execute_sql_return_cursor_dictionary(sql) + cursor_sql_plan_monitor_dfo_op = ( + self.sys_connector.execute_sql_return_cursor_dictionary(sql) + ) if self.ob_major_version >= 4: - self.report_dfo_sched_agg_graph_data_obversion4(cursor_sql_plan_monitor_dfo_op, '调度时序图') + self.report_dfo_sched_agg_graph_data_obversion4( + cursor_sql_plan_monitor_dfo_op, "调度时序图" + ) else: - self.report_dfo_sched_agg_graph_data(cursor_sql_plan_monitor_dfo_op, '调度时序图') + self.report_dfo_sched_agg_graph_data( + cursor_sql_plan_monitor_dfo_op, "调度时序图" + ) self.stdio.verbose("report SQL_PLAN_MONITOR DFO SCHED complete") - cursor_sql_plan_monitor_dfo_op = self.sys_connector.execute_sql_return_cursor_dictionary(sql) + cursor_sql_plan_monitor_dfo_op = ( + self.sys_connector.execute_sql_return_cursor_dictionary(sql) + ) if self.ob_major_version >= 4: - self.report_dfo_agg_graph_data_obversion4(cursor_sql_plan_monitor_dfo_op, '数据时序图') + self.report_dfo_agg_graph_data_obversion4( + cursor_sql_plan_monitor_dfo_op, "数据时序图" + ) else: - self.report_dfo_agg_graph_data(cursor_sql_plan_monitor_dfo_op, '数据时序图') + self.report_dfo_agg_graph_data(cursor_sql_plan_monitor_dfo_op, "数据时序图") self.stdio.verbose("report SQL_PLAN_MONITOR DFO graph data complete") - def report_sql_plan_monitor_svr_agg(self, sql_plan_monitor_svr_agg_v1, sql_plan_monitor_svr_agg_v2): - cursor_sql_plan_monitor_svr_agg = self.sys_connector.execute_sql_return_cursor(sql_plan_monitor_svr_agg_v1) + def report_sql_plan_monitor_svr_agg( + self, sql_plan_monitor_svr_agg_v1, sql_plan_monitor_svr_agg_v2 + ): + cursor_sql_plan_monitor_svr_agg = self.sys_connector.execute_sql_return_cursor( + sql_plan_monitor_svr_agg_v1 + ) self.__report( - "

SQL_PLAN_MONITOR SQC 级汇总

Goto 算子优先 Goto 机器优先
") + "

SQL_PLAN_MONITOR SQC 级汇总

Goto 算子优先 Goto 机器优先
" + ) self.stdio.verbose("report SQL_PLAN_MONITOR SQC complete") - cursor_sql_plan_monitor_svr_agg_v1 = self.sys_connector.execute_sql_return_cursor_dictionary( - sql_plan_monitor_svr_agg_v2) + cursor_sql_plan_monitor_svr_agg_v1 = ( + self.sys_connector.execute_sql_return_cursor_dictionary( + sql_plan_monitor_svr_agg_v2 + ) + ) if self.ob_major_version >= 4: - self.report_svr_agg_graph_data_obversion4('svr_agg_serial_v1', cursor_sql_plan_monitor_svr_agg_v1, '算子优先视图') + self.report_svr_agg_graph_data_obversion4( + "svr_agg_serial_v1", cursor_sql_plan_monitor_svr_agg_v1, "算子优先视图" + ) else: - self.report_svr_agg_graph_data('svr_agg_serial_v1', cursor_sql_plan_monitor_svr_agg_v1, '算子优先视图') + self.report_svr_agg_graph_data( + "svr_agg_serial_v1", cursor_sql_plan_monitor_svr_agg_v1, "算子优先视图" + ) self.stdio.verbose("report SQL_PLAN_MONITOR SQC operator priority complete") - cursor_data_sql_plan_monitor_svr_agg_v2 = self.sys_connector.execute_sql_return_cursor_dictionary( - sql_plan_monitor_svr_agg_v2) + cursor_data_sql_plan_monitor_svr_agg_v2 = ( + self.sys_connector.execute_sql_return_cursor_dictionary( + sql_plan_monitor_svr_agg_v2 + ) + ) if self.ob_major_version >= 4: - self.report_svr_agg_graph_data('svr_agg_serial_v2', cursor_data_sql_plan_monitor_svr_agg_v2, '机器优先视图') + self.report_svr_agg_graph_data( + "svr_agg_serial_v2", + cursor_data_sql_plan_monitor_svr_agg_v2, + "机器优先视图", + ) else: - self.report_svr_agg_graph_data('svr_agg_serial_v2', cursor_data_sql_plan_monitor_svr_agg_v2, '机器优先视图') + self.report_svr_agg_graph_data( + "svr_agg_serial_v2", + cursor_data_sql_plan_monitor_svr_agg_v2, + "机器优先视图", + ) self.stdio.verbose("report SQL_PLAN_MONITOR SQC server priority complete") def report_sql_plan_monitor_detail_operator_priority(self, sql): - cursor_sql_plan_monitor_detail = self.sys_connector.execute_sql_return_cursor(sql) + cursor_sql_plan_monitor_detail = self.sys_connector.execute_sql_return_cursor( + sql + ) self.__report( - "

SQL_PLAN_MONITOR 详情

Goto 算子优先 Goto 线程优先
") + "

SQL_PLAN_MONITOR 详情

Goto 算子优先 Goto 线程优先
" + ) self.stdio.verbose("report SQL_PLAN_MONITOR details complete") - cursor_sql_plan_monitor_detail_v1 = self.sys_connector.execute_sql_return_cursor_dictionary(sql) + cursor_sql_plan_monitor_detail_v1 = ( + self.sys_connector.execute_sql_return_cursor_dictionary(sql) + ) if self.ob_major_version >= 4: - self.report_detail_graph_data_obversion4("detail_serial_v1", - cursor_sql_plan_monitor_detail_v1, - '算子优先视图') + self.report_detail_graph_data_obversion4( + "detail_serial_v1", cursor_sql_plan_monitor_detail_v1, "算子优先视图" + ) else: - self.report_detail_graph_data("detail_serial_v1", cursor_sql_plan_monitor_detail_v1, '算子优先视图') + self.report_detail_graph_data( + "detail_serial_v1", cursor_sql_plan_monitor_detail_v1, "算子优先视图" + ) self.stdio.verbose("report SQL_PLAN_MONITOR details operator priority complete") def reportsql_plan_monitor_detail_svr_priority(self, sql): - cursor_sql_plan_monitor_detail_v2 = self.sys_connector.execute_sql_return_cursor_dictionary(sql) + cursor_sql_plan_monitor_detail_v2 = ( + self.sys_connector.execute_sql_return_cursor_dictionary(sql) + ) if self.ob_major_version >= 4: - self.report_detail_graph_data_obversion4("detail_serial_v2", - cursor_sql_plan_monitor_detail_v2, - '线程优先视图') + self.report_detail_graph_data_obversion4( + "detail_serial_v2", cursor_sql_plan_monitor_detail_v2, "线程优先视图" + ) else: - self.report_detail_graph_data("detail_serial_v2", cursor_sql_plan_monitor_detail_v2, '线程优先视图') + self.report_detail_graph_data( + "detail_serial_v2", cursor_sql_plan_monitor_detail_v2, "线程优先视图" + ) self.stdio.verbose("report SQL_PLAN_MONITOR details server priority complete") diff --git a/handler/gather/gather_scenes.py b/handler/gather/gather_scenes.py index 6fdc9f01..e2771b3f 100644 --- a/handler/gather/gather_scenes.py +++ b/handler/gather/gather_scenes.py @@ -33,7 +33,13 @@ class GatherSceneHandler(SafeStdio): - def __init__(self, context, gather_pack_dir='./', tasks_base_path="~/.obdiag/gather/tasks/", task_type="observer"): + def __init__( + self, + context, + gather_pack_dir="./", + tasks_base_path="~/.obdiag/gather/tasks/", + task_type="observer", + ): self.context = context self.stdio = context.stdio self.is_ssh = True @@ -47,15 +53,15 @@ def __init__(self, context, gather_pack_dir='./', tasks_base_path="~/.obdiag/gat self.tasks_base_path = tasks_base_path self.task_type = task_type self.variables = {} - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): self.cluster = self.context.cluster_config - self.obproxy_nodes = self.context.obproxy_config['servers'] - self.ob_nodes = self.context.cluster_config['servers'] + self.obproxy_nodes = self.context.obproxy_config["servers"] + self.ob_nodes = self.context.cluster_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.ob_nodes, self.stdio) if new_nodes: self.nodes = new_nodes @@ -63,10 +69,10 @@ def init_config(self): def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False self.__init_variables() self.__init_report_path() @@ -76,7 +82,11 @@ def handle(self): def execute(self): try: - self.stdio.verbose("execute_tasks. the number of tasks is {0} ,tasks is {1}".format(len(self.yaml_tasks.keys()), self.yaml_tasks.keys())) + self.stdio.verbose( + "execute_tasks. the number of tasks is {0} ,tasks is {1}".format( + len(self.yaml_tasks.keys()), self.yaml_tasks.keys() + ) + ) for key, value in zip(self.yaml_tasks.keys(), self.yaml_tasks.values()): self.__execute_yaml_task_one(key, value) for task in self.code_tasks: @@ -89,15 +99,24 @@ def __execute_yaml_task_one(self, task_name, task_data): try: self.stdio.print("execute tasks: {0}".format(task_name)) task_type = self.__get_task_type(task_name) - version = get_obproxy_and_ob_version(self.obproxy_nodes, self.ob_nodes, self.task_type, self.stdio) + version = get_obproxy_and_ob_version( + self.obproxy_nodes, self.ob_nodes, self.task_type, self.stdio + ) if version: - match = re.search(r'\d+(\.\d+){2}(?:\.\d+)?', version) + match = re.search(r"\d+(\.\d+){2}(?:\.\d+)?", version) if match: self.cluster["version"] = match.group(0) else: self.stdio.erroe("get cluster.version failed") return - task = SceneBase(context=self.context, scene=task_data["task"], report_dir=self.report_path, env=self.env, scene_variable_dict=self.variables, task_type=task_type) + task = SceneBase( + context=self.context, + scene=task_data["task"], + report_dir=self.report_path, + env=self.env, + scene_variable_dict=self.variables, + task_type=task_type, + ) self.stdio.verbose("{0} execute!".format(task_name)) task.execute() self.stdio.verbose("execute tasks end : {0}".format(task_name)) @@ -111,7 +130,14 @@ def __execute_code_task_one(self, task_name): try: self.stdio.verbose("execute tasks is {0}".format(task_name)) scene = {"name": task_name} - task = SceneBase(context=self.context, scene=scene, report_dir=self.report_path, env=self.env, mode='code', task_type=task_name) + task = SceneBase( + context=self.context, + scene=scene, + report_dir=self.report_path, + env=self.env, + mode="code", + task_type=task_name, + ) self.stdio.verbose("{0} execute!".format(task_name)) task.execute() self.stdio.verbose("execute tasks end : {0}".format(task_name)) @@ -120,8 +146,8 @@ def __execute_code_task_one(self, task_name): def __init_task_names(self): if self.scene: - new = re.sub(r'\{|\}', '', self.scene) - items = re.split(r'[;,]', new) + new = re.sub(r"\{|\}", "", self.scene) + items = re.split(r"[;,]", new) scene = GatherScenesListHandler(self.context) for item in items: yaml_task_data = scene.get_one_yaml_task(item) @@ -142,7 +168,13 @@ def __init_task_names(self): def __init_report_path(self): try: - self.report_path = os.path.join(self.gather_pack_dir, "gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp), self.stdio)) + self.report_path = os.path.join( + self.gather_pack_dir, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp), + self.stdio, + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(self.report_path)) DirectoryUtil.mkdir(path=self.report_path, stdio=self.stdio) except Exception as e: @@ -151,10 +183,18 @@ def __init_report_path(self): def __init_variables(self): try: self.variables = { - "observer_data_dir": self.ob_nodes[0].get("home_path") if self.ob_nodes and self.ob_nodes[0].get("home_path") else "", - "obproxy_data_dir": self.obproxy_nodes[0].get("home_path") if self.obproxy_nodes and self.obproxy_nodes[0].get("home_path") else "", + "observer_data_dir": ( + self.ob_nodes[0].get("home_path") + if self.ob_nodes and self.ob_nodes[0].get("home_path") + else "" + ), + "obproxy_data_dir": ( + self.obproxy_nodes[0].get("home_path") + if self.obproxy_nodes and self.obproxy_nodes[0].get("home_path") + else "" + ), "from_time": self.from_time_str, - "to_time": self.to_time_str + "to_time": self.to_time_str, } self.stdio.verbose("gather scene variables: {0}".format(self.variables)) except Exception as e: @@ -162,20 +202,20 @@ def __init_variables(self): def __get_task_type(self, s): trimmed_str = s.strip() - if '.' in trimmed_str: - parts = trimmed_str.split('.', 1) + if "." in trimmed_str: + parts = trimmed_str.split(".", 1) return parts[0] else: return None def init_option(self): options = self.context.options - from_option = Util.get_option(options, 'from') - to_option = Util.get_option(options, 'to') - since_option = Util.get_option(options, 'since') - store_dir_option = Util.get_option(options, 'store_dir') - env_option = Util.get_option(options, 'env') - scene_option = Util.get_option(options, 'scene') + from_option = Util.get_option(options, "from") + to_option = Util.get_option(options, "to") + since_option = Util.get_option(options, "since") + store_dir_option = Util.get_option(options, "store_dir") + env_option = Util.get_option(options, "env") + scene_option = Util.get_option(options, "scene") if from_option is not None and to_option is not None: try: from_timestamp = TimeUtils.parse_time_str(from_option) @@ -183,28 +223,64 @@ def init_option(self): self.from_time_str = from_option self.to_time_str = to_option except OBDIAGFormatException: - self.stdio.exception('Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}'.format(from_option, to_option)) + self.stdio.exception( + "Error: Datetime is invalid. Must be in format yyyy-mm-dd hh:mm:ss. from_datetime={0}, to_datetime={1}".format( + from_option, to_option + ) + ) return False if to_timestamp <= from_timestamp: - self.stdio.exception('Error: from datetime is larger than to datetime, please check.') + self.stdio.exception( + "Error: from datetime is larger than to datetime, please check." + ) return False elif (from_option is None or to_option is None) and since_option is not None: now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) else: - self.stdio.warn('No time option provided, default processing is based on the last 30 minutes') + self.stdio.warn( + "No time option provided, default processing is based on the last 30 minutes" + ) now_time = datetime.datetime.now() - self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.to_time_str = (now_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) if since_option: - self.from_time_str = (now_time - datetime.timedelta(seconds=TimeUtils.parse_time_length_to_sec(since_option))).strftime('%Y-%m-%d %H:%M:%S') + self.from_time_str = ( + now_time + - datetime.timedelta( + seconds=TimeUtils.parse_time_length_to_sec(since_option) + ) + ).strftime("%Y-%m-%d %H:%M:%S") else: - self.from_time_str = (now_time - datetime.timedelta(minutes=30)).strftime('%Y-%m-%d %H:%M:%S') - self.stdio.print('gather from_time: {0}, to_time: {1}'.format(self.from_time_str, self.to_time_str)) + self.from_time_str = ( + now_time - datetime.timedelta(minutes=30) + ).strftime("%Y-%m-%d %H:%M:%S") + self.stdio.print( + "gather from_time: {0}, to_time: {1}".format( + self.from_time_str, self.to_time_str + ) + ) if store_dir_option: if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) if scene_option: @@ -217,4 +293,10 @@ def init_option(self): return True def __print_result(self): - self.stdio.print(Fore.YELLOW + "\nGather scene results stored in this directory: {0}\n".format(self.report_path) + Style.RESET_ALL) + self.stdio.print( + Fore.YELLOW + + "\nGather scene results stored in this directory: {0}\n".format( + self.report_path + ) + + Style.RESET_ALL + ) diff --git a/handler/gather/gather_sysstat.py b/handler/gather/gather_sysstat.py index 99b86feb..47ff09cf 100644 --- a/handler/gather/gather_sysstat.py +++ b/handler/gather/gather_sysstat.py @@ -33,7 +33,7 @@ class GatherOsInfoHandler(BaseShellHandler): - def __init__(self, context, gather_pack_dir='./', is_scene=False): + def __init__(self, context, gather_pack_dir="./", is_scene=False): super(GatherOsInfoHandler, self).__init__() self.context = context self.stdio = context.stdio @@ -42,13 +42,13 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): self.remote_stored_path = None self.is_scene = is_scene self.config_path = const.DEFAULT_CONFIG_PATH - if self.context.get_variable("gather_timestamp", None) : - self.gather_timestamp=self.context.get_variable("gather_timestamp") + if self.context.get_variable("gather_timestamp", None): + self.gather_timestamp = self.context.get_variable("gather_timestamp") else: self.gather_timestamp = TimeUtils.get_current_us_timestamp() def init_config(self): - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] new_nodes = Util.get_nodes_list(self.context, self.nodes, self.stdio) if new_nodes: self.nodes = new_nodes @@ -57,10 +57,10 @@ def init_config(self): self.file_number_limit = 20 self.file_size_limit = 2 * 1024 * 1024 * 1024 else: - basic_config = self.inner_config['obdiag']['basic'] + basic_config = self.inner_config["obdiag"]["basic"] self.file_number_limit = int(basic_config["file_number_limit"]) self.file_size_limit = int(FileUtil.size(basic_config["file_size_limit"])) - self.config_path = basic_config['config_path'] + self.config_path = basic_config["config_path"] for node in self.nodes: if node.get("ssh_type") == "docker": self.stdio.warn("the ssh_type is docker not support sysstat") @@ -69,27 +69,36 @@ def init_config(self): def init_option(self): options = self.context.options - store_dir_option = Util.get_option(options, 'store_dir') - if store_dir_option and store_dir_option != './': + store_dir_option = Util.get_option(options, "store_dir") + if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + self.stdio.warn( + "warn: args --store_dir [{0}] incorrect: No such directory, Now create it".format( + os.path.abspath(store_dir_option) + ) + ) os.makedirs(os.path.abspath(store_dir_option)) self.local_stored_path = os.path.abspath(store_dir_option) - self.scope_option = Util.get_option(options, 'scope') + self.scope_option = Util.get_option(options, "scope") return True def handle(self): if not self.init_option(): - self.stdio.error('init option failed') + self.stdio.error("init option failed") return False if not self.init_config(): - self.stdio.error('init config failed') + self.stdio.error("init config failed") return False if self.is_scene: pack_dir_this_command = self.local_stored_path else: - pack_dir_this_command = os.path.join(self.local_stored_path,"gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(self.gather_timestamp))) + pack_dir_this_command = os.path.join( + self.local_stored_path, + "gather_pack_{0}".format( + TimeUtils.timestamp_to_filename_time(self.gather_timestamp) + ), + ) self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] @@ -99,11 +108,16 @@ def handle_from_node(node): file_size = "" if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], - file_size, - int(time.time() - st), - resp["gather_pack_path"])) - + gather_tuples.append( + ( + node.get("ip"), + False, + resp["error"], + file_size, + int(time.time() - st), + resp["gather_pack_path"], + ) + ) if self.is_ssh: for node in self.nodes: @@ -118,32 +132,42 @@ def handle_from_node(node): summary_tuples = self.__get_overall_summary(gather_tuples) self.stdio.print(summary_tuples) # Persist the summary results to a file - FileUtil.write_append(os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples) + FileUtil.write_append( + os.path.join(pack_dir_this_command, "result_summary.txt"), summary_tuples + ) def __handle_from_node(self, node, local_stored_path): - resp = { - "skip": False, - "error": "", - "gather_pack_path": "" - } + resp = {"skip": False, "error": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") remote_password = node.get("ssh_password") remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") - self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) + self.stdio.verbose( + "Sending Collect Shell Command to node {0} ...".format(remote_ip) + ) DirectoryUtil.mkdir(path=local_stored_path, stdio=self.stdio) - now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") remote_dir_name = "sysstat_{0}_{1}".format(remote_ip, now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key,node, self.stdio) + ssh_helper = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format( - remote_user, - remote_ip, - self.config_path)) + self.stdio.exception( + "ssh {0}@{1}: failed, Please check the {2}".format( + remote_user, remote_ip, self.config_path + ) + ) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) @@ -161,104 +185,234 @@ def __handle_from_node(self, node, local_stored_path): self.__gather_tcp_udp_info(ssh_helper, remote_dir_full_path) zip_dir(self.is_ssh, ssh_helper, "/tmp", remote_dir_name, self.stdio) remote_file_full_path = "{0}.zip".format(remote_dir_full_path) - file_size = get_file_size(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) + file_size = get_file_size( + self.is_ssh, ssh_helper, remote_file_full_path, self.stdio + ) if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - self.stdio.verbose( - "local file path {0}...".format(local_file_path)) - download_file(self.is_ssh, ssh_helper, remote_file_full_path, local_file_path, self.stdio) + local_file_path = "{0}/{1}.zip".format( + local_stored_path, remote_dir_name + ) + self.stdio.verbose("local file path {0}...".format(local_file_path)) + download_file( + self.is_ssh, + ssh_helper, + remote_file_full_path, + local_file_path, + self.stdio, + ) resp["error"] = "" else: resp["error"] = "File too large" # delete_file_force(self.is_ssh, ssh_helper, remote_file_full_path) ssh_helper.ssh_close() - resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) + resp["gather_pack_path"] = "{0}/{1}.zip".format( + local_stored_path, remote_dir_name + ) return resp def __gather_dmesg_current_info(self, ssh_helper, gather_path): try: dmesg_cmd = "dmesg --ctime > {gather_path}/dmesg.human.current".format( - gather_path=gather_path) - self.stdio.verbose("gather dmesg current info, run cmd = [{0}]".format(dmesg_cmd)) - SshClient(self.stdio).run(ssh_helper, dmesg_cmd) if self.is_ssh else LocalClient(self.stdio).run(dmesg_cmd) + gather_path=gather_path + ) + self.stdio.verbose( + "gather dmesg current info, run cmd = [{0}]".format(dmesg_cmd) + ) + ( + SshClient(self.stdio).run(ssh_helper, dmesg_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(dmesg_cmd) + ) except: - self.stdio.error("Failed to gather dmesg current info on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather dmesg current info on server {0}".format( + ssh_helper.get_name() + ) + ) def __gather_dmesg_boot_info(self, ssh_helper, dir_path): try: - file_exit_cmd = "ls -l {file_path} 2>/dev/null".format(file_path="/var/log/dmesg") - file_exit = SshClient(self.stdio).run(ssh_helper, file_exit_cmd) if self.is_ssh else LocalClient(self.stdio).run(file_exit_cmd) + file_exit_cmd = "ls -l {file_path} 2>/dev/null".format( + file_path="/var/log/dmesg" + ) + file_exit = ( + SshClient(self.stdio).run(ssh_helper, file_exit_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(file_exit_cmd) + ) if file_exit: - dmesg_cmd = 'cp --force /var/log/dmesg {dir_path}/dmesg.boot'.format(dir_path=dir_path) - self.stdio.verbose("gather dmesg boot info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), dmesg_cmd)) - SshClient(self.stdio).run(ssh_helper, dmesg_cmd) if self.is_ssh else LocalClient(self.stdio).run(dmesg_cmd) + dmesg_cmd = "cp --force /var/log/dmesg {dir_path}/dmesg.boot".format( + dir_path=dir_path + ) + self.stdio.verbose( + "gather dmesg boot info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), dmesg_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, dmesg_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(dmesg_cmd) + ) else: - self.stdio.warn("the file /var/log/dmesg on server {0} not found ".format(ssh_helper.get_name())) + self.stdio.warn( + "the file /var/log/dmesg on server {0} not found ".format( + ssh_helper.get_name() + ) + ) except: - self.stdio.error("Failed to gather the /var/log/dmesg on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather the /var/log/dmesg on server {0}".format( + ssh_helper.get_name() + ) + ) def __tsar_exit(self, ssh_helper): try: cmd = "tar --help" - exit = SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + exit = ( + SshClient(self.stdio).run(ssh_helper, cmd) + if self.is_ssh + else LocalClient(self.stdio).run(cmd) + ) if exit: return True except: self.stdio.warn("tsar not found") - + def __gather_cpu_info(self, ssh_helper, gather_path): try: tsar_cmd = "tsar --cpu -i 1 > {gather_path}/one_day_cpu_data.txt".format( - gather_path=gather_path) - self.stdio.verbose("gather cpu info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + gather_path=gather_path + ) + self.stdio.verbose( + "gather cpu info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), tsar_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, tsar_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(tsar_cmd) + ) except: - self.stdio.error("Failed to gather cpu info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather cpu info use tsar on server {0}".format( + ssh_helper.get_name() + ) + ) def __gather_mem_info(self, ssh_helper, gather_path): try: tsar_cmd = "tsar --mem -i 1 > {gather_path}/one_day_mem_data.txt".format( - gather_path=gather_path) - self.stdio.verbose("gather memory info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + gather_path=gather_path + ) + self.stdio.verbose( + "gather memory info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), tsar_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, tsar_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(tsar_cmd) + ) except: - self.stdio.error("Failed to gather memory info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather memory info use tsar on server {0}".format( + ssh_helper.get_name() + ) + ) def __gather_swap_info(self, ssh_helper, gather_path): try: tsar_cmd = "tsar --swap --load > {gather_path}/tsar_swap_data.txt".format( - gather_path=gather_path) - self.stdio.verbose("gather swap info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + gather_path=gather_path + ) + self.stdio.verbose( + "gather swap info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), tsar_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, tsar_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(tsar_cmd) + ) except: - self.stdio.error("Failed to gather swap info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather swap info use tsar on server {0}".format( + ssh_helper.get_name() + ) + ) def __gather_io_info(self, ssh_helper, gather_path): try: tsar_cmd = "tsar --io > {gather_path}/tsar_io_data.txt".format( - gather_path=gather_path) - self.stdio.verbose("gather io info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + gather_path=gather_path + ) + self.stdio.verbose( + "gather io info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), tsar_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, tsar_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(tsar_cmd) + ) except: - self.stdio.error("Failed to gather io info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather io info use tsar on server {0}".format( + ssh_helper.get_name() + ) + ) def __gather_traffic_info(self, ssh_helper, gather_path): try: tsar_cmd = "tsar --traffic > {gather_path}/tsar_traffic_data.txt".format( - gather_path=gather_path) - self.stdio.verbose("gather traffic info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + gather_path=gather_path + ) + self.stdio.verbose( + "gather traffic info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), tsar_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, tsar_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(tsar_cmd) + ) except: - self.stdio.error("Failed to gather traffic info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather traffic info use tsar on server {0}".format( + ssh_helper.get_name() + ) + ) def __gather_tcp_udp_info(self, ssh_helper, gather_path): try: - tsar_cmd = "tsar --tcp --udp -d 1 > {gather_path}/tsar_tcp_udp_data.txt".format( - gather_path=gather_path) - self.stdio.verbose("gather tcp and udp info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + tsar_cmd = ( + "tsar --tcp --udp -d 1 > {gather_path}/tsar_tcp_udp_data.txt".format( + gather_path=gather_path + ) + ) + self.stdio.verbose( + "gather tcp and udp info on server {0}, run cmd = [{1}]".format( + ssh_helper.get_name(), tsar_cmd + ) + ) + ( + SshClient(self.stdio).run(ssh_helper, tsar_cmd) + if self.is_ssh + else LocalClient(self.stdio).run(tsar_cmd) + ) except: - self.stdio.error("Failed to gather tcp and udp info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error( + "Failed to gather tcp and udp info use tsar on server {0}".format( + ssh_helper.get_name() + ) + ) @staticmethod def __get_overall_summary(node_summary_tuple): @@ -274,7 +428,15 @@ def __get_overall_summary(node_summary_tuple): format_file_size = FileUtil.size_format(num=file_size, output_str=True) except: format_file_size = FileUtil.size_format(num=0, output_str=True) - summary_tab.append((node, "Error:" + tup[2] if is_err else "Completed", - format_file_size, "{0} s".format(int(consume_time)), pack_path)) - return "\nGather Sysstat Summary:\n" + \ - tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) + summary_tab.append( + ( + node, + "Error:" + tup[2] if is_err else "Completed", + format_file_size, + "{0} s".format(int(consume_time)), + pack_path, + ) + ) + return "\nGather Sysstat Summary:\n" + tabulate.tabulate( + summary_tab, headers=field_names, tablefmt="grid", showindex=False + ) diff --git a/handler/gather/scenes/__init__.py b/handler/gather/scenes/__init__.py index 40982804..94eb8dd1 100644 --- a/handler/gather/scenes/__init__.py +++ b/handler/gather/scenes/__init__.py @@ -14,4 +14,4 @@ @time: 2023/12/26 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/gather/scenes/base.py b/handler/gather/scenes/base.py index 11bc4de5..c449a0b4 100644 --- a/handler/gather/scenes/base.py +++ b/handler/gather/scenes/base.py @@ -25,14 +25,23 @@ class SceneBase(SafeStdio): - def __init__(self,context, scene, report_dir=None, scene_variable_dict={}, env={}, mode="yaml", task_type="observer"): + def __init__( + self, + context, + scene, + report_dir=None, + scene_variable_dict={}, + env={}, + mode="yaml", + task_type="observer", + ): self.context = context self.stdio = context.stdio self.scene_variable_dict = scene_variable_dict self.scene = scene self.cluster = context.cluster_config - self.ob_nodes = context.cluster_config['servers'] - self.obproxy_nodes = context.obproxy_config['servers'] + self.ob_nodes = context.cluster_config["servers"] + self.obproxy_nodes = context.obproxy_config["servers"] self.report_dir = report_dir self.mode = mode self.env = env @@ -50,7 +59,7 @@ def execute(self): self.__execute_yaml_mode(self.obproxy_nodes) elif self.mode == "code": self.__execute_code_mode() - else: + else: self.stdio.error("Unsupported mode. SKIP") raise Exception("Unsupported mode. SKIP") except Exception as e: @@ -62,22 +71,35 @@ def __execute_yaml_mode(self, nodes): self.stdio.verbose("Unadapted by version. SKIP") return "Unadapted by version.SKIP" self.stdio.verbose("filter_by_version is return {0}".format(steps_nu)) - if len(nodes)==0: + if len(nodes) == 0: self.stdio.warn("node is not exist") return node_number = 0 for node in nodes: - self.stdio.print("run scene excute yaml mode in node: {0} start".format(StringUtils.node_cut_passwd_for_log(node['ip'], self.stdio))) + self.stdio.print( + "run scene excute yaml mode in node: {0} start".format( + StringUtils.node_cut_passwd_for_log(node["ip"], self.stdio) + ) + ) steps = self.scene[steps_nu] nu = 1 node_number = node_number + 1 for step in steps["steps"]: try: self.stdio.verbose("step nu: {0}".format(nu)) - if len(self.cluster)==0: + if len(self.cluster) == 0: self.stdio.error("cluster is not exist") return - step_run = Base(self.context, step, node, self.cluster, self.report_dir, self.scene_variable_dict, self.env, node_number) + step_run = Base( + self.context, + step, + node, + self.cluster, + self.report_dir, + self.scene_variable_dict, + self.env, + node_number, + ) self.stdio.verbose("step nu: {0} initted, to execute".format(nu)) step_run.execute() self.scene_variable_dict = step_run.update_task_variable_dict() @@ -86,23 +108,49 @@ def __execute_yaml_mode(self, nodes): return self.stdio.verbose("step nu: {0} execute end ".format(nu)) nu = nu + 1 - self.stdio.print("run scene excute yaml mode in node: {0} end".format(StringUtils.node_cut_passwd_for_log(node['ip'], self.stdio))) + self.stdio.print( + "run scene excute yaml mode in node: {0} end".format( + StringUtils.node_cut_passwd_for_log(node["ip"], self.stdio) + ) + ) self.stdio.verbose("run scene excute yaml mode in node") def __execute_code_mode(self): - if self.scene["name"] == "observer.perf_sql" or self.scene["name"] == "observer.sql_err": - scene = SQLProblemScene(self.context, self.scene["name"], self.report_dir, self.scene_variable_dict, self.env) + if ( + self.scene["name"] == "observer.perf_sql" + or self.scene["name"] == "observer.sql_err" + ): + scene = SQLProblemScene( + self.context, + self.scene["name"], + self.report_dir, + self.scene_variable_dict, + self.env, + ) elif self.scene["name"] == "observer.cpu_high": - scene = CPUHighScene(self.context, self.report_dir, self.scene_variable_dict, self.env) + scene = CPUHighScene( + self.context, self.report_dir, self.scene_variable_dict, self.env + ) elif self.scene["name"] == "observer.px_collect_log": - scene = SQLPXCollectLogScene(self.context, self.scene["name"], self.report_dir, self.scene_variable_dict, self.env) + scene = SQLPXCollectLogScene( + self.context, + self.scene["name"], + self.report_dir, + self.scene_variable_dict, + self.env, + ) else: - self.stdio.error("unsupported hard code scene {0}".format(self.scene["name"])) + self.stdio.error( + "unsupported hard code scene {0}".format(self.scene["name"]) + ) return try: - self.stdio.verbose("hard code scene {0} execute start".format(self.scene["name"])) + self.stdio.verbose( + "hard code scene {0} execute start".format(self.scene["name"]) + ) scene.execute() - self.stdio.verbose("hard code scene {0} execute end".format(self.scene["name"])) + self.stdio.verbose( + "hard code scene {0} execute end".format(self.scene["name"]) + ) except Exception as e: self.stdio.error("hard code scene execute failed, error :{0}".format(e)) - diff --git a/handler/gather/scenes/cpu_high.py b/handler/gather/scenes/cpu_high.py index 87923419..de3acf56 100644 --- a/handler/gather/scenes/cpu_high.py +++ b/handler/gather/scenes/cpu_high.py @@ -22,6 +22,7 @@ from handler.gather.gather_perf import GatherPerfHandler from handler.gather.gather_log import GatherLogHandler + class CPUHighScene(SafeStdio): def __init__(self, context, report_path, task_variable_dict=None, env={}): self.context = context @@ -33,9 +34,9 @@ def __init__(self, context, report_path, task_variable_dict=None, env={}): self.report_path = report_path self.env = env self.is_ssh = True - self.nodes = self.context.cluster_config['servers'] + self.nodes = self.context.cluster_config["servers"] self.cluster = self.context.cluster_config - self.ob_nodes = self.context.cluster_config['servers'] + self.ob_nodes = self.context.cluster_config["servers"] def execute(self): self.__gather_obstack() @@ -59,15 +60,34 @@ def __gather_current_clocksource(self): try: self.stdio.print("gather current_clocksource start") for node in self.nodes: - ssh_helper = SshHelper(self.is_ssh, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) - cmd = 'cat /sys/devices/system/clocksource/clocksource0/current_clocksource' - self.stdio.verbose("gather current_clocksource, run cmd = [{0}]".format(cmd)) + ssh_helper = SshHelper( + self.is_ssh, + node.get("ip"), + node.get("ssh_username"), + node.get("ssh_password"), + node.get("ssh_port"), + node.get("ssh_key_file"), + node, + ) + cmd = "cat /sys/devices/system/clocksource/clocksource0/current_clocksource" + self.stdio.verbose( + "gather current_clocksource, run cmd = [{0}]".format(cmd) + ) result = ssh_helper.ssh_exec_cmd(cmd) - file_path = os.path.join(self.report_path, "current_clocksource_{ip}_result.txt".format(ip=str(node.get("ip")).replace('.', '_'))) + file_path = os.path.join( + self.report_path, + "current_clocksource_{ip}_result.txt".format( + ip=str(node.get("ip")).replace(".", "_") + ), + ) self.report(file_path, cmd, result) self.stdio.print("gather current_clocksource end") except Exception as e: - self.stdio.error("SshHandler init fail. Please check the node conf. Exception : {0} .".format(e)) + self.stdio.error( + "SshHandler init fail. Please check the node conf. Exception : {0} .".format( + e + ) + ) def __gather_log(self): try: @@ -81,8 +101,10 @@ def __gather_log(self): def report(self, file_path, command, data): try: - with open(file_path, 'a', encoding='utf-8') as f: - f.write('\n\n' + 'shell > ' + command + '\n') - f.write(data + '\n') + with open(file_path, "a", encoding="utf-8") as f: + f.write("\n\n" + "shell > " + command + "\n") + f.write(data + "\n") except Exception as e: - self.stdio.error("report sql result to file: {0} failed, error: ".format(file_path)) + self.stdio.error( + "report sql result to file: {0} failed, error: ".format(file_path) + ) diff --git a/handler/gather/scenes/list.py b/handler/gather/scenes/list.py index a83bde28..b94a503b 100644 --- a/handler/gather/scenes/list.py +++ b/handler/gather/scenes/list.py @@ -22,6 +22,7 @@ from handler.gather.scenes.register import hardcode_scene_list from common.tool import Util + class GatherScenesListHandler(SafeStdio): def __init__(self, context, yaml_tasks_base_path="~/.obdiag/gather/tasks/"): self.context = context @@ -40,8 +41,14 @@ def handle(self): self.stdio.verbose("list gather scene") self.get_all_yaml_tasks() self.get_all_code_tasks() - self.stdio.verbose("len of observer_tasks: {0}; len of observer_tasks: {1}; len of observer_tasks: {2};".format(len(self.observer_tasks), len(self.obproxy_tasks), len(self.other_tasks))) - if (len(self.observer_tasks) + len(self.obproxy_tasks) + len(self.other_tasks)) == 0: + self.stdio.verbose( + "len of observer_tasks: {0}; len of observer_tasks: {1}; len of observer_tasks: {2};".format( + len(self.observer_tasks), len(self.obproxy_tasks), len(self.other_tasks) + ) + ) + if ( + len(self.observer_tasks) + len(self.obproxy_tasks) + len(self.other_tasks) + ) == 0: self.stdio.error("Failed to find any tasks") else: self.print_scene_data() @@ -51,9 +58,9 @@ def get_all_yaml_tasks(self): current_path = self.yaml_tasks_base_path for root, dirs, files in os.walk(current_path): for file in files: - if file.endswith('.yaml'): + if file.endswith(".yaml"): folder_name = os.path.basename(root) - task_name = "{}.{}".format(folder_name, file.split('.')[0]) + task_name = "{}.{}".format(folder_name, file.split(".")[0]) task_data = YamlUtils.read_yaml_data(os.path.join(root, file)) task_data["name"] = task_name if folder_name == "observer": @@ -64,7 +71,7 @@ def get_all_yaml_tasks(self): self.other_tasks[task_name] = task_data except Exception as e: self.stdio.error("get all yaml task failed, error: ", e) - + def get_all_code_tasks(self): try: for scene in hardcode_scene_list: @@ -78,7 +85,12 @@ def get_all_code_tasks(self): self.stdio.error("get all hard code task failed, error: ", e) def __get_hardcode_task(self, scene): - return {"name": scene.name, "command": scene.command, "info_en": scene.info_en, "info_cn": scene.info_cn,} + return { + "name": scene.name, + "command": scene.command, + "info_en": scene.info_en, + "info_cn": scene.info_cn, + } def get_one_yaml_task(self, name): try: @@ -86,13 +98,15 @@ def get_one_yaml_task(self, name): current_path = self.yaml_tasks_base_path for root, dirs, files in os.walk(current_path): for file in files: - if file.endswith('.yaml'): + if file.endswith(".yaml"): folder_name = os.path.basename(root) - task_name = "{}.{}".format(folder_name, file.split('.')[0]) + task_name = "{}.{}".format(folder_name, file.split(".")[0]) if name == task_name: - task_data = YamlUtils.read_yaml_data(os.path.join(root, file)) + task_data = YamlUtils.read_yaml_data( + os.path.join(root, file) + ) task_data["name"] = task_name - return task_data + return task_data except Exception as e: self.stdio.error("get one yaml task failed, error: ", e) @@ -116,14 +130,16 @@ def print_scene_data(self): Util.print_title("Other Problem Gather Scenes") Util.print_scene(sorted_other_tasks_dict) if self.obproxy_tasks: - sorted_obproxy_tasks = sorted(self.obproxy_tasks.items(), key=lambda x: x[0]) + sorted_obproxy_tasks = sorted( + self.obproxy_tasks.items(), key=lambda x: x[0] + ) sorted_obproxy_tasks_dict = {k: v for k, v in sorted_obproxy_tasks} Util.print_title("Obproxy Problem Gather Scenes") Util.print_scene(sorted_obproxy_tasks_dict) if self.observer_tasks: - sorted_observer_tasks = sorted(self.observer_tasks.items(), key=lambda x: x[0]) + sorted_observer_tasks = sorted( + self.observer_tasks.items(), key=lambda x: x[0] + ) sorted_observer_tasks_dict = {k: v for k, v in sorted_observer_tasks} Util.print_title("Observer Problem Gather Scenes") Util.print_scene(sorted_observer_tasks_dict) - - \ No newline at end of file diff --git a/handler/gather/scenes/px_collect_log.py b/handler/gather/scenes/px_collect_log.py index fb6673fb..eeee5247 100644 --- a/handler/gather/scenes/px_collect_log.py +++ b/handler/gather/scenes/px_collect_log.py @@ -16,12 +16,20 @@ @desc: """ from handler.gather.gather_log import GatherLogHandler -from common.command import uzip_dir_local, analyze_log_get_sqc_addr, delete_file_in_folder, find_home_path_by_port +from common.command import ( + uzip_dir_local, + analyze_log_get_sqc_addr, + delete_file_in_folder, + find_home_path_by_port, +) from common.ssh import SshHelper import datetime + class SQLPXCollectLogScene(object): - def __init__(self, context, scene_name, report_path, task_variable_dict=None, env={}): + def __init__( + self, context, scene_name, report_path, task_variable_dict=None, env={} + ): self.context = context self.stdio = context.stdio if task_variable_dict is None: @@ -35,8 +43,8 @@ def __init__(self, context, scene_name, report_path, task_variable_dict=None, en self.db_conn = {} self.trace_id = "FAKE_TRACE_ID" self.sql_task_node = [] - self.ob_nodes = self.context.cluster_config['servers'] - self.obproxy_nodes = self.context.obproxy_config['servers'] + self.ob_nodes = self.context.cluster_config["servers"] + self.obproxy_nodes = self.context.obproxy_config["servers"] self.cluster = self.context.cluster_config self.search_time = None @@ -48,11 +56,11 @@ def execute(self): self.__gather_log() # 3. 分析日志，提取SQC地址 self.__analyze_log() - # 解压日志到一个新的目录 - # 分析日志，提取关键字地址 + # 解压日志到一个新的目录 + # 分析日志，提取关键字地址 # 4. 收集SQC机器的日志 - # 如果存在有效地址，则删除本地被解压的日志和压缩包，重新收集并存储于当前地址 - # 否则不存在，则删除被解压的目录 + # 如果存在有效地址，则删除本地被解压的日志和压缩包，重新收集并存储于当前地址 + # 否则不存在，则删除被解压的目录 if len(self.sql_task_node) != 0: self.stdio.verbose("delete file start") delete_file_in_folder(False, None, self.report_path, self.stdio) @@ -62,13 +70,21 @@ def execute(self): def __gather_log(self): try: - self.stdio.verbose("gather observer log start, trace id: {0}".format(self.trace_id)) - handler = GatherLogHandler(self.context, gather_pack_dir=self.report_path, is_scene=True) - self.context.set_variable('filter_nodes_list', self.sql_task_node) - self.context.set_variable('gather_grep', self.trace_id) - self.context.set_variable('gather_mode', 'trace_id_log') - from_time_str = (self.search_time - datetime.timedelta(days=3)).strftime('%Y-%m-%d %H:%M:%S') - to_time_str = (self.search_time + datetime.timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.stdio.verbose( + "gather observer log start, trace id: {0}".format(self.trace_id) + ) + handler = GatherLogHandler( + self.context, gather_pack_dir=self.report_path, is_scene=True + ) + self.context.set_variable("filter_nodes_list", self.sql_task_node) + self.context.set_variable("gather_grep", self.trace_id) + self.context.set_variable("gather_mode", "trace_id_log") + from_time_str = (self.search_time - datetime.timedelta(days=3)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + to_time_str = (self.search_time + datetime.timedelta(minutes=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) self.context.set_variable("gather_from", from_time_str) self.context.set_variable("gather_to", to_time_str) handler.handle() @@ -83,8 +99,10 @@ def __analyze_log(self): uzip_dir = self.report_path uzip_dir_local(uzip_dir, self.stdio) ip_port_str = analyze_log_get_sqc_addr(uzip_dir, self.stdio) - if ip_port_str is None or len(ip_port_str) == 0: - self.stdio.warn("No logs were found indicating that the SQC interrupted the QC; the error occurred locally in the QC.") + if ip_port_str is None or len(ip_port_str) == 0: + self.stdio.warn( + "No logs were found indicating that the SQC interrupted the QC; the error occurred locally in the QC." + ) self.sql_task_node = [] return self.stdio.verbose("find sqc ip_port_str: {0}".format(ip_port_str)) @@ -96,16 +114,20 @@ def __analyze_log(self): sqc_sql_task_node.append(node) break if self.sql_task_node == sqc_sql_task_node: - self.stdio.verbose("The SQC with an error occurred on the same observer as the QC.") + self.stdio.verbose( + "The SQC with an error occurred on the same observer as the QC." + ) self.sql_task_node = [] else: - self.stdio.verbose("The SQC with an error info: {0}".format(sqc_sql_task_node)) + self.stdio.verbose( + "The SQC with an error info: {0}".format(sqc_sql_task_node) + ) self.sql_task_node = sqc_sql_task_node self.stdio.verbose("analyze observer log end") except Exception as e: self.stdio.exception("analyze observer log failed, error: {0}".format(e)) raise Exception("analyze observer log failed, error: {0}".format(e)) - + def __find_home_path_by_port(self, ip_str, internal_port_str): for node in self.ob_nodes: if node["ip"] == ip_str: @@ -115,24 +137,37 @@ def __find_home_path_by_port(self, ip_str, internal_port_str): remote_port = node.get("ssh_port") remote_private_key = node.get("ssh_key_file") try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh = SshHelper( + self.is_ssh, + remote_ip, + remote_user, + remote_password, + remote_port, + remote_private_key, + node, + self.stdio, + ) except Exception as e: - self.stdio.error("ssh {0}@{1}: failed, Please check the config".format( - remote_user, - remote_ip)) + self.stdio.error( + "ssh {0}@{1}: failed, Please check the config".format( + remote_user, remote_ip + ) + ) return find_home_path_by_port(True, ssh, internal_port_str, self.stdio) def parse_trace_id(self, trace_id): - id_ = trace_id.split('-')[0].split('Y')[1] + id_ = trace_id.split("-")[0].split("Y")[1] uval = int(id_, 16) - ip = uval & 0xffffffff - port = (uval >> 32) & 0xffff - ip_str = "{}.{}.{}.{}".format((ip >> 24) & 0xff, (ip >> 16) & 0xff, (ip >> 8) & 0xff, ip & 0xff) + ip = uval & 0xFFFFFFFF + port = (uval >> 32) & 0xFFFF + ip_str = "{}.{}.{}.{}".format( + (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF + ) origin_ip_port = "{}:{}".format(ip_str, port) return origin_ip_port def parse_trace_id2(self, trace_id): - parts = trace_id.split('-') + parts = trace_id.split("-") processed_parts = [] for idx, t in enumerate(parts): v = int(t) @@ -140,13 +175,13 @@ def parse_trace_id2(self, trace_id): if idx == 1: n = n.zfill(16) processed_parts.append(n.upper()) - s = 'Y' + '-'.join(processed_parts) + s = "Y" + "-".join(processed_parts) origin_ip_port2 = self.parse_trace_id(s) return origin_ip_port2 def analyze_traceid(self, trace_id): - if (len(trace_id) < 50): - if (trace_id[0] == 'Y'): + if len(trace_id) < 50: + if trace_id[0] == "Y": return self.parse_trace_id(trace_id) else: return self.parse_trace_id2(trace_id) @@ -159,22 +194,28 @@ def __parse_env(self): if trace_id: self.trace_id = self.env.get("trace_id") else: - self.stdio.error("option env [--trace_id] not found, please run 'obdiag gather scene list' to check usage") + self.stdio.error( + "option env [--trace_id] not found, please run 'obdiag gather scene list' to check usage" + ) return False search_time = self.env.get("estimated_time") if search_time is None or len(search_time) == 0: search_time = datetime.datetime.now() else: try: - search_time = datetime.datetime.strptime(search_time, "%Y-%m-%d %H:%M:%S") + search_time = datetime.datetime.strptime( + search_time, "%Y-%m-%d %H:%M:%S" + ) except Exception as e: - self.stdio.error("option env [--estimated_time] format error, please use '%Y-%m-%d %H:%M:%S', run 'obdiag gather scene list' to check usage") + self.stdio.error( + "option env [--estimated_time] format error, please use '%Y-%m-%d %H:%M:%S', run 'obdiag gather scene list' to check usage" + ) return False self.search_time = search_time self.stdio.verbose("QC addr analyze begin {0}".format(trace_id)) ip_port_str = self.analyze_traceid(trace_id) self.stdio.verbose("analyze text: {0}".format(ip_port_str)) - ip_str, internal_port_str = ip_port_str.split(':') + ip_str, internal_port_str = ip_port_str.split(":") home_path_str = self.__find_home_path_by_port(ip_str, internal_port_str) for node in self.ob_nodes: if node["ip"] == ip_str and node["home_path"] == home_path_str: @@ -183,4 +224,4 @@ def __parse_env(self): self.stdio.verbose("QC addr analyze end {0}".format(self.sql_task_node)) return True except Exception as e: - self.stdio.error("Parse env fail. Exception : {0} .".format(e)) \ No newline at end of file + self.stdio.error("Parse env fail. Exception : {0} .".format(e)) diff --git a/handler/gather/scenes/register.py b/handler/gather/scenes/register.py index a65b4580..ddeebf8c 100644 --- a/handler/gather/scenes/register.py +++ b/handler/gather/scenes/register.py @@ -19,6 +19,7 @@ from dataclasses import dataclass import datetime + @dataclass class RegisteredHardCodeScene: name: str @@ -26,29 +27,35 @@ class RegisteredHardCodeScene: info_en: str info_cn: str + # 对于不适合通过yaml编排的复杂场景可以用这个类注册，注册后通过代码实现采集逻辑 -db_connect = '-h127.0.0.1 -P2881 -utest@test -p****** -Dtest' -trace_id = 'Yxx' -estimated_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') +db_connect = "-h127.0.0.1 -P2881 -utest@test -p****** -Dtest" +trace_id = "Yxx" +estimated_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") hardcode_scene_list = [ RegisteredHardCodeScene( - 'observer.perf_sql', + "observer.perf_sql", f'''obdiag gather scene run --scene=observer.perf_sql --env "{{db_connect='{db_connect}', trace_id='{trace_id}'}}"''', - '[SQL performance problem]', - '[SQL性能问题]' + "[SQL performance problem]", + "[SQL性能问题]", ), RegisteredHardCodeScene( - 'observer.sql_err', + "observer.sql_err", f'''obdiag gather scene run --scene=observer.sql_err --env "{{db_connect='{db_connect}', trace_id='{trace_id}'}}"''', - '[SQL execution error]', - '[SQL 执行出错]' + "[SQL execution error]", + "[SQL 执行出错]", + ), + RegisteredHardCodeScene( + "observer.cpu_high", + "obdiag gather scene run --scene=observer.cpu_high", + "[High CPU]", + "[CPU高]", ), - RegisteredHardCodeScene('observer.cpu_high', 'obdiag gather scene run --scene=observer.cpu_high', '[High CPU]', '[CPU高]'), RegisteredHardCodeScene( - 'observer.px_collect_log', + "observer.px_collect_log", f'''obdiag gather scene run --scene=observer.px_collect_log --env "{{trace_id='{trace_id}', estimated_time='{estimated_time}'}}"''', - '[Collect error source node logs for SQL PX]', - '[SQL PX 收集报错源节点日志]' + "[Collect error source node logs for SQL PX]", + "[SQL PX 收集报错源节点日志]", ), ] diff --git a/handler/gather/scenes/sql_problem.py b/handler/gather/scenes/sql_problem.py index 2c210dd8..75250fb1 100644 --- a/handler/gather/scenes/sql_problem.py +++ b/handler/gather/scenes/sql_problem.py @@ -24,15 +24,17 @@ class SQLProblemScene(SafeStdio): - def __init__(self, context, scene_name, report_path, task_variable_dict=None, env={}): + def __init__( + self, context, scene_name, report_path, task_variable_dict=None, env={} + ): self.context = context - self.stdio=context.stdio + self.stdio = context.stdio if task_variable_dict is None: self.task_variable_dict = {} else: self.task_variable_dict = task_variable_dict - self.ob_nodes = self.context.cluster_config['servers'] - self.obproxy_nodes = self.context.obproxy_config['servers'] + self.ob_nodes = self.context.cluster_config["servers"] + self.obproxy_nodes = self.context.obproxy_config["servers"] self.cluster = self.context.cluster_config self.report_path = report_path self.env = env @@ -57,16 +59,17 @@ def __gather_log(self): self.stdio.error("gather observer log failed, error: {0}".format(e)) raise Exception("gather observer log failed, error: {0}".format(e)) - def __gather_obproxy_log(self): try: self.stdio.verbose("gather obproxy log start") - handler = GatherObProxyLogHandler(self.context, gather_pack_dir=self.report_path, is_scene=True) + handler = GatherObProxyLogHandler( + self.context, gather_pack_dir=self.report_path, is_scene=True + ) if self.scene_name: - if self.scene_name == "observer.sql_err": + if self.scene_name == "observer.sql_err": pass - elif self.scene_name == "observer.perf_sql": - self.context.set_variable('gather_scope', self.trace_id) + elif self.scene_name == "observer.perf_sql": + self.context.set_variable("gather_scope", self.trace_id) else: self.stdio.warn("unsupported scene {0}".format(self.scene_name)) return @@ -82,9 +85,15 @@ def __gather_obproxy_log(self): def __gather_sql_info(self): try: self.stdio.verbose("gather sql info start") - self.stdio.verbose("gather sql info set_variable, key: gather_plan_monitor_trace_id, value:{0}".format(self.trace_id)) - self.context.set_variable('gather_plan_monitor_trace_id', self.trace_id) - handler = GatherPlanMonitorHandler(self.context, gather_pack_dir=self.report_path, is_scene=True) + self.stdio.verbose( + "gather sql info set_variable, key: gather_plan_monitor_trace_id, value:{0}".format( + self.trace_id + ) + ) + self.context.set_variable("gather_plan_monitor_trace_id", self.trace_id) + handler = GatherPlanMonitorHandler( + self.context, gather_pack_dir=self.report_path, is_scene=True + ) handler.handle() self.stdio.verbose("gather sql info end") except Exception as e: @@ -103,8 +112,12 @@ def __parse_env(self): self.trace_id = self.env.get("trace_id") return True else: - self.stdio.error("option env [--trace_id] not found, please run 'obdiag gather scene list' to check usage") - return False + self.stdio.error( + "option env [--trace_id] not found, please run 'obdiag gather scene list' to check usage" + ) + return False else: - self.stdio.error("option env not found, please run 'obdiag gather scene list' to check usage") + self.stdio.error( + "option env not found, please run 'obdiag gather scene list' to check usage" + ) return False diff --git a/handler/gather/step/__init__.py b/handler/gather/step/__init__.py index 4dea303d..d58f83d8 100644 --- a/handler/gather/step/__init__.py +++ b/handler/gather/step/__init__.py @@ -14,4 +14,4 @@ @time: 2024/01/05 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/gather/step/base.py b/handler/gather/step/base.py index f46c6244..cbedf36f 100644 --- a/handler/gather/step/base.py +++ b/handler/gather/step/base.py @@ -25,9 +25,19 @@ class Base(SafeStdio): - def __init__(self, context, step, node, cluster, report_path, task_variable_dict=None, env={}, node_number = 1): + def __init__( + self, + context, + step, + node, + cluster, + report_path, + task_variable_dict=None, + env={}, + node_number=1, + ): self.context = context - self.stdio=context.stdio + self.stdio = context.stdio if task_variable_dict is None: self.task_variable_dict = {} else: @@ -41,54 +51,119 @@ def __init__(self, context, step, node, cluster, report_path, task_variable_dict def execute(self): self.stdio.verbose("step: {0}".format(self.step)) - no_cluster_name_msg="(Please set ob_cluster_name or obproxy_cluster_name)" + no_cluster_name_msg = "(Please set ob_cluster_name or obproxy_cluster_name)" try: if "ip" in self.node: self.task_variable_dict["remote_ip"] = self.node["ip"] - elif "ssh_type" in self.node and self.node["ssh_type"]=="docker": + elif "ssh_type" in self.node and self.node["ssh_type"] == "docker": self.stdio.verbose("execute ssh_type is docker") - self.task_variable_dict["remote_ip"] = docker.from_env().containers.get(self.node["container_name"]).attrs['NetworkSettings']['Networks']['bridge']["IPAddress"] + self.task_variable_dict["remote_ip"] = ( + docker.from_env() + .containers.get(self.node["container_name"]) + .attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] + ) self.task_variable_dict["remote_home_path"] = self.node["home_path"] if "type" not in self.step: self.stdio.error("Missing field :type") - if (self.node_number > 1) and self.step.get("global") and (self.step.get("global") is True): - self.stdio.verbose("step sets the value of the global is true and it is processing the {0} node, skipping gather".format(self.node_number)) + if ( + (self.node_number > 1) + and self.step.get("global") + and (self.step.get("global") is True) + ): + self.stdio.verbose( + "step sets the value of the global is true and it is processing the {0} node, skipping gather".format( + self.node_number + ) + ) else: if self.step["type"] == "ssh": - handler = SshHandler(self.context, self.step, self.node, self.report_path, self.task_variable_dict) + handler = SshHandler( + self.context, + self.step, + self.node, + self.report_path, + self.task_variable_dict, + ) handler.execute() elif self.step["type"] == "sql": - handler = StepSQLHandler(self.context, self.step, self.cluster, self.report_path, self.task_variable_dict) + handler = StepSQLHandler( + self.context, + self.step, + self.cluster, + self.report_path, + self.task_variable_dict, + ) handler.execute() elif self.step["type"] == "log": - if self.node.get("host_type") and self.node.get("host_type") == "OBSERVER": - handler = GatherLogHandler(self.context, gather_pack_dir=self.report_path, is_scene=True) - self.context.set_variable('filter_nodes_list', [self.node]) - self.context.set_variable('gather_grep', self.step.get("grep")) + if ( + self.node.get("host_type") + and self.node.get("host_type") == "OBSERVER" + ): + handler = GatherLogHandler( + self.context, + gather_pack_dir=self.report_path, + is_scene=True, + ) + self.context.set_variable("filter_nodes_list", [self.node]) + self.context.set_variable("gather_grep", self.step.get("grep")) handler.handle() else: - self.stdio.verbose("node host_type is {0} not OBSERVER, skipping gather log".format(self.node.get("host_type"))) + self.stdio.verbose( + "node host_type is {0} not OBSERVER, skipping gather log".format( + self.node.get("host_type") + ) + ) elif self.step["type"] == "obproxy_log": - if self.node.get("host_type") and self.node.get("host_type") == "OBPROXY": - handler = GatherObProxyLogHandler(self.context, gather_pack_dir=self.report_path, is_scene=True) - self.context.set_variable('filter_nodes_list', [self.node]) - self.context.set_variable('gather_grep', self.step.get("grep")) + if ( + self.node.get("host_type") + and self.node.get("host_type") == "OBPROXY" + ): + handler = GatherObProxyLogHandler( + self.context, + gather_pack_dir=self.report_path, + is_scene=True, + ) + self.context.set_variable("filter_nodes_list", [self.node]) + self.context.set_variable("gather_grep", self.step.get("grep")) handler.handle() else: - self.stdio.verbose("node host_type is {0} not OBPROXY, skipping gather log".format(self.node.get("host_type"))) + self.stdio.verbose( + "node host_type is {0} not OBPROXY, skipping gather log".format( + self.node.get("host_type") + ) + ) elif self.step["type"] == "sysstat": - handler = GatherOsInfoHandler(self.context, gather_pack_dir=self.report_path, is_scene=True) - self.context.set_variable('filter_nodes_list', [self.node]) + handler = GatherOsInfoHandler( + self.context, gather_pack_dir=self.report_path, is_scene=True + ) + self.context.set_variable("filter_nodes_list", [self.node]) handler.handle() else: - self.stdio.error("the type not support: {0}" .format(self.step["type"])) + self.stdio.error( + "the type not support: {0}".format(self.step["type"]) + ) except Exception as e: self.stdio.error("StepBase handler.execute fail, error: {0}".format(e)) if self.step["type"] == "sql": - self.stdio.error("[cluster:{0}] {1}]".format(self.cluster.get("ob_cluster_name") or self.cluster.get("obproxy_cluster_name") or no_cluster_name_msg, e)) + self.stdio.error( + "[cluster:{0}] {1}]".format( + self.cluster.get("ob_cluster_name") + or self.cluster.get("obproxy_cluster_name") + or no_cluster_name_msg, + e, + ) + ) else: - self.stdio.error("[{0}:{1}] {2}]".format(self.node.get("ssh_type") or "", self.node.get("container_name") or self.task_variable_dict.get("remote_ip") or "", e)) + self.stdio.error( + "[{0}:{1}] {2}]".format( + self.node.get("ssh_type") or "", + self.node.get("container_name") + or self.task_variable_dict.get("remote_ip") + or "", + e, + ) + ) self.stdio.error("StepBase handler.execute fail, error: {0}".format(e)) def update_task_variable_dict(self): diff --git a/handler/gather/step/sql.py b/handler/gather/step/sql.py index 9105590e..79e6bb91 100644 --- a/handler/gather/step/sql.py +++ b/handler/gather/step/sql.py @@ -25,21 +25,27 @@ class StepSQLHandler(SafeStdio): def __init__(self, context, step, ob_cluster, report_path, task_variable_dict): self.context = context - self.stdio=context.stdio + self.stdio = context.stdio try: self.ob_cluster = ob_cluster self.ob_cluster_name = ob_cluster.get("cluster_name") self.tenant_mode = None self.sys_database = None self.database = None - self.ob_connector = OBConnector(ip=ob_cluster.get("db_host"), - port=ob_cluster.get("db_port"), - username=ob_cluster.get("tenant_sys").get("user"), - password=ob_cluster.get("tenant_sys").get("password"), - stdio=self.stdio, - timeout=10000) + self.ob_connector = OBConnector( + ip=ob_cluster.get("db_host"), + port=ob_cluster.get("db_port"), + username=ob_cluster.get("tenant_sys").get("user"), + password=ob_cluster.get("tenant_sys").get("password"), + stdio=self.stdio, + timeout=10000, + ) except Exception as e: - self.stdio.error("StepSQLHandler init fail. Please check the OBCLUSTER conf. OBCLUSTER: {0} Exception : {1} .".format(ob_cluster,e)) + self.stdio.error( + "StepSQLHandler init fail. Please check the OBCLUSTER conf. OBCLUSTER: {0} Exception : {1} .".format( + ob_cluster, e + ) + ) self.task_variable_dict = task_variable_dict self.enable_dump_db = False self.enable_fast_dump = False @@ -53,7 +59,9 @@ def execute(self): if "sql" not in self.step: self.stdio.error("StepSQLHandler execute sql is not set") return - sql = StringUtils.build_sql_on_expr_by_dict(self.step["sql"], self.task_variable_dict) + sql = StringUtils.build_sql_on_expr_by_dict( + self.step["sql"], self.task_variable_dict + ) self.stdio.verbose("StepSQLHandler execute: {0}".format(sql)) columns, data = self.ob_connector.execute_sql_return_columns_and_data(sql) if data is None or len(data) == 0: @@ -68,7 +76,9 @@ def update_step_variable_dict(self): def report(self, sql, column_names, data): try: table_data = [list(row) for row in data] - formatted_table = tabulate(table_data, headers=column_names, tablefmt="grid") + formatted_table = tabulate( + table_data, headers=column_names, tablefmt="grid" + ) # Check file size and rename if necessary while True: @@ -81,16 +91,22 @@ def report(self, sql, column_names, data): # Increment file suffix and update self.report_file_path base_name, ext = os.path.splitext(self.report_file_path) - parts = base_name.split('_') - if len(parts) > 1 and parts[-1].isdigit(): # Check if the last part is a digit + parts = base_name.split("_") + if ( + len(parts) > 1 and parts[-1].isdigit() + ): # Check if the last part is a digit suffix = int(parts[-1]) + 1 - new_base_name = '_'.join(parts[:-1]) + '_{}'.format(suffix) + new_base_name = "_".join(parts[:-1]) + "_{}".format(suffix) else: - new_base_name = base_name + '_1' - self.report_file_path = '{}{}'.format(new_base_name, ext) + new_base_name = base_name + "_1" + self.report_file_path = "{}{}".format(new_base_name, ext) - with open(self.report_file_path, 'a', encoding='utf-8') as f: - f.write('\n\n' + 'obclient > ' + sql + '\n') + with open(self.report_file_path, "a", encoding="utf-8") as f: + f.write("\n\n" + "obclient > " + sql + "\n") f.write(formatted_table) except Exception as e: - self.stdio.error("report sql result to file: {0} failed, error: {1}".format(self.report_file_path, str(e))) \ No newline at end of file + self.stdio.error( + "report sql result to file: {0} failed, error: {1}".format( + self.report_file_path, str(e) + ) + ) diff --git a/handler/gather/step/ssh.py b/handler/gather/step/ssh.py index 89f2a879..f7de7974 100644 --- a/handler/gather/step/ssh.py +++ b/handler/gather/step/ssh.py @@ -24,7 +24,7 @@ class SshHandler(SafeStdio): def __init__(self, context, step, node, report_path, task_variable_dict): self.context = context - self.stdio=context.stdio + self.stdio = context.stdio self.ssh_report_value = None self.parameters = None self.step = step @@ -32,9 +32,21 @@ def __init__(self, context, step, node, report_path, task_variable_dict): self.report_path = report_path try: is_ssh = True - self.ssh_helper = SshHelper(is_ssh, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + self.ssh_helper = SshHelper( + is_ssh, + node.get("ip"), + node.get("ssh_username"), + node.get("ssh_password"), + node.get("ssh_port"), + node.get("ssh_key_file"), + node, + ) except Exception as e: - self.stdio.error("SshHandler init fail. Please check the NODES conf. node: {0}. Exception : {1} .".format(node, e)) + self.stdio.error( + "SshHandler init fail. Please check the NODES conf. node: {0}. Exception : {1} .".format( + node, e + ) + ) self.task_variable_dict = task_variable_dict self.parameter = [] self.report_file_path = os.path.join(self.report_path, "shell_result.txt") @@ -44,7 +56,9 @@ def execute(self): if "ssh" not in self.step: self.stdio.error("SshHandler execute ssh is not set") return - ssh_cmd = StringUtils.build_str_on_expr_by_dict_2(self.step["ssh"], self.task_variable_dict) + ssh_cmd = StringUtils.build_str_on_expr_by_dict_2( + self.step["ssh"], self.task_variable_dict + ) self.stdio.verbose("step SshHandler execute :{0} ".format(ssh_cmd)) ssh_report_value = self.ssh_helper.ssh_exec_cmd(ssh_cmd) if ssh_report_value is None: @@ -56,15 +70,23 @@ def execute(self): self.stdio.error("ssh execute Exception:{0}".format(e).strip()) finally: self.ssh_helper.ssh_close() - self.stdio.verbose("gather step SshHandler ssh_report_value:{0}".format(self.ssh_report_value)) + self.stdio.verbose( + "gather step SshHandler ssh_report_value:{0}".format(self.ssh_report_value) + ) def update_step_variable_dict(self): return self.task_variable_dict def report(self, command, data): try: - with open(self.report_file_path, 'a', encoding='utf-8') as f: - f.write('\n\n' + '['+ self.node.get("ip") + '] shell > ' + command + '\n') - f.write(data + '\n') + with open(self.report_file_path, "a", encoding="utf-8") as f: + f.write( + "\n\n" + "[" + self.node.get("ip") + "] shell > " + command + "\n" + ) + f.write(data + "\n") except Exception as e: - self.stdio.error("report sql result to file: {0} failed, error: ".format(self.report_file_path)) + self.stdio.error( + "report sql result to file: {0} failed, error: ".format( + self.report_file_path + ) + ) diff --git a/handler/meta/__init__.py b/handler/meta/__init__.py index 29f4a072..d3a64b03 100644 --- a/handler/meta/__init__.py +++ b/handler/meta/__init__.py @@ -14,4 +14,4 @@ @time: 2023/9/20 @file: __init__.py @desc: -""" \ No newline at end of file +""" diff --git a/handler/meta/check_meta.py b/handler/meta/check_meta.py index 84edf924..9573a004 100644 --- a/handler/meta/check_meta.py +++ b/handler/meta/check_meta.py @@ -31,23 +31,23 @@ def get_value(self, key): try: return self._check_dict[key] except: - print('get' + key + 'failed\r\n') + print("get" + key + "failed\r\n") def rm_value(self, key): try: return self._check_dict.pop(key) except: - print('delete' + key + 'failed\r\n') + print("delete" + key + "failed\r\n") check_dict = GlobalCheckMeta() check_dict.set_value( "check_verify_shell", - ''' + """ if ${new_expr}; then echo "true" else echo "false" fi - ''' + """, ) diff --git a/handler/meta/html_meta.py b/handler/meta/html_meta.py index e8388a59..90406ffd 100644 --- a/handler/meta/html_meta.py +++ b/handler/meta/html_meta.py @@ -31,20 +31,20 @@ def get_value(self, key): try: return self._html_dict[key] except: - print('get' + key + 'failed\r\n') + print("get" + key + "failed\r\n") def rm_value(self, key): try: return self._html_dict.pop(key) except: - print('delete' + key + 'failed\r\n') + print("delete" + key + "failed\r\n") html_dict = GlobalHtmlMeta() html_dict.set_value( "sql_plan_monitor_report_header", - ''' + """ @@ -200,12 +200,12 @@ def rm_value(self, key): topnode.get(0).innerHTML = table; } - ''' + """, ) html_dict.set_value( "sql_plan_monitor_report_footer", - ''' + """ - ''' + """, ) html_dict.set_value( "sql_plan_monitor_report_footer_obversion4", - ''' + """