Format code

.github/workflows/format_check.yml

@@ -0,0 +1,29 @@
+name: Black Code Formatter Check
+
+on:
+  pull_request:
+    branches: "*"
+  push:
+    branches: "*"
+
+jobs:
+  format_check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+
+    - name: Install Black
+      run: |
+        python -m pip install --upgrade pip
+        pip install black
+
+    - name: Run Black Formatter Check
+      run: black --check -S -l 256 .
+      # `--check`选项会让black检查代码而不做实际修改，如果格式不符合black规范，则此步骤会失败

common/__init__.py

@@ -14,4 +14,4 @@
 @time: 2022/6/20
 @file: __init__.py
 @desc:
-"""
+"""

common/command.py

@@ -86,7 +86,7 @@ def download_file(is_ssh, ssh_helper, remote_path, local_path, stdio=None):
     """
     try:
         if is_ssh:
-            stdio.verbose("Please wait a moment, download file [{0}] from server {1} to [{2}]".format(remote_path, ssh_helper.get_name(),local_path))
+            stdio.verbose("Please wait a moment, download file [{0}] from server {1} to [{2}]".format(remote_path, ssh_helper.get_name(), local_path))
             ssh_helper.download(remote_path, local_path)
         else:
             cmd = "cp -r {0} {1}".format(remote_path, local_path)
@@ -102,8 +102,7 @@ def upload_file(is_ssh, ssh_helper, local_path, remote_path, stdio=None):
     :param args: is_ssh, ssh helper, local file path, remote file path
     :return: local path
     """
-    stdio.verbose("Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format(
-        ssh_helper.get_name(), local_path, remote_path))
+    stdio.verbose("Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format(ssh_helper.get_name(), local_path, remote_path))
     try:
         if is_ssh:
             ssh_helper.upload(local_path, remote_path)
@@ -126,6 +125,7 @@ def rm_rf_file(is_ssh, ssh_helper, dir, stdio=None):
     else:
         LocalClient(stdio).run(cmd)
 
+
 def delete_file_in_folder(is_ssh, ssh_helper, file_path, stdio):
     """
     delete file
@@ -160,7 +160,7 @@ def is_empty_dir(is_ssh, ssh_helper, dir, stdio=None):
 
 def get_file_start_time(is_ssh, ssh_helper, file_name, dir, stdio=None):
     """
-    get log file start time 
+    get log file start time
     :param args: is_ssh, ssh helper, gather log full path
     :return: true or false
     """
@@ -174,7 +174,7 @@ def get_file_start_time(is_ssh, ssh_helper, file_name, dir, stdio=None):
 
 def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_dir, log_files, stdio=None):
     """
-    get log name list 
+    get log name list
     :param args: is_ssh, ssh helper, from time, to time, log dir, log file list
     :return: true or false
     """
@@ -197,11 +197,9 @@ def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_di
             elif last_file_dict["prefix_file_name"] == "":
                 file_start_time_str = get_file_start_time(is_ssh, ssh_helper, file_name, log_dir, stdio)
             # When two time intervals overlap, need to add the file
-            if (file_end_time_str != "") and (file_start_time_str != "") and (file_start_time_str <= to_time_str) and (
-                    file_end_time_str >= from_time_str):
+            if (file_end_time_str != "") and (file_start_time_str != "") and (file_start_time_str <= to_time_str) and (file_end_time_str >= from_time_str):
                 log_name_list.append(file_name)
-            last_file_dict = {"prefix_file_name": prefix_name, "file_name": file_name,
-                              "file_end_time": file_end_time_str}
+            last_file_dict = {"prefix_file_name": prefix_name, "file_name": file_name, "file_end_time": file_end_time_str}
         elif file_name.endswith("log") or file_name.endswith("wf"):
             # Get the first and last lines of text of the file. Here, use a command
             get_first_line_cmd = "head -n 1 {0}/{1} && tail -n 1 {0}/{1}".format(log_dir, file_name)
@@ -219,16 +217,14 @@ def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_di
                 # Time to parse the first and last lines of text
                 file_start_time_str = TimeUtils.extract_time_from_log_file_text(first_line_text, stdio)
                 file_end_time = TimeUtils.extract_time_from_log_file_text(last_line_text, stdio)
-                stdio.verbose("The log file {0} starts at {1} ends at {2}".format(file_name, file_start_time_str,file_end_time))
+                stdio.verbose("The log file {0} starts at {1} ends at {2}".format(file_name, file_start_time_str, file_end_time))
                 stdio.verbose("to_time_str {0} from_time_str {1}".format(to_time_str, from_time_str))
                 if (file_start_time_str <= to_time_str) and (file_end_time >= from_time_str):
                     log_name_list.append(file_name)
     if len(log_name_list) > 0:
-        stdio.verbose("Find the qualified log file {0} on Server [{1}], "
-                    "wait for the next step".format(log_name_list, "localhost" if not is_ssh else ssh_helper.get_name()))
+        stdio.verbose("Find the qualified log file {0} on Server [{1}], " "wait for the next step".format(log_name_list, "localhost" if not is_ssh else ssh_helper.get_name()))
     else:
-        stdio.warn("No found the qualified log file on Server [{0}]".format(
-            "localhost" if not is_ssh else ssh_helper.get_name()))
+        stdio.warn("No found the qualified log file on Server [{0}]".format("localhost" if not is_ssh else ssh_helper.get_name()))
     return log_name_list
 
 
@@ -264,9 +260,7 @@ def zip_dir(is_ssh, ssh_helper, father_dir, zip_dir, stdio=None):
     :param args: is_ssh, ssh helper, father dir, zip dir
     :return:
     """
-    cmd = "cd {father_dir} && zip {zip_dir}.zip -rm {zip_dir}".format(
-        father_dir=father_dir,
-        zip_dir=zip_dir)
+    cmd = "cd {father_dir} && zip {zip_dir}.zip -rm {zip_dir}".format(father_dir=father_dir, zip_dir=zip_dir)
     stdio.verbose("Please wait a moment ...")
     if is_ssh:
         SshClient(stdio).run(ssh_helper, cmd)
@@ -280,16 +274,14 @@ def zip_encrypt_dir(is_ssh, ssh_helper, zip_password, father_dir, zip_dir, stdio
     :param args: is_ssh, ssh helper, password, raw_log_dir, gather dir name
     :return:
     """
-    cmd = "cd {father_dir} && zip --password {zip_password} {zip_dir}.zip -rm {zip_dir}".format(
-        zip_password=zip_password,
-        father_dir=father_dir,
-        zip_dir=zip_dir)
+    cmd = "cd {father_dir} && zip --password {zip_password} {zip_dir}.zip -rm {zip_dir}".format(zip_password=zip_password, father_dir=father_dir, zip_dir=zip_dir)
     stdio.verbose("Please wait a moment ...")
     if is_ssh:
         SshClient(stdio).run(ssh_helper, cmd)
     else:
         LocalClient(stdio).run(cmd)
 
+
 def is_support_arch(is_ssh, ssh_helper, stdio=None):
     """
     Determine if it is a supported operating system
@@ -331,8 +323,7 @@ def get_observer_version(is_ssh, ssh_helper, ob_install_dir, stdio):
             result = re.sub(r'[a-zA-Z]', '', ob_version[0])
             return result.strip()
         else:
-            cmd = "export LD_LIBRARY_PATH={ob_install_dir}/lib && {ob_install_dir}/bin/observer --version".format(
-                ob_install_dir=ob_install_dir)
+            cmd = "export LD_LIBRARY_PATH={ob_install_dir}/lib && {ob_install_dir}/bin/observer --version".format(ob_install_dir=ob_install_dir)
             if is_ssh:
                 ob_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd)
             else:
@@ -363,13 +354,12 @@ def get_obproxy_version(is_ssh, ssh_helper, obproxy_install_dir, stdio):
         if len(ob_version) > 0:
             return ob_version[0]
         else:
-            cmd = "export LD_LIBRARY_PATH={obproxy_install_dir}/lib && {obproxy_install_dir}/bin/obproxy --version".format(
-                obproxy_install_dir=obproxy_install_dir)
+            cmd = "export LD_LIBRARY_PATH={obproxy_install_dir}/lib && {obproxy_install_dir}/bin/obproxy --version".format(obproxy_install_dir=obproxy_install_dir)
             if is_ssh:
                 obproxy_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd)
             else:
                 obproxy_version_info = LocalClient(stdio).run_get_stderr(cmd)
-            stdio.verbose("get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format(cmd,obproxy_version_info))
+            stdio.verbose("get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format(cmd, obproxy_version_info))
             if "REVISION" not in obproxy_version_info:
                 raise Exception("Please check conf about proxy,{0}".format(obproxy_version_info))
             pattern = r"(\d+\.\d+\.\d+\.\d+)"
@@ -383,17 +373,15 @@ def get_obproxy_version(is_ssh, ssh_helper, obproxy_install_dir, stdio):
                 obproxy_version_info = match.group(1)
                 obproxy_version_info = obproxy_version_info.split()[0]
             return obproxy_version_info
+
+
 # Only applicable to the community version
 
+
 def get_observer_version_by_sql(ob_cluster, stdio=None):
     stdio.verbose("start get_observer_version_by_sql . input: {0}".format(ob_cluster))
     try:
-        ob_connector = OBConnector(ip=ob_cluster.get("db_host"),
-                                   port=ob_cluster.get("db_port"),
-                                   username=ob_cluster.get("tenant_sys").get("user"),
-                                   password=ob_cluster.get("tenant_sys").get("password"),
-                                   stdio=stdio,
-                                   timeout=100)
+        ob_connector = OBConnector(ip=ob_cluster.get("db_host"), port=ob_cluster.get("db_port"), username=ob_cluster.get("tenant_sys").get("user"), password=ob_cluster.get("tenant_sys").get("password"), stdio=stdio, timeout=100)
         ob_version_info = ob_connector.execute_sql("select version();")
     except Exception as e:
         raise Exception("get_observer_version_by_sql Exception. Maybe cluster'info is error: " + e.__str__())
@@ -528,15 +516,14 @@ def uzip_dir_local(uzip_dir, stdio=None):
     cmd = f"cd {uzip_dir} && unzip *.zip && rm -rf *.zip"
     LocalClient(stdio).run(cmd)
 
+
 def analyze_log_get_sqc_addr(uzip_dir, stdio):
     """
     analyze files
     :param args: father dir, uzip dir
     :return: ip_port
     """
-    cmd = "cd {uzip_dir} && cd ob_log* && grep {key_words} * | grep -oP '{key_words}=\"\\K[^\"]+' | sort | uniq".format(
-        uzip_dir=uzip_dir,
-        key_words = "px_obdiag_sqc_addr")
+    cmd = "cd {uzip_dir} && cd ob_log* && grep {key_words} * | grep -oP '{key_words}=\"\\K[^\"]+' | sort | uniq".format(uzip_dir=uzip_dir, key_words="px_obdiag_sqc_addr")
     stdout = LocalClient(stdio).run(cmd)
     sqc_addrs = stdout.decode().strip().split('\n')
     if len(sqc_addrs) > 0:
@@ -547,14 +534,14 @@ def analyze_log_get_sqc_addr(uzip_dir, stdio):
     else:
         return None
 
+
 def find_home_path_by_port(is_ssh, ssh_helper, internal_port_str, stdio):
-    cmd = "ps aux | grep observer | grep 'P {internal_port_str}' |  grep -oP '/[^\s]*/bin/observer' ".format(
-        internal_port_str = internal_port_str)
+    cmd = "ps aux | grep observer | grep 'P {internal_port_str}' |  grep -oP '/[^\s]*/bin/observer' ".format(internal_port_str=internal_port_str)
     if is_ssh:
         stdout = SshClient(stdio).run(ssh_helper, cmd)
     else:
         stdout = LocalClient(stdio).run(cmd)
-    
+
     str_list = stdout.strip().split('\n')
     home_path = ""
     for original_str in str_list:

common/config_helper.py

@@ -41,17 +41,18 @@ def __init__(self, context):
         self.db_port = Util.get_option(options, 'P')
         self.config_path = os.path.expanduser('~/.obdiag/config.yml')
         self.inner_config = self.context.inner_config
-        self.ob_cluster = {"db_host": self.db_host, "db_port": self.db_port, "tenant_sys": {"password": self.sys_tenant_password, "user": self.sys_tenant_user, }}
+        self.ob_cluster = {
+            "db_host": self.db_host,
+            "db_port": self.db_port,
+            "tenant_sys": {
+                "password": self.sys_tenant_password,
+                "user": self.sys_tenant_user,
+            },
+        }
 
     def get_cluster_name(self):
         ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio)
-        obConnetcor = OBConnector(
-            ip=self.db_host, 
-            port=self.db_port, 
-            username=self.sys_tenant_user,
-            password=self.sys_tenant_password, 
-            stdio=self.stdio,
-            timeout=100)
+        obConnetcor = OBConnector(ip=self.db_host, port=self.db_port, username=self.sys_tenant_user, password=self.sys_tenant_password, stdio=self.stdio, timeout=100)
         if ob_version.startswith("3") or ob_version.startswith("2"):
             sql = "select cluster_name from oceanbase.v$ob_cluster"
             res = obConnetcor.execute_sql(sql)
@@ -64,19 +65,13 @@ def get_cluster_name(self):
 
     def get_host_info_list_by_cluster(self):
         ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio)
-        obConnetcor = OBConnector(ip=self.db_host,
-                                  port=self.db_port,
-                                  username=self.sys_tenant_user,
-                                  password=self.sys_tenant_password,
-                                  stdio=self.stdio,
-                                  timeout=100)
+        obConnetcor = OBConnector(ip=self.db_host, port=self.db_port, username=self.sys_tenant_user, password=self.sys_tenant_password, stdio=self.stdio, timeout=100)
         sql = "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.DBA_OB_SERVERS"
         if ob_version.startswith("3") or ob_version.startswith("2") or ob_version.startswith("1"):
             sql = "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.__all_server"
         res = obConnetcor.execute_sql(sql)
         if len(res) == 0:
-            raise Exception("Failed to get the node from cluster config, "
-                            "please check whether the cluster config correct!!!")
+            raise Exception("Failed to get the node from cluster config, " "please check whether the cluster config correct!!!")
         host_info_list = []
         for row in res:
             host_info = OrderedDict()
@@ -111,25 +106,8 @@ def build_configuration(self):
         global_data_dir = self.input_with_default("oceanbase data_dir", default_data_dir)
         global_redo_dir = self.input_with_default("oceanbase redo_dir", default_data_dir)
         tenant_sys_config = {"user": self.sys_tenant_user, "password": self.sys_tenant_password}
-        global_config = {
-            "ssh_username": global_ssh_username,
-            "ssh_password": global_ssh_password,
-            "ssh_port": global_ssh_port,
-            "ssh_key_file": "",
-            "home_path": global_home_path,
-            "data_dir": global_data_dir,
-            "redo_dir": global_redo_dir
-        }
-        new_config = {
-            "obcluster": {
-                "ob_cluster_name": ob_cluster_name,
-                "db_host": self.db_host,
-                "db_port": self.db_port,
-                "tenant_sys": tenant_sys_config,
-                "servers": {
-                    "nodes": nodes_config,
-                    "global": global_config
-                }}}
+        global_config = {"ssh_username": global_ssh_username, "ssh_password": global_ssh_password, "ssh_port": global_ssh_port, "ssh_key_file": "", "home_path": global_home_path, "data_dir": global_data_dir, "redo_dir": global_redo_dir}
+        new_config = {"obcluster": {"ob_cluster_name": ob_cluster_name, "db_host": self.db_host, "db_port": self.db_port, "tenant_sys": tenant_sys_config, "servers": {"nodes": nodes_config, "global": global_config}}}
         YamlUtils.write_yaml_data(new_config, self.config_path)
         need_config_obproxy = self.input_choice_default("need config obproxy [y/N]", "N")
         if need_config_obproxy:
@@ -154,13 +132,7 @@ def build_obproxy_configuration(self, path):
                 "ssh_key_file": "",
                 "home_path": global_home_path,
             }
-            new_config = {
-                "obproxy": {
-                    "obproxy_cluster_name": "obproxy",
-                    "servers": {
-                        "nodes": nodes_config,
-                        "global": global_config
-                    }}}
+            new_config = {"obproxy": {"obproxy_cluster_name": "obproxy", "servers": {"nodes": nodes_config, "global": global_config}}}
             YamlUtils.write_yaml_data_append(new_config, path)
 
     def get_old_configuration(self, path):
@@ -185,8 +157,7 @@ def input_with_default(self, prompt, default):
             return value
 
     def input_password_with_default(self, prompt, default):
-        value = pwinput.pwinput(prompt="\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default),
-                                mask='*')
+        value = pwinput.pwinput(prompt="\033[32mEnter your {0} (default:'{1}'): \033[0m".format(prompt, default), mask='*')
         if value == '' or value.lower() == "y" or value.lower() == "yes":
             return default
         else:

common/constant.py

@@ -62,41 +62,15 @@ def __setattr__(self, name, value):
 const.FLT_TRACE_OUTPUT = 50
 
 const.OBDIAG_BASE_DEFAULT_CONFIG = {
-  "obdiag": {
-    "basic": {
-      "config_path": "~/.obdiag/config.yml",
-      "config_backup_dir": "~/.obdiag/backup_conf",
-      "file_number_limit": 20,
-      "file_size_limit": "2G"
-    },
-    "logger": {
-      "file_handler_log_level": "DEBUG",
-      "log_dir": "~/.obdiag/log",
-      "log_filename": "obdiag.log",
-      "log_level": "INFO",
-      "mode": "obdiag",
-      "stdout_handler_log_level": "INFO"
+    "obdiag": {
+        "basic": {"config_path": "~/.obdiag/config.yml", "config_backup_dir": "~/.obdiag/backup_conf", "file_number_limit": 20, "file_size_limit": "2G"},
+        "logger": {"file_handler_log_level": "DEBUG", "log_dir": "~/.obdiag/log", "log_filename": "obdiag.log", "log_level": "INFO", "mode": "obdiag", "stdout_handler_log_level": "INFO"},
     }
-  }
 }
 
-const.OBDIAG_CHECK_DEFAULT_CONFIG = {
-  "check": {
-    "ignore_version": "false",
-    "report": {
-      "report_path": "./check_report/",
-      "export_type": "table"
-    },
-    "package_file": "~/.obdiag/check/check_package.yaml",
-    "tasks_base_path": "~/.obdiag/check/tasks/"
-  }
-}
+const.OBDIAG_CHECK_DEFAULT_CONFIG = {"check": {"ignore_version": "false", "report": {"report_path": "./check_report/", "export_type": "table"}, "package_file": "~/.obdiag/check/check_package.yaml", "tasks_base_path": "~/.obdiag/check/tasks/"}}
 
-const.OBDIAG_GATHER_DEFAULT_CONFIG = {
-  "gather": {
-    "cases_base_path": "~/.obdiag/gather/tasks"
-  }
-}
+const.OBDIAG_GATHER_DEFAULT_CONFIG = {"gather": {"cases_base_path": "~/.obdiag/gather/tasks"}}
 
 const.OBDIAG_RCA_DEFAULT_CONFIG = {
     "rca": {
@@ -110,4 +84,4 @@ def __setattr__(self, name, value):
 const.UPDATE_REMOTE_SERVER = 'https://obbusiness-private.oss-cn-shanghai.aliyuncs.com'
 const.UPDATE_REMOTE_VERSION_FILE_NAME = 'https://obbusiness-private.oss-cn-shanghai.aliyuncs.com/download-center/opensource/obdiag/version.yaml'
 const.UPDATE_REMOTE_UPDATE_FILE_NAME = 'https://obbusiness-private.oss-cn-shanghai.aliyuncs.com/download-center/opensource/obdiag/data.tar'
-const.RCA_WORK_PATH= '~/.obdiag/rca'
+const.RCA_WORK_PATH = '~/.obdiag/rca'

common/log.py

@@ -27,4 +27,4 @@ def __init__(self, name, level=logging.DEBUG):
         self.buffer_size = 0
 
     def _log(self, level, msg, args, end='\n', **kwargs):
-        return super(Logger, self)._log(level, msg, args, **kwargs)
+        return super(Logger, self)._log(level, msg, args, **kwargs)