Merge branch 'jingyd6_2.5_lock' of github.com:jingyd66/obdiag into ji…

…ngyd6_2.5_lock

.github/workflows/build_package.yml

@@ -16,28 +16,18 @@ jobs:
     name: Run on CentOS 7
     runs-on: ubuntu-latest
     container:
-      image: "centos:7"
+      image: "oceanbase/obdiag-builder:latest"
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
       - name: Install dependencies
         run: |
-          rm -rf /etc/yum.repos.d/*
-          curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
-          yum install -y wget 
-          wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --no-check-certificate
-          sh Miniconda3-latest-Linux-x86_64.sh -p /opt/miniconda3 -b
           export PATH=/opt/miniconda3/bin:$PATH
-          conda init
-          conda create --name obdiag python=3.8 -y
           source activate obdiag
           ldd --version
-          python3 -m pip install --upgrade pip setuptools wheel
           pip3 install -r requirements3.txt
-          python3 --version
-          yum install rpm-build -y
 
       - name: Build package
         run: |
@@ -49,8 +39,8 @@ jobs:
           sed -i 's/pip install -r requirements3.txt/curl https:\/\/bootstrap.pypa.io\/get-pip.py -o get-pip.py\n\
           python3 get-pip.py\n\
           pip3 install -r requirements3.txt/' ./rpm/oceanbase-diagnostic-tool.spec
+          python3 -m pip install --upgrade pip wheel
           cat ./rpm/oceanbase-diagnostic-tool.spec
-          python3 -m pip install --upgrade pip setuptools wheel
           python3 --version
           rpmbuild -bb ./rpm/oceanbase-diagnostic-tool.spec
 

.github/workflows/test_command_scene_configHelper.yml

@@ -0,0 +1,31 @@
+# common包下command、scene、config_helper的测试用例
+name: Test command_scene_configHelper
+
+on:
+  push:
+    branches: "*"
+  pull_request:
+    branches: "*"
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # Fetch all history for proper version detection
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements3.txt
+
+    - name: Run tests
+      run:  python -m unittest discover -s test/common -p 'test_*.py'

.github/workflows/test_ssh_client.yml

@@ -0,0 +1,30 @@
+name: Test Ssh Client
+
+on:
+  push:
+    branches: "dev*"
+  pull_request:
+    branches: "dev*"
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # Fetch all history for proper version detection
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements3.txt
+
+    - name: Run tests
+      run: python -m unittest discover -s test/common/ssh_client -p 'test_*.py'

README-CN.md

@@ -77,13 +77,13 @@ obdiag 期望构建一个开放的社区，我们欢迎任何形式的贡献，
 |2.0.0|2024.03| 2024.04.11|<ul><li> context改造，场景化扩展能力增强 </li><li> 支持在线更新巡检、采集的task </li><li> 根因分析二期 </li></ul>|
 |2.1.0|2024.04| 2024.05.13|<ul><li> 根因分析场景扩展 </li><li> 新增 ash 报告 采集 </li></ul>|
 |2.2.0|2024.05| 2024.06.14 |<ul><li> 根因分析场景扩展 </li><li> 巡检场景扩展 </li></ul>|
-|2.3.0|2024.06| - |<ul><li> 根因分析场景扩展 </li><li> 支持 SQL 诊断 </li></ul>|
-|2.4.0|2024.07| - |<ul><li> 根因分析场景扩展 </li><li> 适配两款内核的诊断工具 </li></ul>|
-|2.5.0|2024.08| - |<ul><li> 根因分析场景扩展 </li><li> 支持 OMS 诊断 </li></ul>|
-|3.0.0|2024.09| - |<ul><li> 根因分析场景扩展 </li><li> 服务化改造 </li></ul>|
-|3.1.0|2024.10| - |<ul><li> 根因分析场景扩展 </li><li> 支持巡检报告比对 </li></ul>|
-|3.2.0|2024.11| - |<ul><li> 根因分析场景扩展 </li><li> SQL 诊断二期，支持SQL问题的根因分析 </li></ul>|
-|3.3.0|2024.12| - |<ul><li> AI 化探索 </li></ul>|
+|2.3.0|2024.06| 2024.07.24 |<ul><li> 根因分析场景扩展 </li><li> 新增基础采集功能: tabledump </li><li> 新增参数/变量比对分析功能 </li><li> 执行底座改造，支持 k8s 部署的 OceanBase 集群诊断 </li></ul>|
+|2.4.0|2024.07| 2024.09.03 |<ul><li> 易用性改造 </li><li> 索引空间分析 </li></ul>|
+|2.5.0|2024.09| - |<ul><li> 场景化一键展示集群诊断信息功能 </li><li> 队列积压分析 </li></ul>|
+|3.0.0|2024.10| - |<ul><li> SQL 诊断 </li><li> 支持 OMS 诊断 </li></ul>|
+|3.1.0|2024.11| - |<ul><li> 根因分析场景扩展 </li><li> 支持巡检报告比对 </li></ul>|
+|3.2.0|2024.12| - |<ul><li> 根因分析场景扩展 </li><li> SQL 诊断二期，支持SQL问题的根因分析 </li></ul>|
+|4.0.0|2025.01| - |<ul><li> AI 化探索 </li></ul>|
 
 # 支持
 

README.md

@@ -81,13 +81,13 @@ obdiag envisions an open community. We welcome your contributions in any form:
 |2.0.0|2024.03| 2024.04.11| <ul><li> Context Transformation, Enhanced Scene Expansion Capabilities</li><li> Support online updating of inspection and gather tasks </li><li>Root Cause Analysis Phase II Transformation </li></ul>|
 |2.1.0|2024.04| 2024.05.13| <ul><li> Root Cause Analysis Scenario Expansion </li><li> Gather ash report </li></ul>|
 |2.2.0|2024.05| 2024.06.14 |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Check Scenario Expansion </li></ul>|
-|2.3.0|2024.06| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Support SQL Diagnosis </li></ul>|
-|2.4.0|2024.07| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Adapting Two Additional Kernel Diagnostic Tools </li></ul>|
-|2.5.0|2024.08| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Support OMS Diagnosis </li></ul>|
-|3.0.0|2024.09| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Service-ification Transformation </li></ul>|
-|3.1.0|2024.10| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Supporting Comparative Functionality for Patrol Inspection Reports </li></ul>|
-|3.2.0|2024.11| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> SQL Diagnosis Phase II, Supporting Root Cause Analysis for SQL problems </li></ul>|
-|3.3.0|2024.12| - |<ul><li> AI for obdiag </li></ul>|
+|2.3.0|2024.06| 2024.07.24 |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Added basic gather feature: tabledump </li><li> Added parameter/variable gather and analyze feature </li><li> Execute infrastructure modifications to support diagnostics for OceanBase clusters deployed on Kubernetes (k8s) </li></ul>|
+|2.4.0|2024.07| 2024.09.03 |<ul><li> usability improvement </li><li> Index Space Size Analysis </li></ul>|
+|2.5.0|2024.09| - |<ul><li> Cluster Diagnosis Information Display </li><li> Queue Analysis </li></ul>|
+|3.0.0|2024.10| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> SQL Diagnosis </li></ul>|
+|3.1.0|2024.11| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> Supporting Comparative Functionality for Patrol Inspection Reports </li></ul>|
+|3.2.0|2024.12| - |<ul><li> Root Cause Analysis Scenario Expansion </li><li> SQL Diagnosis Phase II, Supporting Root Cause Analysis for SQL problems </li></ul>|
+|4.0.0|2025.01| - |<ul><li> AI for obdiag </li></ul>|
 
 # Support
 

common/command.py

@@ -48,37 +48,6 @@ def run_get_stderr(self, cmd):
             self.stdio.error("run cmd = [{0}] on localhost".format(cmd))
 
 
-#
-# class SshClient(object):
-#     def __init__(self, stdio=None):
-#         self.stdio = stdio
-#
-#     def run(self, ssh_helper, cmd):
-#         try:
-#             self.stdio.verbose("[remote host {0}] excute cmd = [{1}]".format(ssh_helper.get_name(), cmd))
-#             stdout = ssh_helper.ssh_exec_cmd(cmd)
-#             self.stdio.verbose("[remote host {0}] excute cmd = [{1}] complete, stdout=[{2}]".format(ssh_helper.get_name(), cmd, stdout))
-#             return stdout
-#         except Exception as e:
-#             self.stdio.error("[remote host {0}] excute cmd = [{1}] except: [{2}]".format(ssh_helper.get_name(), cmd, e))
-#
-#     def run_get_stderr(self, ssh_helper, cmd):
-#         try:
-#             self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd))
-#             std = ssh_helper.ssh_exec_cmd_get_stderr(cmd)
-#             return std
-#         except Exception as e:
-#             self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e))
-#
-#     def run_ignore_err(self, ssh_helper, cmd):
-#         try:
-#             self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd))
-#             std = ssh_helper.ssh_exec_cmd_ignore_err(cmd)
-#             return std
-#         except SSHException as e:
-#             self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e))
-
-
 def download_file(ssh_client, remote_path, local_path, stdio=None):
     """
     download file
@@ -99,7 +68,7 @@ def upload_file(ssh_client, local_path, remote_path, stdio=None):
     """
     stdio.verbose("Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format(ssh_client.get_name(), local_path, remote_path))
     try:
-        ssh_client.upload(local_path, remote_path)
+        ssh_client.upload(remote_path, local_path)
     except Exception as e:
         stdio.error("Upload File Failed error: {0}".format(e))
 
@@ -270,9 +239,9 @@ def get_observer_version(context):
                 ob_install_dir = nodes[0].get("home_path")
                 observer_version = get_observer_version_by_ssh(sshclient, ob_install_dir, stdio)
         except Exception as e:
-            raise Exception("get observer version fail.")
+            raise Exception("get observer version fail. Please check conf about observer's node or obconnector's info.")
     if observer_version == "":
-        raise Exception("get observer version fail.")
+        raise Exception("get observer version fail. Please check conf about observer's node or obconnector's info.")
     return observer_version
 
 
@@ -317,7 +286,8 @@ def get_obproxy_version(context):
     obproxy_version_info = ssh_client.exec_cmd(cmd)
     stdio.verbose("get obproxy version, run cmd = [{0}] ".format(cmd))
     if obproxy_version_info is not None:
-        ob_version = re.findall(r'[(]OceanBase.(.+? +?)[)]', obproxy_version_info)
+        pattern = r"(\d+\.\d+\.\d+\.\d+)"
+        ob_version = re.findall(pattern, obproxy_version_info)
         if len(ob_version) > 0:
             return ob_version[0]
         else:
@@ -326,7 +296,6 @@ def get_obproxy_version(context):
             stdio.verbose("get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format(cmd, obproxy_version_info))
             if "REVISION" not in obproxy_version_info:
                 raise Exception("Please check conf about proxy,{0}".format(obproxy_version_info))
-            pattern = r"(\d+\.\d+\.\d+\.\d+)"
             match = re.search(pattern, obproxy_version_info)
             if match:
                 obproxy_version_info = match.group(1)
@@ -436,12 +405,12 @@ def is_empty_file(ssh_client, file_path, stdio=None):
         return False
 
 
-def get_obdiag_display(log_dir, trace_id, stdio=None):
-    cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id)
-    stdout = LocalClient(stdio).run(cmd)
-    print_stdout = str(stdout).replace('\\n', '\n').replace('\\t', '\t')
-    if len(print_stdout) > 0:
-        print(print_stdout)
+# def get_obdiag_display(log_dir, trace_id, stdio=None):
+#     cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id)
+#     stdout = LocalClient(stdio).run(cmd)
+#     print_stdout = str(stdout).replace('\\n', '\n').replace('\\t', '\t')
+#     if len(print_stdout) > 0:
+#         print(print_stdout)
 
 
 def uzip_dir_local(uzip_dir, stdio=None):

common/config_helper.py

@@ -98,14 +98,14 @@ def build_configuration(self):
         self.save_old_configuration(old_config)
         # rewrite config
         ob_cluster_name = self.get_cluster_name()
-        print("\033[33mPlease enter the following configuration !!!\033[0m")
+        self.stdio.print("\033[33mPlease enter the following configuration !!!\033[0m")
         global_ssh_username = self.input_with_default("oceanbase host ssh username", "")
         global_ssh_password = self.input_password_with_default("oceanbase host ssh password", "")
         global_ssh_port = self.input_with_default("oceanbase host ssh_port", "22")
         global_home_path = self.input_with_default("oceanbase install home_path", const.OB_INSTALL_DIR_DEFAULT)
         default_data_dir = os.path.join(global_home_path, "store")
-        global_data_dir = self.input_with_default("oceanbase data_dir", default_data_dir)
-        global_redo_dir = self.input_with_default("oceanbase redo_dir", default_data_dir)
+        global_data_dir = default_data_dir
+        global_redo_dir = default_data_dir
         tenant_sys_config = {"user": self.sys_tenant_user, "password": self.sys_tenant_password}
         global_config = {"ssh_username": global_ssh_username, "ssh_password": global_ssh_password, "ssh_port": global_ssh_port, "ssh_key_file": "", "home_path": global_home_path, "data_dir": global_data_dir, "redo_dir": global_redo_dir}
         new_config = {"obcluster": {"ob_cluster_name": ob_cluster_name, "db_host": self.db_host, "db_port": self.db_port, "tenant_sys": tenant_sys_config, "servers": {"nodes": nodes_config, "global": global_config}}}