Merge branch 'master' of github.com:oraclebird/obdiag

� Conflicts:
�	core.py
�	diag_cmd.py
�	handler/analyzer/analyze_parameter.py
�	handler/analyzer/analyze_variable.py
�	handler/gather/gather_parameters.py
�	handler/gather/gather_variables.py

.github/workflows/build_package.yml

@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - master
+
 env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 

.github/workflows/test_sql_rule.yml

@@ -0,0 +1,30 @@
+name: Test Full Scan Rule
+
+on:
+  push:
+    branches: "*"
+  pull_request:
+    branches: "*"
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # Fetch all history for proper version detection
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements3.txt
+
+    - name: Run tests
+      run: python -m unittest discover -s test/analyzer/sql -p 'test_*.py'

.gitignore

@@ -1,4 +1,5 @@
 .idea/
+.vscode/
 venv/
 *.pyc
 *site-packages/

clean_all_result.sh

@@ -1,4 +1,4 @@
-rm -rf ./gather_pack_*
-rm -rf ./analyze_pack_*
-rm -rf ./analyze_flt_result*
-rm -rf ./check_report
+rm -rf ./obdiag_gather_pack_*
+rm -rf ./obdiag_analyze_pack_*
+rm -rf ./obdiag_analyze_flt_result*
+rm -rf ./obdiag_check_report

common/ob_connector.py

@@ -46,6 +46,16 @@ def init(self):
         except Exception as e:
             self.stdio.verbose(e)
 
+    def __enter__(self):
+        """Ensures the database connection is open upon entering the 'with' block."""
+        self._connect_to_db()
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        """Automatically closes the database connection when exiting the 'with' block."""
+        if self.connection:
+            self.connection.close()
+
     def _connect_db(self):
         try:
             self.conn = mysql.connect(
@@ -82,17 +92,28 @@ def execute_sql(self, sql):
         cursor.close()
         return ret
 
-    def execute_sql_return_columns_and_data(self, sql):
+    def execute_sql_return_columns_and_data(self, sql, params=None):
+        """
+        Executes an SQL query and returns column names and data.
+
+        :param sql: The SQL statement to execute, using %s as a placeholder for parameters.
+        :param parameters: A tuple or list of parameters to substitute into the SQL statement.
+        :return: A tuple containing a list of column names and a list of rows (each a tuple).
+        """
         if self.conn is None:
             self._connect_db()
         else:
             self.conn.ping(reconnect=True)
-        cursor = self.conn.cursor()
-        cursor.execute(sql)
-        column_names = [col[0] for col in cursor.description]
-        ret = cursor.fetchall()
-        cursor.close()
-        return column_names, ret
+
+        with self.conn.cursor() as cursor:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            column_names = [col[0] for col in cursor.description]
+            data = cursor.fetchall()
+        return column_names, data
 
     def execute_sql_return_cursor_dictionary(self, sql):
         if self.conn is None:

common/ssh_client/kubernetes_client.py

@@ -30,8 +30,10 @@ def __init__(self, context=None, node=None):
             self.container_name = self.node.get("container_name") or "observer"
             config_file = self.node.get("kubernetes_config_file")
             if config_file is None or config_file == "":
-                config.load_kube_config()
+                context.stdio.verbose("KubernetesClient load_kube_config from default config file in cluster.")
+                config.load_incluster_config()
             else:
+                context.stdio.verbose("KubernetesClient load_kube_config from {0}".format(config_file))
                 config.kube_config.load_kube_config(config_file=config_file)
             self.client = client.CoreV1Api()
         except Exception as e:

common/ssh_client/remote_client.py

@@ -42,7 +42,7 @@ def __init__(self, context, node):
         self._sftp_client = None
         self._disabled_rsa_algorithms = None
         self.host_ip = self.node.get("ip")
-        self.username = self.node.get("username")
+        self.username = self.node.get("ssh_username")
         self.ssh_port = self.node.get("ssh_port")
         self.need_password = True
         self.password = self.node.get("ssh_password")

common/ssh_client/ssh.py

@@ -29,16 +29,23 @@ def __init__(self, context=None, node=None):
             raise Exception("SshHelper init error: node is None")
         self.node = node
         self.context = context
+        self.stdio = None
+        if self.context is not None:
+            self.stdio = self.context.stdio
         self.ssh_type = node.get("ssh_type") or "remote"
         self.client = None
         self.init()
 
     def local_ip(self):
         local_ip_list = []
-        hostname = socket.gethostname()
-        addresses = socket.getaddrinfo(hostname, None)
-        for address in addresses:
-            local_ip_list.append(address[4][0])
+        try:
+            hostname = socket.gethostname()
+            addresses = socket.getaddrinfo(hostname, None)
+            for address in addresses:
+                local_ip_list.append(address[4][0])
+        except Exception as e:
+            if self.stdio is not None:
+                self.stdio.warn("get local ip warn: {} . Set local_ip Is 127.0.0.1".format(e))
         local_ip_list.append('127.0.0.1')
         return list(set(local_ip_list))
 

common/tool.py

@@ -49,7 +49,7 @@
 from datetime import timedelta
 from random import choice
 from io import BytesIO
-from copy import copy
+import copy
 from colorama import Fore, Style
 from ruamel.yaml import YAML
 from err import EC_SQL_EXECUTE_FAILED
@@ -1208,6 +1208,24 @@ def compare_versions_lower(v1, v2, stdio=None):
             return i < j
         return len(v1.split(".")) < len(v2.split("."))
 
+    @staticmethod
+    def mask_passwords(data):
+        # Make a deep copy of the data to avoid modifying the original
+        masked_data = copy.deepcopy(data)
+
+        if isinstance(masked_data, dict):
+            for key, value in masked_data.items():
+                if 'password' in key.lower():
+                    masked_data[key] = '*' * (len(value) if value else 1)
+                elif isinstance(value, (dict, list)):
+                    masked_data[key] = StringUtils.mask_passwords(value)
+        elif isinstance(masked_data, list):
+            for index, item in enumerate(masked_data):
+                if isinstance(item, (dict, list)):
+                    masked_data[index] = StringUtils.mask_passwords(item)
+
+        return masked_data
+
 
 class Cursor(SafeStdio):
 
@@ -1396,3 +1414,59 @@ def get_nodes_list(context, nodes, stdio=None):
                 return None
             return new_nodes
         return None
+
+
+class SQLUtil(object):
+    re_trace = re.compile(r'''\/\*.*trace_id((?!\/\*).)*rpc_id.*\*\/''', re.VERBOSE)
+    re_annotation = re.compile(r'''\/\*((?!\/\*).)*\*\/''', re.VERBOSE)
+    re_interval = re.compile(
+        r'''interval\s?(\?|\-?\d+)\s?(day|hour|minute|second|microsecond|week|month|quarter|year|second_microsecond|minute_microsecond|minute_second|hour_microsecond|hour_second|hour_minute|day_microsecond|day_second|day_minute|day_hour|year_month)''',
+        re.VERBOSE,
+    )
+    re_force_index = re.compile(r'''force[\s]index[\s][(]\w+[)]''', re.VERBOSE)
+    re_cast_1 = re.compile(r'''cast\(.*?\(.*?\)\)''', re.VERBOSE)
+    re_cast_2 = re.compile(r'''cast\(.*?\)''', re.VERBOSE)
+    re_now = re.compile(r'''now\(\)''', re.VERBOSE)
+
+    def remove_sql_text_affects_parser(self, sql):
+        sql = sql.lower().strip()
+        sql = self.remove_hint_and_annotate(sql)
+        sql = self.remove_force_index(sql)
+        sql = self.remove_now_in_insert(sql)
+        sql = self.remove_semicolon(sql)
+        return sql
+
+    def remove_hint_and_annotate(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_annotation, '', sql)
+        sql = re.sub(self.re_trace, '', sql)
+        return sql
+
+    def replace_interval_day(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_interval, '?', sql)
+        return sql
+
+    def remove_force_index(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_force_index, '', sql)
+        return sql
+
+    def remove_cast(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_cast_1, '?', sql)
+        sql = re.sub(self.re_cast_2, '?', sql)
+        return sql
+
+    def remove_now_in_insert(self, sql):
+        sql = sql.lower().lstrip()
+        if sql.startswith('insert'):
+            sql = re.sub(self.re_now, '?', sql)
+        return sql
+
+    def remove_semicolon(self, sql):
+        sql = sql.strip()
+        return sql[:-1] if sql[-1] == ';' else sql
+
+    def get_db_id(self, database_alias, user_id):
+        return database_alias + '-' + user_id

core.py

@@ -34,6 +34,10 @@
 from handler.analyzer.analyze_log import AnalyzeLogHandler
 from handler.analyzer.analyze_parameter import AnalyzeParameterHandler
 from handler.analyzer.analyze_variable import AnalyzeVariableHandler
+from handler.analyzer.analyze_sql import AnalyzeSQLHandler
+from handler.analyzer.analyze_sql_review import AnalyzeSQLReviewHandler
+from handler.analyzer.analyze_parameter import AnalyzeParameterHandler
+from handler.analyzer.analyze_variable import AnalyzeVariableHandler
 from handler.checker.check_handler import CheckHandler
 from handler.checker.check_list import CheckListHandler
 from handler.gather.gather_log import GatherLogHandler
@@ -48,6 +52,9 @@
 from handler.gather.scenes.list import GatherScenesListHandler
 from handler.gather.gather_parameters import GatherParametersHandler
 from handler.gather.gather_variables import GatherVariablesHandler
+from handler.gather.gather_tabledump import GatherTableDumpHandler
+from handler.gather.gather_parameters import GatherParametersHandler
+from handler.gather.gather_variables import GatherVariablesHandler
 from telemetry.telemetry import telemetry
 from update.update import UpdateHandler
 from colorama import Fore, Style
@@ -234,6 +241,15 @@ def gather_function(self, function_type, opt):
             elif function_type == 'gather_ash_report':
                 handler = GatherAshReportHandler(self.context)
                 return handler.handle()
+            elif function_type == 'gather_tabledump':
+                handler = GatherTableDumpHandler(self.context)
+                return handler.handle()
+            elif function_type == 'gather_parameters':
+                handler = GatherParametersHandler(self.context)
+                return handler.handle()
+            elif function_type == 'gather_variables':
+                handler = GatherVariablesHandler(self.context)
+                return handler.handle()
             elif function_type == 'gather_parameters':
                 handler = GatherParametersHandler(self.context)
                 return handler.handle()
@@ -290,6 +306,26 @@ def analyze_fuction(self, function_type, opt):
                 self.set_context(function_type, 'analyze', config)
                 handler = AnalyzeVariableHandler(self.context)
                 handler.handle()
+            elif function_type == 'analyze_sql':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeSQLHandler(self.context)
+                handler.handle()
+            elif function_type == 'analyze_sql_review':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeSQLReviewHandler(self.context)
+                handler.handle()
+            elif function_type == 'analyze_parameter_non_default':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeParameterHandler(self.context, 'non_default')
+                handler.handle()
+            elif function_type == 'analyze_parameter_diff':
+                self.set_context_skip_cluster_conn(function_type, 'analyze', config)
+                handler = AnalyzeParameterHandler(self.context, 'diff')
+                handler.handle()
+            elif function_type == 'analyze_variable':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeVariableHandler(self.context)
+                handler.handle()
             else:
                 self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type))
                 return False