2.5.0 gather redact without passwd (oceanbase#481)

* delete useless data

* check glibc version

* check glibc version

* check glibc version

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

* gather redact support zip_passwd

handler/gather/gather_log.py

@@ -52,6 +52,7 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False):
         self.zip_encrypt = False
         self.is_scene = is_scene
         self.config_path = const.DEFAULT_CONFIG_PATH
+        self.zip_password = None
         if self.context.get_variable("gather_timestamp", None):
             self.gather_timestamp = self.context.get_variable("gather_timestamp")
         else:
@@ -190,7 +191,7 @@ def handle_from_node(node):
                 self.stdio.verbose("redact_option is {0}".format(self.redact))
                 redact_dir = "{0}_redact".format(pack_dir_this_command)
                 self.redact_dir = redact_dir
-                redact = Redact(self.context, self.pack_dir_this_command, redact_dir)
+                redact = Redact(self.context, self.pack_dir_this_command, redact_dir, zip_password=self.zip_password)
                 redact.redact_files(self.redact)
                 self.stdio.print("redact success the log save on {0}".format(self.redact_dir))
                 return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"store_dir": redact_dir})
@@ -357,6 +358,7 @@ def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_com
         self.stdio.start_loading('[ip: {0}] zip observer log start'.format(ssh_client.get_name()))
         if self.zip_encrypt:
             zip_password = Util.gen_password(16)
+            self.zip_password = zip_password
             zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
         else:
             zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)

handler/gather/plugins/redact.py

@@ -7,12 +7,13 @@
 
 
 class Redact:
-    def __init__(self, context, input_file_dir, output_file_dir):
+    def __init__(self, context, input_file_dir, output_file_dir, zip_password=None):
         self.context = context
         self.stdio = context.stdio
         self.redacts = {}
         self.input_file_dir = input_file_dir
         self.output_file_dir = output_file_dir
+        self.zip_password = zip_password
         self.module_dir = os.path.expanduser('~/.obdiag/gather/redact')
         self.inner_config = self.context.inner_config
 
@@ -55,14 +56,20 @@ def redact_files(self, input_redacts):
                 self.stdio.verbose("open zip file: {0}".format(os.path.join(self.input_file_dir, zip_file)))
                 with zipfile.ZipFile(os.path.join(self.input_file_dir, zip_file), 'r') as zip_ref:
                     # Extract all files to the current directory
-                    zip_ref.extractall(self.input_file_dir)
+                    if self.zip_password is not None:
+                        zip_ref.extractall(self.input_file_dir, pwd=self.zip_password.encode('utf-8'))
+                    else:
+                        zip_ref.extractall(self.input_file_dir)
         gather_log_files = []
         for file_name in os.listdir(self.input_file_dir):
             if "zip" not in file_name and "result_summary.txt" not in file_name:
                 log_dir = os.path.join(self.input_file_dir, file_name)
                 for log_file in os.listdir(log_dir):
                     gather_log_files.append(os.path.join(log_dir, log_file))
                     self.stdio.verbose("result_log_files add {0}".format(os.path.join(log_dir, log_file)))
+        if len(gather_log_files) == 0:
+            self.stdio.warn("No log file found. The redact process will be skipped.")
+            return False
         file_queue = []
         max_processes = int(self.inner_config.get('gather').get('redact_processing_num')) or 3
         self.stdio.verbose("max_processes: {0}".format(max_processes))
@@ -78,16 +85,24 @@ def redact_files(self, input_redacts):
             file_queue.append(file_thread)
         for file_thread in file_queue:
             file_thread.join()
+        # delete gather_log_files
+        self.stdio.verbose("redact end. delete all gather_log_files")
+        for file_name in gather_log_files:
+            self.stdio.verbose("delete file: {0}".format(file_name))
+            os.remove(file_name)
         # zip the dir by node
         subfolders = [f for f in os.listdir(self.output_file_dir) if os.path.isdir(os.path.join(self.output_file_dir, f))]
         for subfolder in subfolders:
             subfolder_path = os.path.join(self.output_file_dir, subfolder)
             zip_filename = os.path.join(self.output_file_dir, f"{subfolder}.zip")
+            if self.zip_password is not None:
+                self.stdio.warn("the redacted log without passwd")
             with zipfile.ZipFile(zip_filename, 'w') as zipf:
                 for root, dirs, files in os.walk(subfolder_path):
                     for file in files:
                         file_path = os.path.join(root, file)
                         zipf.write(file_path, os.path.relpath(file_path, subfolder_path))
+                self.stdio.verbose("zip the redact log with passwd: {0}".format(self.zip_password.encode('utf-8')))
             self.stdio.verbose("delete the dir: {0}".format(subfolder_path))
             shutil.rmtree(subfolder_path)
             self.stdio.print(f"{subfolder} is zipped on {zip_filename}")