diff --git a/.github/workflows/build_base_docker.yml b/.github/workflows/build_base_docker.yml new file mode 100644 index 00000000..2cd8de4b --- /dev/null +++ b/.github/workflows/build_base_docker.yml @@ -0,0 +1,35 @@ +name: build docker helper + +on: + push: + tags: + - 'helper-[0-9]+.[0-9]+.[0-9]+' + +jobs: + build-docker: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Create and push manifests w/o cache + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + file: build/DockerFile + push: true + tags: | + ${{ vars.DOCKER_PUSH_BASE }}/obdiag-builder:latest diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 6287fa6e..c595dc05 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -86,4 +86,4 @@ jobs: with: name: obdiag-deb-package path: ./oceanbase-diagnostic-tool_*.deb - retention-days: 3 \ No newline at end of file + retention-days: 3 diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 00000000..0f7753c6 --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,12 @@ +FROM centos:7 +RUN rm -rf /etc/yum.repos.d/* +RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo +RUN yum install -y wget rpm-build +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --no-check-certificate +RUN sh Miniconda3-latest-Linux-x86_64.sh -p /opt/miniconda3 -b +RUN export PATH=/opt/miniconda3/bin:$PATH +RUN /opt/miniconda3/bin/conda init +RUN /opt/miniconda3/bin/conda create --name obdiag python=3.8 -y +RUN source /opt/miniconda3/bin/activate obdiag +RUN /opt/miniconda3/envs/obdiag/bin/python3.8 -m pip install --upgrade pip setuptools wheel +RUN yum install -y gcc gcc-c++ make \ No newline at end of file diff --git a/common/command.py b/common/command.py index 8e08df6b..9e1ded07 100644 --- a/common/command.py +++ b/common/command.py @@ -193,6 +193,20 @@ def zip_dir(ssh_client, father_dir, zip_dir, stdio=None): ssh_client.exec_cmd(cmd) +def tar_gz_dir(ssh_client, father_dir, tar_dir, stdio=None): + """ + Compress files through tar.gz on a remote server + :param ssh_client: An instance of SSHClient + :param father_dir: The parent directory containing the directory to be compressed + :param tar_dir: The directory to be compressed + :param stdio: An optional stream for logging output + :return: None + """ + tar_gz_file = f"{tar_dir}.tar.gz" + cmd = f"cd {father_dir} && tar -zcvf {tar_gz_file} {tar_dir} && rm {tar_dir}" + ssh_client.exec_cmd(cmd) + + def zip_encrypt_dir(ssh_client, zip_password, father_dir, zip_dir, stdio=None): """ Compress files by encryption diff --git a/common/tool.py b/common/tool.py index efb17eab..aa12d4a2 100644 --- a/common/tool.py +++ b/common/tool.py @@ -46,6 +46,10 @@ import ast import lzma import pymysql as mysql +import shutil +import tarfile +import pyminizip +import os from datetime import timedelta from random import choice from io import BytesIO @@ -655,6 +659,61 @@ def write_append(filename, result, stdio=None): with io.open(filename, 'a', encoding='utf-8') as fileobj: fileobj.write(u'{}'.format(result)) + def tar_gz_to_zip(temp_dir, tar_gz_file, output_zip, password, stdio): + extract_dir = os.path.join(temp_dir, 'extracted_files') + + try: + # 1. Extract the tar.gz file + with tarfile.open(tar_gz_file, 'r:gz') as tar: + tar.extractall(path=extract_dir) + stdio.verbose("tar.gz file extracted to {0}".format(extract_dir)) + + # 2. Gather all extracted files and their relative paths + files_to_compress = [] + base_paths = [] + for root, dirs, files in os.walk(extract_dir): + for file in files: + file_path = os.path.join(root, file) + base_path = os.path.basename(root) + files_to_compress.append(file_path) + base_paths.append(base_path) + + # 3. Compress the extracted files into a (possibly) encrypted zip file + if password: + # Use pyminizip to create the encrypted zip file + pyminizip.compress_multiple(files_to_compress, base_paths, output_zip, password, 5) # 5 is the compression level + stdio.verbose("extracted files compressed into encrypted {0}".format(output_zip)) + else: + # Create an unencrypted zip file + pyminizip.compress_multiple(files_to_compress, base_paths, output_zip, None, 5) + stdio.verbose("extracted files compressed into unencrypted {0}".format(output_zip)) + + # 4. Remove the extracted directory + shutil.rmtree(extract_dir) + stdio.verbose("extracted directory {0} removed".format(extract_dir)) + + # 5. Optionally remove the original tar.gz file + os.remove(tar_gz_file) + stdio.verbose("original tar.gz file {0} removed".format(tar_gz_file)) + + except tarfile.TarError as te: + stdio.exception("tar file error: {0}".format(te)) + if os.path.exists(extract_dir): + shutil.rmtree(extract_dir) + return False + except pyminizip.compress_error as ce: + stdio.exception("compression error: {0}".format(ce)) + if os.path.exists(extract_dir): + shutil.rmtree(extract_dir) + return False + except Exception as e: + stdio.exception("an error occurred: {0}".format(e)) + if os.path.exists(extract_dir): + shutil.rmtree(extract_dir) + return False + + return True + class YamlLoader(YAML): diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 89200b9f..e9fa0132 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -23,7 +23,7 @@ from handler.base_shell_handler import BaseShellHandler from common.obdiag_exception import OBDIAGFormatException from common.constant import const -from common.command import get_file_size, download_file, is_empty_dir, rm_rf_file, get_logfile_name_list, mkdir, delete_empty_file, zip_encrypt_dir, zip_dir +from common.command import get_file_size, download_file, is_empty_dir, rm_rf_file, get_logfile_name_list, mkdir, delete_empty_file, tar_gz_dir from common.command import SshClient from common.tool import TimeUtils from common.tool import Util @@ -365,21 +365,20 @@ def __pharse_log(self, ssh_client, home_path, log_name, gather_path): def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_command): zip_password = "" gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name) - self.stdio.print('[ip: {0}] zip observer log start'.format(ssh_client.get_name())) + self.stdio.print('[ip: {0}] gather observer log start'.format(ssh_client.get_name())) if self.zip_encrypt: zip_password = Util.gen_password(16) - self.zip_password = zip_password - zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) - else: - zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) - self.stdio.print('[{0}] zip observer log end'.format(ssh_client.get_name())) - gather_package_dir = "{0}.zip".format(gather_dir_full_path) + tar_gz_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + self.stdio.print('[{0}] gather observer log end'.format(ssh_client.get_name())) + gather_package_dir = "{0}.tar.gz".format(gather_dir_full_path) gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio) self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size)) local_store_path = "" if int(gather_log_file_size) < self.file_size_limit: + local_store_tar_gz_file = pack_dir_this_command + "/{0}.tar.gz".format(gather_dir_name) + download_file(ssh_client, gather_package_dir, local_store_tar_gz_file, self.stdio) local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) - download_file(ssh_client, gather_package_dir, local_store_path, self.stdio) + FileUtil.tar_gz_to_zip(pack_dir_this_command, local_store_tar_gz_file, local_store_path, zip_password, self.stdio) resp["error"] = "" resp["zip_password"] = zip_password else: diff --git a/handler/gather/gather_obadmin.py b/handler/gather/gather_obadmin.py index 47e0d271..96e9b415 100644 --- a/handler/gather/gather_obadmin.py +++ b/handler/gather/gather_obadmin.py @@ -24,7 +24,7 @@ from common.constant import const from common.command import LocalClient, SshClient, is_empty_dir from handler.base_shell_handler import BaseShellHandler -from common.command import download_file, rm_rf_file, get_file_size, zip_encrypt_dir, zip_dir, get_observer_version +from common.command import download_file, rm_rf_file, get_file_size, get_observer_version, tar_gz_dir from common.tool import TimeUtils from common.tool import StringUtils from common.tool import Util @@ -221,15 +221,16 @@ def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_com gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) - else: - zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) - gather_package_dir = "{0}.zip".format(gather_dir_full_path) + tar_gz_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + gather_package_dir = "{0}.tar.gz".format(gather_dir_full_path) gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio) self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size, self.stdio)) local_path = "" if int(gather_log_file_size) < self.file_size_limit: - local_path = download_file(ssh_client, gather_package_dir, pack_dir_this_command, self.stdio) + local_store_tar_gz_file = pack_dir_this_command + "/{0}.tar.gz".format(gather_dir_name) + download_file(ssh_client, gather_package_dir, local_store_tar_gz_file, self.stdio) + local_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) + FileUtil.tar_gz_to_zip(pack_dir_this_command, local_store_tar_gz_file, local_path, zip_password, self.stdio) resp["error"] = "" resp["zip_password"] = zip_password else: diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index 6706e061..cc9a7539 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -26,7 +26,7 @@ from common.obdiag_exception import OBDIAGFormatException from common.command import SshClient from common.constant import const -from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, rm_rf_file, zip_encrypt_dir, zip_dir +from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, rm_rf_file, tar_gz_dir from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil @@ -306,17 +306,16 @@ def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_com gather_dir_full_path = "{0}/{1}".format(self.gather_log_temporary_dir, gather_dir_name) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(ssh_client, zip_password, self.gather_log_temporary_dir, gather_dir_name, self.stdio) - else: - zip_dir(ssh_client, self.gather_log_temporary_dir, gather_dir_name, self.stdio) - gather_package_dir = "{0}.zip".format(gather_dir_full_path) - + tar_gz_dir(ssh_client, self.gather_log_temporary_dir, gather_dir_name, self.stdio) + gather_package_dir = "{0}.tar.gz".format(gather_dir_full_path) gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio) self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size)) local_path = "" if int(gather_log_file_size) < self.file_size_limit: - local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) - local_path = download_file(ssh_client, gather_package_dir, local_store_path, self.stdio) + local_store_tar_gz_file = pack_dir_this_command + "/{0}.tar.gz".format(gather_dir_name) + download_file(ssh_client, gather_package_dir, local_store_tar_gz_file, self.stdio) + local_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) + FileUtil.tar_gz_to_zip(pack_dir_this_command, local_store_tar_gz_file, local_path, zip_password, self.stdio) resp["error"] = "" resp["zip_password"] = zip_password else: diff --git a/handler/gather/gather_obstack2.py b/handler/gather/gather_obstack2.py index b57e6f60..f71afe73 100644 --- a/handler/gather/gather_obstack2.py +++ b/handler/gather/gather_obstack2.py @@ -22,7 +22,7 @@ import tabulate -from common.command import download_file, is_empty_dir, is_support_arch, get_observer_version, get_observer_pid, mkdir, zip_dir, get_file_size, delete_file_force, is_empty_file, upload_file +from common.command import download_file, is_empty_dir, is_support_arch, get_observer_version, get_observer_pid, mkdir, tar_gz_dir, get_file_size, delete_file_force, is_empty_file, upload_file from common.constant import const from common.command import LocalClient, SshClient from handler.base_shell_handler import BaseShellHandler @@ -167,15 +167,17 @@ def __handle_from_node(self, local_stored_path, node): if is_empty_dir(ssh_client, "/tmp/{0}".format(remote_dir_name), self.stdio): resp["error"] = "gather failed, folder is empty" return resp + tar_gz_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) + remote_tar_file_path = "{0}.tar.gz".format(remote_dir_full_path) - zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) - remote_zip_file_path = "{0}.zip".format(remote_dir_full_path) - - file_size = get_file_size(ssh_client, remote_zip_file_path, self.stdio) - remote_file_full_path = "{0}.zip".format(remote_dir_full_path) + file_size = get_file_size(ssh_client, remote_tar_file_path, self.stdio) + remote_file_full_path = "{0}.tar.gz".format(remote_dir_full_path) if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio) + local_tar_file_path = "{0}/{1}.tar.gz".format(local_stored_path, remote_dir_name) + self.stdio.verbose("local tar file path {0}...".format(local_tar_file_path)) + download_file(ssh_client, remote_file_full_path, local_tar_file_path, self.stdio) + local_zip_file_path = local_stored_path + "/{0}.zip".format(remote_dir_name) + FileUtil.tar_gz_to_zip(local_stored_path, local_tar_file_path, local_zip_file_path, None, self.stdio) resp["error"] = "" else: resp["error"] = "File too large" diff --git a/handler/gather/gather_perf.py b/handler/gather/gather_perf.py index 84b217ce..330b8ead 100644 --- a/handler/gather/gather_perf.py +++ b/handler/gather/gather_perf.py @@ -21,7 +21,7 @@ import tabulate -from common.command import get_observer_pid, mkdir, zip_dir, get_file_size, download_file, delete_file_force +from common.command import get_observer_pid, mkdir, tar_gz_dir, get_file_size, download_file, delete_file_force from common.command import SshClient from common.constant import const from handler.base_shell_handler import BaseShellHandler @@ -151,12 +151,15 @@ def __handle_from_node(self, node, local_stored_path): self.__gather_perf_flame(ssh_client, remote_dir_full_path, pid_observer) self.__gather_top(ssh_client, remote_dir_full_path, pid_observer) - zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) - remote_file_full_path = "{0}.zip".format(remote_dir_full_path) + tar_gz_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) + remote_file_full_path = "{0}.tar.gz".format(remote_dir_full_path) file_size = get_file_size(ssh_client, remote_file_full_path, self.stdio) if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio) + local_tar_file_path = "{0}/{1}.tar.gz".format(local_stored_path, remote_dir_name) + self.stdio.verbose("local tar file path {0}...".format(local_tar_file_path)) + download_file(ssh_client, remote_file_full_path, local_tar_file_path, self.stdio) + local_zip_file_path = local_stored_path + "/{0}.zip".format(remote_dir_name) + FileUtil.tar_gz_to_zip(local_stored_path, local_tar_file_path, local_zip_file_path, None, self.stdio) resp["error"] = "" else: resp["error"] = "File too large" diff --git a/handler/gather/gather_sysstat.py b/handler/gather/gather_sysstat.py index f78d0af8..da4d413b 100644 --- a/handler/gather/gather_sysstat.py +++ b/handler/gather/gather_sysstat.py @@ -22,7 +22,7 @@ import tabulate from common.constant import const from common.command import LocalClient, SshClient -from common.command import get_file_size, download_file, mkdir, zip_dir +from common.command import get_file_size, download_file, mkdir, tar_gz_dir from handler.base_shell_handler import BaseShellHandler from common.tool import Util from common.tool import DirectoryUtil @@ -147,13 +147,15 @@ def __handle_from_node(self, node, local_stored_path): self.__gather_io_info(ssh_client, remote_dir_full_path) self.__gather_traffic_info(ssh_client, remote_dir_full_path) self.__gather_tcp_udp_info(ssh_client, remote_dir_full_path) - zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) - remote_file_full_path = "{0}.zip".format(remote_dir_full_path) + tar_gz_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) + remote_file_full_path = "{0}.tar.gz".format(remote_dir_full_path) file_size = get_file_size(ssh_client, remote_file_full_path, self.stdio) if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - self.stdio.verbose("local file path {0}...".format(local_file_path)) - download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio) + local_tar_file_path = "{0}/{1}.tar.gz".format(local_stored_path, remote_dir_name) + self.stdio.verbose("local tar file path {0}...".format(local_tar_file_path)) + download_file(ssh_client, remote_file_full_path, local_tar_file_path, self.stdio) + local_zip_file_path = local_stored_path + "/{0}.zip".format(remote_dir_name) + FileUtil.tar_gz_to_zip(local_stored_path, local_tar_file_path, local_zip_file_path, None, self.stdio) resp["error"] = "" else: resp["error"] = "File too large" diff --git a/requirements3.txt b/requirements3.txt index e80cc1dc..0fa4ff3a 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -39,3 +39,4 @@ netifaces==0.11.0 netifaces==0.11.0 kubernetes==30.1.0 setuptools==65.6.3 +pyminizip==0.2.6