diff --git a/common/command.py b/common/command.py index f62134f8..71755cfe 100644 --- a/common/command.py +++ b/common/command.py @@ -93,19 +93,6 @@ def delete_file_in_folder(ssh_client, file_path, stdio=None): ssh_client.exec_cmd(cmd) -def is_empty_dir(ssh_client, dir, stdio=None): - """ - determine whether it is an empty folder - :return: true or false - """ - cmd = "ls -A {gather_path}|wc -w".format(gather_path=dir) - file_num = ssh_client.exec_cmd(cmd) - if int(file_num) == 0: - return True - else: - return False - - def get_file_start_time(ssh_client, file_name, dir, stdio=None): """ get log file start time @@ -367,15 +354,6 @@ def delete_file_force(ssh_client, file_name, stdio=None): ssh_client.exec_cmd(cmd) -def delete_empty_file(ssh_client, file_path, stdio=None): - """ - delete empty file - :return: - """ - cmd = "find {file_path} -name '*' -type f -size 0c | xargs -n 1 rm -f".format(file_path=file_path) - ssh_client.exec_cmd(cmd) - - def delete_file(ssh_client, file_path, stdio=None): """ delete file diff --git a/common/ssh.py b/common/ssh.py index ccd8a40c..699d5d2c 100644 --- a/common/ssh.py +++ b/common/ssh.py @@ -22,26 +22,17 @@ import os import tempfile import warnings -import sys -import paramiko -import time -import docker from glob import glob from paramiko import AuthenticationException, SFTPClient from paramiko.client import SSHClient, AutoAddPolicy from paramiko.ssh_exception import NoValidConnectionsError, SSHException -from multiprocessing.queues import Empty from multiprocessing import Queue from multiprocessing.pool import ThreadPool from common.tool import COMMAND_ENV, DirectoryUtil, FileUtil, Timeout -from common.obdiag_exception import OBDIAGSSHConnException -from common.obdiag_exception import OBDIAGShellCmdException -from common.tool import StringUtils -from common.tool import TimeUtils from stdio import SafeStdio from err import EC_SSH_CONNECT -from subprocess32 import Popen, PIPE +from subprocess import Popen, PIPE warnings.filterwarnings("ignore") @@ -761,44 +752,3 @@ def _filter_dir_in_file_path(files, directories): directories.remove(dir_name) skip_directories.append(dir_name) dir_name = os.path.dirname(dir_name) - - def file_downloader(self, local_dir, remote_dir, stdio=None): - try: - client = SshClient(config=self.config, stdio=None) - client._open_sftp(stdio=stdio) - client._remote_transporter = self.remote_transporter - while True: - remote_path = self.task_queue.get(block=False) - local_path = os.path.join(local_dir, os.path.relpath(remote_path, remote_dir)) - if client.get_file(local_path, remote_path, stdio=stdio): - self.result_queue.put(remote_path) - else: - stdio.error('Fail to get %s' % remote_path) - except Empty: - return - except: - stdio.exception("") - stdio.exception('Failed to get %s' % remote_dir) - - def file_uploader(self, local_dir, remote_dir, stdio=None): - try: - client = SshClient(config=self.config, stdio=None) - client._remote_transporter = self.remote_transporter - while True: - local_path, is_dir = self.task_queue.get(block=False) - remote_path = os.path.join(remote_dir, os.path.relpath(local_path, local_dir)) - if is_dir: - stat = oct(os.stat(local_path).st_mode)[-3:] - cmd = '[ -d "{remote_path}" ] || (mkdir -p {remote_path}; chmod {stat} {remote_path})'.format(remote_path=remote_path, stat=stat) - if client.execute_command(cmd): - self.result_queue.put(remote_path) - else: - if client.put_file(local_path, remote_path, stdio=stdio): - self.result_queue.put(remote_path) - else: - stdio.error('Fail to get %s' % remote_path) - except Empty: - return - except: - stdio.exception("") - stdio.verbose('Failed to get %s' % remote_dir) diff --git a/common/ssh_client/local_client.py b/common/ssh_client/local_client.py index 71e55b6d..59797e9f 100644 --- a/common/ssh_client/local_client.py +++ b/common/ssh_client/local_client.py @@ -17,7 +17,7 @@ """ from common.ssh_client.base import SsherClient -import subprocess32 as subprocess +import subprocess import shutil diff --git a/common/tool.py b/common/tool.py index 1a03f949..4d47a1d5 100644 --- a/common/tool.py +++ b/common/tool.py @@ -16,9 +16,6 @@ """ from __future__ import absolute_import, division, print_function -import multiprocessing as mp - -import os import io import bz2 import random @@ -27,26 +24,19 @@ import gzip import fcntl import signal -import shutil import re -import json import hashlib -import datetime import uuid import tabulate -import tarfile import socket import requests import decimal -import time import json import time -import traceback import datetime import string import oyaml as yaml -import ast import lzma import pymysql as mysql import shutil @@ -66,7 +56,7 @@ encoding_open = open -__all__ = ("Timeout", "DynamicLoading", "ConfigUtil", "DirectoryUtil", "FileUtil", "YamlLoader", "OrderedDict", "COMMAND_ENV", "TimeUtils", "NetUtils", "StringUtils", "YamlUtils", "Util") +__all__ = ("Timeout", "DynamicLoading", "ConfigUtil", "DirectoryUtil", "FileUtil", "YamlLoader", "COMMAND_ENV", "TimeUtils", "NetUtils", "StringUtils", "YamlUtils", "Util") _WINDOWS = os.name == 'nt' diff --git a/common/types.py b/common/types.py index 67576319..b07a94ee 100644 --- a/common/types.py +++ b/common/types.py @@ -21,7 +21,6 @@ import os import re import uuid -import traceback __all__ = ("Moment", "Time", "Capacity", "CapacityWithB", "CapacityMB", "StringList", "Dict", "List", "StringOrKvList", "Double", "Boolean", "Integer", "String", "Path", "SafeString", "PathList", "SafeStringList", "DBUrl", "WebUrl", "OBUser") diff --git a/core.py b/core.py index 5fe6210b..0a2e0c80 100644 --- a/core.py +++ b/core.py @@ -64,7 +64,6 @@ from common.tool import TimeUtils, Util from common.command import get_observer_version_by_sql from common.ob_connector import OBConnector -from collections import OrderedDict class ObdiagHome(object): @@ -173,7 +172,7 @@ def update_obcluster_nodes(self, config): if not all(ob_cluster.values()) or not all(ob_cluster['tenant_sys'].values()): raise ValueError("Missing required configuration values in ob_cluster or tenant_sys") - if (obcluster := config_data.get('obcluster')) and (servers := obcluster.get('servers')) and servers.get('nodes'): + if config_data.get('obcluster') and config_data.get('obcluster').get('servers') and config_data.get('obcluster').get('servers').get('nodes'): return ob_version = get_observer_version_by_sql(self.context, ob_cluster) @@ -548,7 +547,7 @@ def update(self, opts): self.stdio.print("update start ...") self.set_offline_context('update', 'update') handler = UpdateHandler(self.context) - return handler.execute() + return handler.handle() def config(self, opt): config = self.config_manager diff --git a/handler/analyzer/analyze_index_space.py b/handler/analyzer/analyze_index_space.py index 5909e275..abcc868d 100644 --- a/handler/analyzer/analyze_index_space.py +++ b/handler/analyzer/analyze_index_space.py @@ -17,7 +17,6 @@ """ -import sys from prettytable import PrettyTable from common.tool import StringUtils, Util from common.ob_connector import OBConnector diff --git a/handler/analyzer/analyze_queue.py b/handler/analyzer/analyze_queue.py index fc26a46c..6a0b721d 100644 --- a/handler/analyzer/analyze_queue.py +++ b/handler/analyzer/analyze_queue.py @@ -17,7 +17,6 @@ """ import datetime import os -import re import csv from tabulate import tabulate from common.command import get_observer_version_by_sql @@ -27,7 +26,7 @@ from common.command import SshClient from common.ob_log_level import OBLogLevel from common.command import download_file, get_logfile_name_list, mkdir, delete_file -from common.tool import DirectoryUtil, TimeUtils, Util, StringUtils +from common.tool import StringUtils from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil diff --git a/handler/analyzer/analyze_sql_review.py b/handler/analyzer/analyze_sql_review.py index 03245d38..0afdf67d 100644 --- a/handler/analyzer/analyze_sql_review.py +++ b/handler/analyzer/analyze_sql_review.py @@ -19,14 +19,12 @@ import time import sqlparse from tabulate import tabulate -from colorama import Fore, Style from common.constant import const from common.tool import StringUtils, Util from common.tool import TimeUtils from common.tool import FileUtil from common.ob_connector import OBConnector from handler.analyzer.sql.rule_manager import SQLReviewRuleManager -from handler.meta.sql_meta import GlobalSqlMeta from handler.meta.html_meta import GlobalHtmlMeta diff --git a/handler/checker/check_list.py b/handler/checker/check_list.py index fc016939..cc3d86fc 100644 --- a/handler/checker/check_list.py +++ b/handler/checker/check_list.py @@ -66,14 +66,14 @@ def handle(self): cases_map[package_data] = { "name": package_data, - "command": "obdiag check --{0}={1}".format(package_target, package_data), + "command": "obdiag check run --{0}={1}".format(package_target, package_data), "info_en": package_file_data[package_data].get("info_en") or "", "info_cn": package_file_data[package_data].get("info_cn") or "", } result_map[target]["commands"].append( { "name": package_data, - "command": "obdiag check --{0}={1}".format(package_target, package_data), + "command": "obdiag check run --{0}={1}".format(package_target, package_data), "info_en": package_file_data[package_data].get("info_en") or "", "info_cn": package_file_data[package_data].get("info_cn") or "", } diff --git a/handler/checker/check_task.py b/handler/checker/check_task.py index 83d0af28..16da9a97 100644 --- a/handler/checker/check_task.py +++ b/handler/checker/check_task.py @@ -17,7 +17,6 @@ """ import threading -from common.ob_connector import OBConnector from handler.checker.check_exception import StepResultFailException, StepExecuteFailException, StepResultFalseException, TaskException from handler.checker.step.stepbase import StepBase from common.tool import StringUtils diff --git a/handler/checker/result/result.py b/handler/checker/result/result.py index 52153c2e..b132479d 100644 --- a/handler/checker/result/result.py +++ b/handler/checker/result/result.py @@ -15,7 +15,7 @@ @file: result.py @desc: """ -from handler.checker.check_exception import ResultFalseException, ResultFailException, VerifyFailException +from handler.checker.check_exception import ResultFalseException, ResultFailException from handler.checker.result.verify import VerifyResult import re diff --git a/handler/checker/result/verify.py b/handler/checker/result/verify.py index 06dde6b0..ce21ea43 100644 --- a/handler/checker/result/verify.py +++ b/handler/checker/result/verify.py @@ -17,7 +17,7 @@ """ import decimal import re -import subprocess32 as subprocess +import subprocess from handler.checker.check_exception import VerifyFalseException, VerifyFailException from handler.meta.check_meta import GlobalCheckMeta from common.tool import StringUtils diff --git a/handler/checker/step/sleep.py b/handler/checker/step/sleep.py index 5ce45daf..e5fe2363 100644 --- a/handler/checker/step/sleep.py +++ b/handler/checker/step/sleep.py @@ -18,9 +18,6 @@ import time from handler.checker.check_exception import StepExecuteFailException -from common.ob_connector import OBConnector -from common.tool import StringUtils -from common.tool import Util class StepSleepHandler: diff --git a/handler/checker/step/sql.py b/handler/checker/step/sql.py index 3514859d..0fd475e7 100644 --- a/handler/checker/step/sql.py +++ b/handler/checker/step/sql.py @@ -17,7 +17,6 @@ """ from handler.checker.check_exception import StepExecuteFailException -from common.ob_connector import OBConnector from common.tool import StringUtils from common.tool import Util diff --git a/handler/checker/step/stepbase.py b/handler/checker/step/stepbase.py index 78461f29..f0b429ce 100644 --- a/handler/checker/step/stepbase.py +++ b/handler/checker/step/stepbase.py @@ -22,7 +22,6 @@ from handler.checker.result.result import CheckResult from handler.checker.step.ssh import SshHandler from handler.checker.step.sql import StepSQLHandler -import docker from common.ssh_client.ssh import SshClient diff --git a/handler/display/display_scenes.py b/handler/display/display_scenes.py index e20c876e..25bc0867 100644 --- a/handler/display/display_scenes.py +++ b/handler/display/display_scenes.py @@ -16,7 +16,6 @@ @desc: """ -import os import re from result_type import ObdiagResult from stdio import SafeStdio @@ -24,10 +23,8 @@ from handler.display.scenes.base import SceneBase from common.obdiag_exception import OBDIAGFormatException from handler.display.scenes.list import DisplayScenesListHandler -from common.tool import DirectoryUtil from common.tool import StringUtils from common.scene import get_version_by_type -from colorama import Fore, Style from common.tool import Util from common.tool import TimeUtils from common.ob_connector import OBConnector @@ -129,7 +126,7 @@ def __execute_code_task_one(self, task_name): try: self.stdio.verbose("execute tasks is {0}".format(task_name)) scene = {"name": task_name} - task = SceneBase(context=self.context, scene=scene, env=self.env, mode='code', task_type=task_name) + task = SceneBase(context=self.context, scene=scene, env=self.env, mode='code', task_type=task_name, db_connector=self.db_connector) self.stdio.verbose("{0} execute!".format(task_name)) task.execute() self.stdio.verbose("execute tasks end : {0}".format(task_name)) diff --git a/handler/display/scenes/base.py b/handler/display/scenes/base.py index a15b546b..936af7d2 100644 --- a/handler/display/scenes/base.py +++ b/handler/display/scenes/base.py @@ -18,11 +18,14 @@ from stdio import SafeStdio from common.scene import filter_by_version from handler.display.step.base import Base -from common.tool import StringUtils class SceneBase(SafeStdio): - def __init__(self, context, scene, db_connector, report_dir=None, scene_variable_dict={}, env={}, mode="yaml", task_type="observer"): + def __init__(self, context, scene, db_connector, report_dir=None, scene_variable_dict=None, env=None, mode="yaml", task_type="observer"): + if env is None: + env = {} + if scene_variable_dict is None: + scene_variable_dict = {} self.context = context self.stdio = context.stdio self.scene_variable_dict = scene_variable_dict diff --git a/handler/display/step/base.py b/handler/display/step/base.py index 9642a721..52141c79 100644 --- a/handler/display/step/base.py +++ b/handler/display/step/base.py @@ -17,12 +17,13 @@ """ from common.ssh_client.ssh import SshClient from stdio import SafeStdio -from handler.display.step.ssh import SshHandler from handler.display.step.sql import StepSQLHandler class Base(SafeStdio): - def __init__(self, context, step, node, cluster, task_variable_dict=None, env={}, node_number=1, db_connector=None): + def __init__(self, context, step, node, cluster, task_variable_dict=None, env=None, node_number=1, db_connector=None): + if env is None: + env = {} self.context = context self.stdio = context.stdio if task_variable_dict is None: diff --git a/handler/display/step/sql.py b/handler/display/step/sql.py index 370fccdd..a5ad8fc3 100644 --- a/handler/display/step/sql.py +++ b/handler/display/step/sql.py @@ -15,10 +15,7 @@ @file: sql.py @desc: """ -import os from stdio import SafeStdio -from common.ob_connector import OBConnector -from tabulate import tabulate from common.tool import StringUtils from prettytable import PrettyTable diff --git a/handler/gather/gather_obadmin.py b/handler/gather/gather_obadmin.py index 96e9b415..1b05d8eb 100644 --- a/handler/gather/gather_obadmin.py +++ b/handler/gather/gather_obadmin.py @@ -22,7 +22,7 @@ import tabulate from common.obdiag_exception import OBDIAGFormatException from common.constant import const -from common.command import LocalClient, SshClient, is_empty_dir +from common.command import SshClient, is_empty_dir from handler.base_shell_handler import BaseShellHandler from common.command import download_file, rm_rf_file, get_file_size, get_observer_version, tar_gz_dir from common.tool import TimeUtils diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index c6e90a72..bbec3e49 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -217,7 +217,7 @@ def __handle_from_node(self, node, pack_dir_this_command): return resp for log_name in log_list: self.__pharse_log(ssh_client=ssh_client, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) - delete_empty_file(ssh_client, gather_dir_full_path, self.stdio) + delete_empty_file(ssh_client, gather_dir_full_path) is_empty = is_empty_dir(ssh_client, gather_dir_full_path, self.stdio) if is_empty: diff --git a/handler/gather/gather_obstack2.py b/handler/gather/gather_obstack2.py index 6c80c1fb..fe30d9c0 100644 --- a/handler/gather/gather_obstack2.py +++ b/handler/gather/gather_obstack2.py @@ -24,13 +24,12 @@ from common.command import download_file, is_empty_dir, is_support_arch, get_observer_version, get_observer_pid, mkdir, zip_dir, get_file_size, delete_file_force, is_empty_file, upload_file from common.constant import const -from common.command import LocalClient, SshClient +from common.command import SshClient from handler.base_shell_handler import BaseShellHandler from common.tool import TimeUtils from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil -from common.tool import NetUtils from common.tool import StringUtils from result_type import ObdiagResult diff --git a/handler/gather/gather_plan_monitor.py b/handler/gather/gather_plan_monitor.py index 1656ed39..908c9a48 100644 --- a/handler/gather/gather_plan_monitor.py +++ b/handler/gather/gather_plan_monitor.py @@ -21,7 +21,6 @@ import shutil import time from decimal import Decimal -import pymysql as mysql import tabulate from prettytable import from_db_cursor from common.ob_connector import OBConnector diff --git a/handler/gather/gather_sysstat.py b/handler/gather/gather_sysstat.py index f78d0af8..ed7285a9 100644 --- a/handler/gather/gather_sysstat.py +++ b/handler/gather/gather_sysstat.py @@ -21,7 +21,7 @@ import tabulate from common.constant import const -from common.command import LocalClient, SshClient +from common.command import SshClient from common.command import get_file_size, download_file, mkdir, zip_dir from handler.base_shell_handler import BaseShellHandler from common.tool import Util diff --git a/handler/gather/gather_tabledump.py b/handler/gather/gather_tabledump.py index bffd9b80..f9c77c7b 100644 --- a/handler/gather/gather_tabledump.py +++ b/handler/gather/gather_tabledump.py @@ -24,11 +24,9 @@ from common.ob_connector import OBConnector from common.tool import StringUtils from common.command import get_observer_version -from colorama import Fore, Style from common.tool import Util from common.tool import TimeUtils from tabulate import tabulate -from colorama import Fore, Style class GatherTableDumpHandler(SafeStdio): @@ -39,8 +37,8 @@ def __init__(self, context, store_dir="./obdiag_gather_report", is_inner=False): self.report = None self.report_path = None self.ob_cluster = {} - self.ob_connector = {} - self.tenant_connector = {} + self.ob_connector = None + self.tenant_connector = None self.database = None self.table = None self.result_list = [] diff --git a/handler/gather/scenes/base.py b/handler/gather/scenes/base.py index d02845e0..e71aec9f 100644 --- a/handler/gather/scenes/base.py +++ b/handler/gather/scenes/base.py @@ -22,7 +22,11 @@ class SceneBase(SafeStdio): - def __init__(self, context, scene, report_dir=None, scene_variable_dict={}, env={}, mode="yaml", task_type="observer"): + def __init__(self, context, scene, report_dir=None, scene_variable_dict=None, env=None, mode="yaml", task_type="observer"): + if env is None: + env = {} + if scene_variable_dict is None: + scene_variable_dict = {} self.context = context self.stdio = context.stdio self.scene_variable_dict = scene_variable_dict diff --git a/handler/gather/step/ssh.py b/handler/gather/step/ssh.py index cdc1511b..859c0fc2 100644 --- a/handler/gather/step/ssh.py +++ b/handler/gather/step/ssh.py @@ -67,4 +67,4 @@ def report(self, command, data): f.write('\n\n' + '[' + self.node.get("ip") + '] shell > ' + command + '\n') f.write(data + '\n') except Exception as e: - self.stdio.error("report sql result to file: {0} failed, error: ".format(self.report_file_path)) + self.stdio.error("report sql result to file: {0} failed, error: {1}".format(self.report_file_path, e)) diff --git a/handler/rca/rca_handler.py b/handler/rca/rca_handler.py index 7ac87949..a51d8d8b 100644 --- a/handler/rca/rca_handler.py +++ b/handler/rca/rca_handler.py @@ -268,10 +268,6 @@ def get_scene_info(self): def export_result(self): return self.Result.export() - def save_data(self, data, save_path): - with open(save_path, "w") as f: - f.write(str(data)) - def get_all_tenants_id(self): try: if self.ob_connector is None: @@ -286,6 +282,7 @@ class Result: def __init__(self, context): # self.suggest = "" + self.record_file_name = "" self.records = [] self.context = context self.stdio = context.stdio @@ -297,6 +294,7 @@ def __init__(self, context): if self.context.get_variable("ob_cluster").get("db_host") is not None or len(self.context.cluster_config.get("servers")) > 0: self.version = get_version_by_type(self.context, "observer") except Exception as e: + self.stdio.verbose("rca get obcluster version fail. Maybe the scene need not it, skip it. Exception: {0}".format(e)) self.stdio.warn("rca get obcluster version fail. if the scene need not it, skip it") def set_save_path(self, save_path): @@ -345,8 +343,7 @@ def export_report_json(self): def export_report_xml(self): with open(self.record_file_name, 'w', encoding='utf-8') as f: - allreport = {} - allreport["report"] = self.records_data() + allreport = {"report": self.records_data()} json_str = json.dumps(allreport) xml_str = xmltodict.unparse(json.loads(json_str)) f.write(xml_str)