Skip to content

Commit

Permalink
V2.10.0 (#205)
Browse files Browse the repository at this point in the history
  • Loading branch information
@frf12
frf12 authored Sep 3, 2024
1 parent b00c72b commit 23cb913
Show file tree
Hide file tree
Showing 351 changed files with 17,031 additions and 4,150 deletions.
35 changes: 21 additions & 14 deletions _cmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter

from core import ObdHome
from _stdio import IO, FormtatText
from _stdio import IO, FormatText
from _lock import LockMode
from _types import Capacity
from tool import DirectoryUtil, FileUtil, NetUtil, COMMAND_ENV
Expand Down Expand Up @@ -226,17 +226,22 @@ def parse_command(self):
self.parser.allow_undefine = self.dev_mode
return super(ObdCommand, self).parse_command()

def _init_log(self):
trace_id = uuid()
log_dir = os.path.join(self.OBD_PATH, 'log')
DirectoryUtil.mkdir(log_dir)
log_path = os.path.join(log_dir, 'obd')
ROOT_IO.init_trace_logger(log_path, 'obd', trace_id)
ROOT_IO.exit_msg = '''Trace ID: {trace_id}
If you want to view detailed obd logs, please run: obd display-trace {trace_id}'''.format(trace_id=trace_id)

def do_command(self):
self.parse_command()
self.init_home()
trace_id = uuid()
ret = False
try:
log_dir = os.path.join(self.OBD_PATH, 'log')
DirectoryUtil.mkdir(log_dir)
log_path = os.path.join(log_dir, 'obd')
if self.enable_log:
ROOT_IO.init_trace_logger(log_path, 'obd', trace_id)
if self.has_trace and self.enable_log:
self._init_log()
ROOT_IO.track_limit += 1
ROOT_IO.verbose('cmd: %s' % self.cmds)
ROOT_IO.verbose('opts: %s' % self.opts)
Expand All @@ -245,7 +250,7 @@ def do_command(self):
obd.set_cmds(self.cmds)
ret = self._do_command(obd)
if not ret:
ROOT_IO.print(DOC_LINK_MSG)
ROOT_IO.exit_msg = DOC_LINK_MSG + "\n" + ROOT_IO.exit_msg
except NotImplementedError:
ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd)
except LockError:
Expand All @@ -257,9 +262,6 @@ def do_command(self):
except:
e = sys.exc_info()[1]
ROOT_IO.exception('Running Error: %s' % e)
if self.has_trace:
ROOT_IO.print('Trace ID: %s' % trace_id)
ROOT_IO.print('If you want to view detailed obd logs, please run: obd display-trace %s' % trace_id)
return ret

def _do_command(self, obd):
Expand Down Expand Up @@ -859,7 +861,7 @@ def _do_command(self, obd):
res = obd.deploy_cluster(self.cmds[0])
self.background_telemetry_task(obd)
if res:
obd.stdio.print(FormtatText.success('Please execute ` obd cluster start %s ` to start' % self.cmds[0]))
obd.stdio.print(FormatText.success('Please execute ` obd cluster start %s ` to start' % self.cmds[0]))
return res
else:
return self._show_help()
Expand Down Expand Up @@ -900,6 +902,7 @@ class ClusterComponentDeleteCommand(ClusterMirrorCommand):

def __init__(self):
super(ClusterComponentDeleteCommand, self).__init__('del', 'Add components for cluster')
self.parser.add_option('-f', '--force', action='store_true', help="Force delete components.", default=False)
self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force kill the observer while standby tenant in others cluster exists.")

def init(self, cmd, args):
Expand Down Expand Up @@ -1129,6 +1132,7 @@ def __init__(self):
self.parser.add_option('--tablegroup', type='string', help="Tenant tablegroup.")
self.parser.add_option('--primary-zone', type='string', help="Tenant primary zone. [RANDOM].", default='RANDOM')
self.parser.add_option('--locality', type='string', help="Tenant locality.")
self.parser.add_option('--time-zone', type='string', help="Tenant time zone. The default tenant time_zone is [+08:00].")
self.parser.add_option('-s', '--variables', type='string', help="Set the variables for the system tenant. [ob_tcp_invited_nodes='%'].", default="ob_tcp_invited_nodes='%'")
self.parser.add_option('-o', '--optimize', type='string', help="Specify scenario optimization when creating a tenant, the default is consistent with the cluster dimension.\n{express_oltp, complex_oltp, olap, htap, kv}\nSupported since version 4.3.")

Expand Down Expand Up @@ -1406,7 +1410,7 @@ def __init__(self):
self.parser.add_option('--sysbench-script-dir', type='string', help='The directory of the sysbench lua script file. [/usr/sysbench/share/sysbench]', default='/usr/sysbench/share/sysbench')
self.parser.add_option('--table-size', type='int', help='Number of data initialized per table. [20000]', default=20000)
self.parser.add_option('--tables', type='int', help='Number of initialization tables. [30]', default=30)
self.parser.add_option('--threads', type='int', help='Number of threads to use. [16]', default=16)
self.parser.add_option('--threads', type='string', help='Number of threads to use. [16]', default='16')
self.parser.add_option('--time', type='int', help='Limit for total execution time in seconds. [60]', default=60)
self.parser.add_option('--interval', type='int', help='Periodically report intermediate statistics with a specified time interval in seconds. 0 disables intermediate reports. [10]', default=10)
self.parser.add_option('--events', type='int', help='Limit for total number of events.')
Expand Down Expand Up @@ -1447,6 +1451,8 @@ def __init__(self):
self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1/2}. [1] 0 - No optimization. 1 - Optimize some of the parameters which do not need to restart servers. 2 - Optimize all the parameters and maybe RESTART SERVERS for better performance.', default=1)
self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.')
self.parser.add_option('-S', '--skip-cluster-status-check', action='store_true', help='Skip cluster status check', default=False)
self.parser.add_option('--direct-load', action='store_true', help="Enable load data by direct feature.")
self.parser.add_option('--parallel', type='int', help='The degree of parallelism for loading data. [max_cpu * unit_count]')

def _do_command(self, obd):
if self.cmds:
Expand Down Expand Up @@ -1605,9 +1611,10 @@ def __init__(self):
super(CommandsCommand, self).__init__('command', 'Common tool commands')
self.parser.add_option('-c', '--components', type='string', help='The components used by the command. The first component in the configuration will be used by default in interactive commands, and all available components will be used by default in non-interactive commands.')
self.parser.add_option('-s', '--servers', type='string', help='The servers used by the command. The first server in the configuration will be used by default in interactive commands, and all available servers will be used by default in non-interactive commands.')
self.parser.undefine_warn = False

def _do_command(self, obd):
if len(self.cmds) == 2:
if len(self.cmds) in [2, 3]:
return obd.commands(self.cmds[0], self.cmds[1], self.opts)
else:
return self._show_help()
Expand Down
30 changes: 24 additions & 6 deletions _deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import hashlib
from copy import deepcopy
from enum import Enum
from datetime import datetime

from ruamel.yaml.comments import CommentedMap

Expand Down Expand Up @@ -628,9 +629,15 @@ def del_depend(self, name, component_name):

def get_depend_servers(self, name):
if name not in self._depends:
return None
return []
cluster_config = self._depends[name]
return deepcopy(cluster_config.original_servers)

def get_be_depend_servers(self, name):
if name not in self._be_depends:
return []
cluster_config = self._be_depends[name]
return deepcopy(cluster_config.original_servers)

def get_depend_added_servers(self, name):
if name not in self._depends:
Expand Down Expand Up @@ -1033,11 +1040,12 @@ class DeployInstallMode(object):

class DeployInfo(object):

def __init__(self, name, status, components=None, config_status=DeployConfigStatus.UNCHNAGE):
def __init__(self, name, status, components=None, config_status=DeployConfigStatus.UNCHNAGE, create_date=None):
self.status = status
self.name = name
self.components = components if components else {}
self.config_status = config_status
self.create_date = create_date

def __str__(self):
info = ['%s (%s)' % (self.name, self.status.value)]
Expand Down Expand Up @@ -1218,19 +1226,23 @@ def scale_out(self, config_path):
source_data = self.yaml_loader.load(f)
for key in source_data:
if key in ['user', 'unuse_lib_repository', 'auto_create_tenant']:
self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(key))
self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(message=key))
ret = False
elif issubclass(type(source_data[key]), dict):
new_depends = source_data[key].get('depends', [])
if key not in self.components:
self.stdio.error(err.EC_COMPONENT_NOT_EXISTS.format(component=key))
ret = False
break
if new_depends and new_depends != self.components[key].depends:
self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(message='depends:{}'.format(key)))
ret = False
# temp _depends
depends[key] = self.components[key].depends

if not self._merge_component(key, source_data[key]):
ret = False

for comp in depends:
conf = self.components[comp]
for name in depends[comp]:
Expand All @@ -1243,7 +1255,7 @@ def scale_out(self, config_path):
ret = False
return ret

def add_components(self, config_path):
def add_components(self, config_path, ignore_exist=False):
ret = True
depends = {}
try:
Expand All @@ -1255,6 +1267,8 @@ def add_components(self, config_path):
ret = False
elif issubclass(type(source_data[key]), dict):
if key in self.components:
if ignore_exist:
continue
self.stdio.error(err.EC_COMPONENT_EXISTS.format(component=key))
ret = False
continue
Expand Down Expand Up @@ -1578,6 +1592,7 @@ def deploy_info(self):
getattr(DeployStatus, data['status'], DeployStatus.STATUS_CONFIGURED),
ConfigUtil.get_value_from_dict(data, 'components', OrderedDict()),
getattr(DeployConfigStatus, ConfigUtil.get_value_from_dict(data, 'config_status', '_'), DeployConfigStatus.UNCHNAGE),
ConfigUtil.get_value_from_dict(data, 'create_date', None),
)
except:
self._info = DeployInfo(self.name, DeployStatus.STATUS_CONFIGURED)
Expand Down Expand Up @@ -1654,6 +1669,7 @@ def dump_deploy_info(self):
'components': self.deploy_info.components,
'status': self.deploy_info.status.name,
'config_status': self.deploy_info.config_status.name,
'create_date': self.deploy_info.create_date,
}
yaml.dump(data, f)
return True
Expand All @@ -1664,6 +1680,8 @@ def dump_deploy_info(self):
def _update_deploy_status(self, status):
old = self.deploy_info.status
self.deploy_info.status = status
if old == DeployStatus.STATUS_DEPLOYED and status == DeployStatus.STATUS_RUNNING:
self.deploy_info.create_date = datetime.now().strftime('%Y-%m-%d')
if self.dump_deploy_info():
return True
self.deploy_info.status = old
Expand Down
13 changes: 11 additions & 2 deletions _errno.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ class InitDirFailedErrorMessage(object):
EC_COMPONENT_FAILED_TO_MERGE_CONFIG = OBDErrorCodeTemplate(1023, 'Failed to merge config: {message}')
EC_COMPONENT_NO_REMAINING_COMPS = OBDErrorCodeTemplate(1024, 'The cluster will have no remaining components. If you are absolutely sure about DELETING ALL COMPONENTS, please use "obd cluster destroy <deploy>" command to completely destroy the cluster')
EC_COMPONENT_PASSWD_ERROR = OBDErrorCodeTemplate(1025, '({ip}) {component} {key} invalid. (Rule: {rule})')
EC_RUNNING_CLUSTER_NO_REDEPLOYED = OBDErrorCodeTemplate(1026, 'Could not modify {key} when the cluster is in the working status(`production_mode` is True, not support this operation).')
EC_COMPONENT_DIR_NOT_EMPTY = OBDErrorCodeTemplate(1027, 'If you are sure the directory can be emptied, run `obd cluster deploy -f {deploy_name}` to perform forced deployment.')

WC_ULIMIT_CHECK = OBDErrorCodeTemplate(1007, '({server}) The recommended number of {key} is {need} (Current value: {now})')
WC_AIO_NOT_ENOUGH = OBDErrorCodeTemplate(1011, '({ip}) The recommended value of fs.aio-max-nr is 1048576 (Current value: {current})')
Expand All @@ -160,6 +162,8 @@ class InitDirFailedErrorMessage(object):
EC_OBSERVER_FAILED_TO_REGISTER_WITH_DETAILS = OBDErrorCodeTemplate(2005, 'Failed to register cluster. {appname} may have been registered in {obconfig_url}.')
EC_OBSERVER_MULTI_NET_DEVICE = OBDErrorCodeTemplate(2006, '{ip} has more than one network interface. Please set `devname` for ({server})')
EC_OBSERVER_PING_FAILED = OBDErrorCodeTemplate(2007, '{ip1} {devname} fail to ping {ip2}. Please check configuration `devname`')
EC_OBSERVER_PING_NOT_FOUND = OBDErrorCodeTemplate(2007, '/usr/bin/ping: No such file or directory. You can run `sudo yum install iputils` or `sudo apt-get install iputils-ping`.')
EC_OBSERVER_PING_FAILED_SUID = OBDErrorCodeTemplate(2007, 'If the error message `operation not permitted` appears, please check the ping file permissions. You can try running `sudo chmod u+s /usr/bin/ping`')
EC_OBSERVER_PING_FAILED_WITH_NO_DEVNAME = OBDErrorCodeTemplate(2007, '{ip1} fail to ping {ip2}. Please check your network')
EC_OBSERVER_TIME_OUT_OF_SYNC = OBDErrorCodeTemplate(2008, 'Cluster clocks are out of sync')
EC_OBSERVER_PRODUCTION_MODE_LIMIT = OBDErrorCodeTemplate(2009, '({server}): when production_mode is True, {key} can not be less then {limit}')
Expand All @@ -180,6 +184,7 @@ class InitDirFailedErrorMessage(object):
# obagent
EC_OBAGENT_RELOAD_FAILED = OBDErrorCodeTemplate(4000, 'Fail to reload {server}')
EC_OBAGENT_SEND_CONFIG_FAILED = OBDErrorCodeTemplate(4001, 'Fail to send config file to {server}')
WC_OBAGENT_SERVER_NAME_ERROR = OBDErrorCodeTemplate(4002, '{servers}: Failed to obtain the configuration of the OceanBase database component. \nPlease ensure that the server configurations are consistent between the OBAgent and OceanBase database components.')

# obproxy
EC_OBPROXY_NEED_CONFIG = OBDErrorCodeTemplate(4100, '{server} need config "rs_list" or "obproxy_config_server_url"')
Expand Down Expand Up @@ -220,8 +225,9 @@ class InitDirFailedErrorMessage(object):
EC_OCP_SERVER_NOT_ENOUGH_MEMORY_CACHED = OBDErrorCodeTemplate(4364, '({ip}) not enough memory. (Free: {free}, Buff/Cache: {cached}, Need: {need})')
EC_OCP_SERVER_NOT_ENOUGH_MEMORY = OBDErrorCodeTemplate(4364, '({ip}) not enough memory. (Free: {free}, Need: {need})')
EC_OCP_SERVER_NOT_ENOUGH_DISK = OBDErrorCodeTemplate(4365, '({ip}) {disk} not enough disk space. (Avail: {avail}, Need: {need})')
EC_OCP_SERVER_RESOURCE_NOT_ENOUGH = OBDErrorCodeTemplate(4366, 'There is not enough {resource}. (Avail: {avail}, need: {need})')

EC_OCP_SERVER_RESOURCE_NOT_ENOUGH = OBDErrorCodeTemplate(4366, 'There is not enough {resource}. (Avail: {avail}, Need: {need})')
EC_OCP_SERVER_EXIST_METADB_TENANT_MEMORY_NOT_ENOUGH = OBDErrorCodeTemplate(4367, 'The allocated memory for the provided meta database is currently insufficient for creating a tenant. Available: {avail}, Need: {need}.')
EC_OCP_SERVER_NOT_EXIST_METADB_TENANT_MEMORY_NOT_ENOUGH = OBDErrorCodeTemplate(4368, 'The allocated memory for the provided meta database is currently insufficient for creating a tenant. Available: {avail}, Need: {need}(Available = memory_limit [{memory_limit}] - system_memory [{system_memory}] - sys tenant memory [{sys_tenant_memory}]. Need = ocp meta tenant memory [{ocp_meta_tenant_memory}] + ocp_monitor_tenant_memory [{ocp_monitor_tenant_memory}]).')


WC_OCP_EXPRESS_FAILED_TO_GET_DISK_INFO = OBDErrorCodeTemplate(4303, '({ip}) failed to get disk information, skip disk space check')
Expand Down Expand Up @@ -300,7 +306,10 @@ class InitDirFailedErrorMessage(object):
SUG_OCP_SERVER_REDUCE_MEM = OBDErrorSuggestionTemplate('Please reduce the `memory_size`', fix_eval=[FixEval(FixEval.DEL, 'memory_size')])
SUG_OCP_SERVER_REDUCE_DISK = OBDErrorSuggestionTemplate('Please reduce the `logging_file_total_size_cap`', fix_eval=[FixEval(FixEval.DEL, 'logging_file_total_size_cap')])
SUG_OCP_SERVER_EDIT_ADMIN_PASSWD_ERROR = OBDErrorSuggestionTemplate('Please edit the `admin_password`, must be 8 to 32 characters in length, containing at least 3 types from digits, lowercase letters, uppercase letters and the following special characters: ~!@#%^&*_-+=|(){{}}[]:;,.?/)', fix_eval=[FixEval(FixEval.DEL, 'admin_password')], auto_fix=True)
SUG_OCP_SERVER_MACHINE_TIME = OBDErrorSuggestionTemplate('Please ensure that the machine time is synchronized with the ob time')
SUG_SUDO_NOPASSWD = OBDErrorSuggestionTemplate('Please execute `bash -c \'echo "{user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers`\' as root in {ip}.')
SUG_OCP_SERVER_EXIST_METADB_TENANT_NOT_ENOUGH = OBDErrorSuggestionTemplate('Please reduce the ocp meta tenant memory or ocp monitor tenant memory')
SUG_OCP_SERVER_NOT_EXIST_METADB_TENANT_NOT_ENOUGH = OBDErrorSuggestionTemplate('Please increase the meta db memory_limit and reduce the ocp meta tenant memory or ocp monitor tenant memory')
SUG_OB_SYS_USERNAME = OBDErrorSuggestionTemplate('Please delete the "ob_sys_username" parameter.')
SUG_OB_SYS_PASSWORD = OBDErrorSuggestionTemplate('''Please set the "ob_sys_password" for oblogproxy by configuring the "cdcro_password" parameter in the "oceanbase" or "oceanbase-ce" component.''')
SUG_OBAGENT_EDIT_HTTP_BASIC_AUTH_PASSWORD = OBDErrorSuggestionTemplate('Please edit the `http_basic_auth_password`, cannot contain characters other than uppercase letters, lowercase characters, digits, special characters:~^*{{}}[]_-+', fix_eval=[FixEval(FixEval.DEL, 'http_basic_auth_password')], auto_fix=True)
Expand Down
Loading

0 comments on commit 23cb913

Please sign in to comment.