Skip to content

Commit

Permalink
V2.3.0 (#177)
Browse files Browse the repository at this point in the history 
* V2.3.0

* V2.3.0

* update docs
  • Loading branch information
@frf12
frf12 authored Sep 15, 2023
1 parent d84e56c commit 68d897d
Show file tree
Hide file tree
Showing 129 changed files with 11,550 additions and 4,042 deletions.
104 changes: 103 additions & 1 deletion _cmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@
from uuid import uuid1 as uuid, UUID
from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter

from colorama import Fore

from core import ObdHome
from _stdio import IO
from _stdio import IO, FormtatText
from _lock import LockMode
from tool import DirectoryUtil, FileUtil, NetUtil, COMMAND_ENV
from _errno import DOC_LINK_MSG, LockError
Expand Down Expand Up @@ -780,6 +782,8 @@ def _do_command(self, obd):
obd.set_options(self.opts)
res = obd.deploy_cluster(self.cmds[0])
self.background_telemetry_task(obd)
if res:
obd.stdio.print(FormtatText('Please execute ` obd cluster start %s ` to start' % self.cmds[0], Fore.GREEN))
return res
else:
return self._show_help()
Expand Down Expand Up @@ -826,6 +830,7 @@ class ClusterDestroyCommand(ClusterMirrorCommand):
def __init__(self):
super(ClusterDestroyCommand, self).__init__('destroy', 'Destroy a deployed cluster.')
self.parser.add_option('-f', '--force-kill', action='store_true', help="Force kill the running observer process in the working directory.")
self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force kill the observer while standby tenant in others cluster exists.")

def _do_command(self, obd):
if self.cmds:
Expand Down Expand Up @@ -873,6 +878,7 @@ def __init__(self):
super(ClusterRedeployCommand, self).__init__('redeploy', 'Redeploy a started cluster.')
self.parser.add_option('-f', '--force-kill', action='store_true', help="Force kill the running observer process in the working directory.")
self.parser.add_option('--confirm', action='store_true', help='Confirm to redeploy.')
self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force kill the observer while standby tenant in others cluster exists.")

def _do_command(self, obd):
if self.cmds:
Expand Down Expand Up @@ -951,6 +957,7 @@ def __init__(self):
self.parser.add_option('--disable', type='string', help="Hash list for disabled mirrors, separated with `,`.", default='')
self.parser.add_option('-e', '--executer-path', type='string', help="Executer path.", default=os.path.join(ObdCommand.OBD_INSTALL_PATH, 'lib/executer'))
self.parser.add_option('-t', '--script-query-timeout', type='string', help="The timeout(s) for executing sql in upgrade scripts. Supported since version 4.1.0", default='')
self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force upgrade, before upgrade standby tenant`s cluster.")

def _do_command(self, obd):
if self.cmds:
Expand Down Expand Up @@ -996,11 +1003,100 @@ def _do_command(self, obd):
return self._show_help()


class ClusterTenantCreateStandByCommand(ClusterTenantCreateCommand):

def __init__(self):
super(ClusterTenantCreateStandByCommand, self).__init__()
self.name = 'create-standby'
self.summary = 'Create a standby tenant.'
self.parser.remove_option('-t')
self.parser.remove_option('--max-memory')
self.parser.remove_option('--min-memory')
self.parser.remove_option('--max-disk-size')
self.parser.remove_option('--max-session-num')
self.parser.remove_option('--mode')
self.parser.remove_option('--charset')
self.parser.remove_option('--collate')
self.parser.remove_option('--logonly-replica-num')
self.parser.remove_option('--tablegroup')
self.parser.remove_option('-s')
self.parser.add_option('-t', '-n', '--tenant-name', type='string', help="The standby tenant name. The default tenant name is consistent with the primary tenant name.", default='')
self.parser.add_option('--standbyro-password', type='string', help="standbyro user password.")
self.parser.add_option('-p', '--tenant-root-password', type='string', help="tenant root password,for crate standby user.")


def init(self, cmd, args):
super(ClusterTenantCreateStandByCommand, self).init(cmd, args)
self.parser.set_usage('%s <standby deploy name> <primary deploy name> <primary tenant name> [options]' % self.prev_cmd)
return self

def _do_command(self, obd):
if len(self.cmds) == 3:
return obd.create_standby_tenant(self.cmds[0], self.cmds[1], self.cmds[2])
else:
return self._show_help()


class ClusterTenantSwitchoverCommand(ClusterMirrorCommand):

def __init__(self):
super(ClusterTenantSwitchoverCommand, self).__init__('switchover', 'Switchover primary-standby tenant.')
self.parser.add_option('-p', '--tenant-root-password', type='string', help="tenant root password")
self.parser.add_option('--standbyro-password', type='string', help="standbyro user password.")

def init(self, cmd, args):
super(ClusterTenantSwitchoverCommand, self).init(cmd, args)
self.parser.set_usage('%s <standby deploy name> <standby tenant name> [options]' % self.prev_cmd)
return self

def _do_command(self, obd):
if len(self.cmds) == 2:
return obd.switchover_tenant(self.cmds[0], self.cmds[1])
else:
return self._show_help()


class ClusterTenantFailoverCommand(ClusterMirrorCommand):
def __init__(self):
super(ClusterTenantFailoverCommand, self).__init__('failover', 'failover standby tenant.')
self.parser.add_option('-p', '--tenant-root-password', type='string', help="tenant root password")

def init(self, cmd, args):
super(ClusterTenantFailoverCommand, self).init(cmd, args)
self.parser.set_usage('%s <standby deploy name> <standby tenant name> [options]' % self.prev_cmd)
return self

def _do_command(self, obd):
if len(self.cmds) == 2:
return obd.failover_decouple_tenant(self.cmds[0], self.cmds[1], 'failover')
else:
return self._show_help()


class ClusterTenantDecoupleCommand(ClusterMirrorCommand):

def __init__(self):
super(ClusterTenantDecoupleCommand, self).__init__('decouple', 'decouple standby tenant.')
self.parser.add_option('-p', '--tenant-root-password', type='string', help="tenant root password")

def init(self, cmd, args):
super(ClusterTenantDecoupleCommand, self).init(cmd, args)
self.parser.set_usage('%s <standby deploy name> <standby tenant name> [options]' % self.prev_cmd)
return self

def _do_command(self, obd):
if len(self.cmds) == 2:
return obd.failover_decouple_tenant(self.cmds[0], self.cmds[1], 'decouple')
else:
return self._show_help()


class ClusterTenantDropCommand(ClusterMirrorCommand):

def __init__(self):
super(ClusterTenantDropCommand, self).__init__('drop', 'Drop a tenant.')
self.parser.add_option('-t', '-n', '--tenant-name', type='string', help="Tenant name.")
self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force drop tenant when it has standby tenant, the standby tenants will become unavailable.")

def _do_command(self, obd):
if self.cmds:
Expand All @@ -1013,6 +1109,8 @@ class ClusterTenantListCommand(ClusterMirrorCommand):

def __init__(self):
super(ClusterTenantListCommand, self).__init__('show', 'Show the list of tenant.')
self.parser.add_option('-t', '--tenant', type='string', help='Tenant name', default='')
self.parser.add_option('-g', '--graph', action='store_true', help='view standby by graph')

def _do_command(self, obd):
if self.cmds:
Expand All @@ -1028,6 +1126,10 @@ def __init__(self):
self.register_command(ClusterTenantCreateCommand())
self.register_command(ClusterTenantDropCommand())
self.register_command(ClusterTenantListCommand())
self.register_command(ClusterTenantCreateStandByCommand())
self.register_command(ClusterTenantSwitchoverCommand())
self.register_command(ClusterTenantFailoverCommand())
self.register_command(ClusterTenantDecoupleCommand())


class ClusterMajorCommand(MajorCommand):
Expand Down
124 changes: 99 additions & 25 deletions _deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@


from __future__ import absolute_import, division, print_function
from collections.abc import Iterable, Iterator

import os
import re
Expand Down Expand Up @@ -111,48 +112,107 @@ class InnerConfigItem(str):
pass


class InnerConfigKey(object):

keyword_symbol = "$_"

@classmethod
def is_keyword(cls, s):
return s.startswith(cls.keyword_symbol)

@classmethod
def to_keyword(cls, key):
return "{}{}".format(cls.keyword_symbol, key)

@classmethod
def keyword_to_str(cls, _keyword):
return str(_keyword.replace(cls.keyword_symbol, '', 1))


class ComponentInnerConfig(dict):

COMPONENT_GLOBAL_ATTRS = InnerConfigKey.to_keyword('component_global_attrs')

def __init__(self, component_name, config):
super(ComponentInnerConfig, self).__init__()
self.component_name = component_name
c_config = {}
for server in config:
i_config = {}
s_config = config[server]
for key in s_config:
i_config[InnerConfigItem(key)] = s_config[key]
c_config[server] = i_config
self.update(c_config)

def __iter__(self):
keys = self.keys()
servers = []
for key in keys:
if key != self.COMPONENT_GLOBAL_ATTRS:
servers.append(key)
return iter(servers)

def update_server_config(self, server, key, value):
self._update_config(server, key, value)

def update_attr(self, key, value):
self._update_config(self.COMPONENT_GLOBAL_ATTRS, key, value)

def del_server_config(self, server, key):
self._del_config(server, key)

def del_attr(self, key):
self._del_config(self.COMPONENT_GLOBAL_ATTRS, key)

def get_attr(self, key):
return self._get_config(self.COMPONENT_GLOBAL_ATTRS, key)

def _update_config(self, item, key, value):
if not self.__contains__(item):
self[item] = {}
if not InnerConfigKey.is_keyword(key):
key = InnerConfigKey.to_keyword(key)
self[item][key] = value

def _del_config(self, item, key):
if not self.__contains__(item):
return
if not InnerConfigKey.is_keyword(key):
key = InnerConfigKey.to_keyword(key)
if key in self[item]:
del self[item][key]

def _get_config(self, item, key):
if not self.__contains__(item):
return
if not InnerConfigKey.is_keyword(key):
key = InnerConfigKey.to_keyword(key)
return self[item].get(key)

class InnerConfigKeywords(object):

DEPLOY_INSTALL_MODE = 'deploy_install_mode'
DEPLOY_BASE_DIR = 'deploy_base_dir'


class InnerConfig(object):

keyword_symbol = "$_"

def __init__(self, path, yaml_loader):
self.path = path
self.yaml_loader = yaml_loader
self.config = {}
self._load()

def is_keyword(self, s):
return s.startswith(self.keyword_symbol)

def to_keyword(self, key):
return "{}{}".format(self.keyword_symbol, key)

def keyword_to_str(self, _keyword):
return str(_keyword.replace(self.keyword_symbol, '', 1))

def _load(self):
self.config = {}
try:
with FileUtil.open(self.path, 'rb') as f:
config = self.yaml_loader.load(f)
for component_name in config:
if self.is_keyword(component_name):
if InnerConfigKey.is_keyword(component_name):
self.config[InnerConfigItem(component_name)] = config[component_name]
continue
self.config[component_name] = {}
c_config = config[component_name]
for server in c_config:
i_config = {}
s_config = c_config[server]
for key in s_config:
i_config[InnerConfigItem(key)] = s_config[key]
self.config[component_name][server] = i_config
else:
self.config[component_name] = ComponentInnerConfig(component_name, config[component_name])
except:
pass

Expand All @@ -176,11 +236,11 @@ def get_server_config(self, component_name, server):
return self.get_component_config(component_name).get(server, {})

def get_global_config(self, key, default=None):
key = self.to_keyword(key)
key = InnerConfigKey.to_keyword(key)
return self.config.get(key, default)

def update_global_config(self, key, value):
self.config[self.to_keyword(key)] = value
self.config[InnerConfigKey.to_keyword(key)] = value

def update_component_config(self, component_name, config):
self.config[component_name] = {}
Expand All @@ -192,6 +252,8 @@ def update_component_config(self, component_name, config):
key = InnerConfigItem(key)
c_config[key] = data[key]
self.config[component_name][server] = c_config
if ComponentInnerConfig.COMPONENT_GLOBAL_ATTRS in config:
self.config[component_name][ComponentInnerConfig.COMPONENT_GLOBAL_ATTRS] = config[ComponentInnerConfig.COMPONENT_GLOBAL_ATTRS]


class ConfigParser(object):
Expand Down Expand Up @@ -449,6 +511,10 @@ def set_base_dir(self, base_dir):
self._include_config = None
self._global_conf = None

@property
def deploy_name(self):
return self._deploy_config.name

@property
def original_servers(self):
return self._original_servers
Expand All @@ -465,7 +531,14 @@ def _clear_cache_server(self):

def get_inner_config(self):
return self._inner_config


def update_component_attr(self, key, value, save=True):
self._inner_config.update_attr(key, value)
return self._deploy_config.dump() if save else True

def get_component_attr(self, key):
return self._inner_config.get_attr(key)

def is_cp_install_mode(self):
return self._deploy_config.is_cp_install_mode()

Expand Down Expand Up @@ -889,6 +962,7 @@ def __init__(self, yaml_path, yaml_loader=yaml, inner_config=None, config_parser
self._inner_config = inner_config
self.components = OrderedDict()
self._src_data = None
self.name = os.path.split(os.path.split(yaml_path)[0])[-1]
self.yaml_path = yaml_path
self.yaml_loader = yaml_loader
self.config_parser_manager = config_parser_manager if config_parser_manager else DEFAULT_CONFIG_PARSER_MANAGER
Expand Down
Loading

0 comments on commit 68d897d

Please sign in to comment.