V2.1.1 (#175)

_cmd.py

@@ -422,7 +422,7 @@ def __init__(self):
 class TelemetryPostCommand(HiddenObdCommand):
 
     def __init__(self):
-        super(TelemetryPostCommand, self).__init__("post", "Post telemetry data to OceanBase.By default, OBD telemetry is enabled. To disable OBD telemetry, run the `obd env set TELEMETRY_MODE 0` command. To enable OBD telemetry data printing, run `obd env set TELEMETRY_LOG_MODE 1`.")
+        super(TelemetryPostCommand, self).__init__('post', "Post telemetry data to OceanBase.By default, OBD telemetry is enabled. To disable OBD telemetry, run the `obd env set TELEMETRY_MODE 0` command. To enable OBD telemetry data printing, run `obd env set TELEMETRY_LOG_MODE 1`.")
         self.parser.add_option('-d', '--data', type='string', help="post obd data")
 
     @property
@@ -431,9 +431,7 @@ def lock_mode(self):
 
     @property
     def enable_log(self):
-        if COMMAND_ENV.get(ENV.TELEMETRY_LOG_MODE, default='1') == '0':
-            return False
-        return True
+        return COMMAND_ENV.get(ENV.TELEMETRY_LOG_MODE, default='0') == '1'
 
     def init(self, cmd, args):
         super(TelemetryPostCommand, self).init(cmd, args)
@@ -447,7 +445,7 @@ def _do_command(self, obd):
 class TelemetryMajorCommand(HiddenMajorCommand):
 
     def __init__(self):
-        super(TelemetryMajorCommand, self).__init__("telemetry", "Telemetry for OB-Deploy.By default, OBD telemetry is enabled. To disable OBD telemetry, run the `obd env set TELEMETRY_MODE 0` command. To enable OBD telemetry data printing, run `obd env set TELEMETRY_LOG_MODE 1`.")
+        super(TelemetryMajorCommand, self).__init__('telemetry', "Telemetry for OB-Deploy.By default, OBD telemetry is enabled. To disable OBD telemetry, run the `obd env set TELEMETRY_MODE 0` command. To enable OBD telemetry data printing, run `obd env set TELEMETRY_LOG_MODE 1`.")
         self.register_command(TelemetryPostCommand())
 
     def do_command(self):
@@ -654,9 +652,11 @@ def get_obd_namespaces_data(self, obd):
             data[component] = _.get_variable('run_result')
         return data
 
-    def background_telemetry_task(self, obd):
+    def background_telemetry_task(self, obd, demploy_name=None):
+        if demploy_name is None:
+            demploy_name = self.cmds[0]
         data = json.dumps(self.get_obd_namespaces_data(obd))
-        LocalClient.execute_command_background(f"nohup obd telemetry post {self.cmds[0]} --data='{data}' >/dev/null 2>&1 &")
+        LocalClient.execute_command_background(f"nohup obd telemetry post {demploy_name} --data='{data}' >/dev/null 2>&1 &")
 
 
 class ClusterConfigStyleChange(ClusterMirrorCommand):
@@ -704,7 +704,10 @@ def _do_command(self, obd):
         setattr(self.opts, 'force', True)
         setattr(self.opts, 'force_delete', True)
         obd.set_options(self.opts)
-        return obd.demo()
+
+        res = obd.demo()
+        self.background_telemetry_task(obd, 'demo')
+        return res
 
 
 class WebCommand(ObdCommand):
@@ -1479,7 +1482,7 @@ def __init__(self):
         self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all')
         self.parser.add_option('--grep', type='string', help="specify keywords constrain")
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 class ObdiagGatherLogCommand(ObdiagGatherMirrorCommand):
@@ -1500,7 +1503,7 @@ def __init__(self):
         self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all')
         self.parser.add_option('--grep', type='string', help="specify keywords constrain")
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1516,7 +1519,7 @@ def lock_mode(self):
 
     def __init__(self):
         super(ObdiagGatherSysStatCommand, self).__init__('sysstat', 'Gather Host information')
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1532,7 +1535,7 @@ def lock_mode(self):
 
     def __init__(self):
         super(ObdiagGatherStackCommand, self).__init__('stack', 'Gather stack')
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1548,7 +1551,7 @@ def lock_mode(self):
 
     def __init__(self):
         super(ObdiagGatherPerfCommand, self).__init__('perf', 'Gather perf')
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--scope', type='string', help="perf type constrains, choices=[sample, flame, pstack, all]",default='all')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
@@ -1569,7 +1572,7 @@ def __init__(self):
         self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss")
         self.parser.add_option('--since', type='string',  help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.",default='30m')
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1589,7 +1592,7 @@ def __init__(self):
         self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss")
         self.parser.add_option('--since', type='string',  help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.",default='30m')
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1607,7 +1610,7 @@ def __init__(self):
         super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information')
         self.parser.add_option('-c', '--component', type='string', help="Component name to connect.", default='oceanbase-ce')
         self.parser.add_option('--trace_id', type='string', help='sql trace id')
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('-u', '--user', type='string', help='The username used by database connection. [root]',default='root')
         self.parser.add_option('-p', '--password', type='string', help='The password used by database connection.',default='')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
@@ -1631,7 +1634,7 @@ def __init__(self):
         self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all')
         self.parser.add_option('--grep', type='string', help="specify keywords constrain")
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd())
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1673,7 +1676,7 @@ def __init__(self):
         sys.setdefaultencoding(defaultencoding)
     sys.path.append(os.path.join(ObdCommand.OBD_INSTALL_PATH, 'lib/site-packages'))
     ROOT_IO.track_limit += 2
-    if MainCommand().init('obd', sys.argv[1:]).do_command():
+    if MainCommand().init(sys.argv[0], sys.argv[1:]).do_command():
         ROOT_IO.exit(0)
     ROOT_IO.exit(1)
 

_mirror.py

@@ -44,6 +44,7 @@
 from _rpm import Package, PackageInfo
 from tool import ConfigUtil, FileUtil, var_replace
 from _manager import Manager
+from tool import timeout
 
 
 _ARCH = getArchList()
@@ -276,10 +277,13 @@ def __init__(self, mirror_path, meta_data, stdio=None):
 
     @property
     def available(self):
+        if not self.enabled:
+            return False
         if self._available is None:
             try:
-                req = requests.request('get', self.baseurl)
-                self._available = req.status_code < 400
+                with timeout(5):
+                    req = requests.request('get', self.baseurl)
+                    self._available = req.status_code < 400
             except Exception:
                 self.stdio and getattr(self.stdio, 'exception', print)('')
                 self._available = False

const.py

@@ -21,7 +21,8 @@
 OB_OFFICIAL_WEBSITE = 'https://www.oceanbase.com/'
 
 # post telemetry data to OceanBase official
-TELEMETRY_URL = 'http://openwebapi.dev.alipay.net/api/web/oceanbase/report'
+TELEMETRY_WEBSITE = '<TELEMETRY_WEBSITE>'
+TELEMETRY_URL = '{}/api/web/oceanbase/report'.format(TELEMETRY_WEBSITE if TELEMETRY_WEBSITE else 'https://openwebapi.oceanbase.com')
 
 # obdeploy version
 VERSION = '<VERSION>'

core.py

@@ -3963,21 +3963,14 @@ def telemetry_post(self, name):
         if repositories == []:
             return
         self.set_repositories(repositories)
-        target_repository = None
-        for repository in repositories:
-            if repository.name in ['oceanbase', 'oceanbase-ce']:
-                target_repository = repository
-                break
-            else:
-                target_repository = repository
-        telemetry_info_collect_plugin = self.plugin_manager.get_best_py_script_plugin('telemetry_info_collect', 'general', '0.1')
-        ret = self.call_plugin(telemetry_info_collect_plugin, target_repository, target_repository=target_repository)
-        if ret:
-            post_data = ret.get_return('post_data')
-            self._call_stdio('verbose', 'telemetry_data: %s' % post_data)
 
-            telemetry_post_plugin = self.plugin_manager.get_best_py_script_plugin('telemetry_post', 'general', '0.1')
-            return self.call_plugin(telemetry_post_plugin, target_repository, data=post_data)
+        telemetry_info_collect_plugin = self.plugin_manager.get_best_py_script_plugin('telemetry_info_collect', 'general', '0.1')
+        for repository in repositories:
+            if not self.call_plugin(telemetry_info_collect_plugin, repository, spacename='telemetry'):
+                return False
+
+        telemetry_post_plugin = self.plugin_manager.get_best_py_script_plugin('telemetry_post', 'general', '0.1')
+        return self.call_plugin(telemetry_post_plugin, repository, spacename='telemetry')
 
 
     def obdiag_gather(self, name, gather_type, opts):

example/all-components-min.yaml

@@ -107,6 +107,129 @@ obagent:
       ip: 172.19.33.4
   global:
     home_path: /root/obagent
+prometheus:
+  servers:
+    - 192.168.1.5
+  depends:
+    - obagent
+  global:
+    # The working directory for prometheus. prometheus is started under this directory. This is a required field.
+    home_path: /root/prometheus
+    # address: 0.0.0.0  # The ip address to bind to. Along with port, corresponds to the `web.listen-address` parameter.
+    # port: 9090 # The http port to use. Along with address, corresponds to the `web.listen-address` parameter.
+    # enable_lifecycle: true # Enable shutdown and reload via HTTP request. Corresponds to the `web.enable-lifecycle` parameter.
+    # data_dir: /root/prometheus/data # Base path for metrics storage. Corresponds to the `storage.tsdb.path` parameter.
+    # basic_auth_users:  # Usernames and passwords that have full access to the web server via basic authentication. Corresponds to the `basic_auth_users` parameter.
+    #   <username>: <password>  # The format of `basic_auth_users` : the key is the user name and the value is the password.
+    # web_config: # Content of Prometheus web service config file. The format is consistent with the file. However, `basic_auth_users` cannot be set in it. Please set `basic_auth_users` above if needed. Corresponds to the `web.config.file` parameter.
+    #   tls_server_config:
+    #     # Certificate and key files for server to use to authenticate to client.
+    #     cert_file: <filename>
+    #     key_file: <filename>
+    # config: # Configuration of the Prometheus service. The format is consistent with the Prometheus config file. Corresponds to the `config.file` parameter.
+    #   rule_files:
+    #   - rules/*rules.yaml
+    #   scrape_configs:
+    #   - job_name: prometheus
+    #     metrics_path: /metrics
+    #     scheme: http
+    #     static_configs:
+    #       - targets:
+    #           - localhost:9090
+    #   - job_name: node
+    #     basic_auth:
+    #       username: admin
+    #       password: root
+    #     metrics_path: /metrics/node/host
+    #     scheme: http
+    #     file_sd_configs:  # Set the targets to be collected by reading local files. The example is to collect targets corresponding to all yaml files in the 'targets' directory under $home_path.
+    #     - files:
+    #       - 'targets/*.yaml'
+    #   - job_name: ob_basic
+    #     basic_auth:
+    #       username: admin
+    #       password: root
+    #     metrics_path: /metrics/ob/basic
+    #     scheme: http
+    #     file_sd_configs:
+    #     - files:
+    #       - 'targets/*.yaml'
+    #   - job_name: ob_extra
+    #     basic_auth:
+    #       username: admin
+    #       password: root
+    #     metrics_path: /metrics/ob/extra
+    #     scheme: http
+    #     file_sd_configs:
+    #     - files:
+    #       - 'targets/*.yaml'
+    #   - job_name: agent
+    #     basic_auth:
+    #       username: admin
+    #       password: root
+    #     metrics_path: /metrics/stat
+    #     scheme: http
+    #     file_sd_configs:
+    #     - files:
+    #       - 'targets/*.yaml'
+    # additional_parameters: # Additional parameters for Prometheus service, among which `web.listen-address`, `web.enable-lifecycle`, `storage.tsdb.path`, `config.file` and `web.config.file` cannot be set. Please set them in the corresponding configuration above if needed.
+    # - log.level: debug
+grafana:
+  servers:
+    - 192.168.1.5
+  depends:
+    - prometheus
+  global:
+    home_path: /root/grafana
+    login_password: oceanbase # Grafana login password. The default value is 'oceanbase'.
+    # data_dir: # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used).$data_dir can be empty. The default value is $home_path/data.
+    # logs_dir: # Directory where grafana can store logs, can be empty. The default value is $data_dir/log.
+    # plugins_dir: # Directory where grafana will automatically scan and look for plugins, can be empty. The default value is $data_dir/plugins.
+    # provisioning_dir: # folder that contains provisioning config files that grafana will apply on startup and while running, can be empty. The default value is $home_path/conf/provisioning.
+    # temp_data_lifetime: # How long temporary images in data directory should be kept. Supported modifiers h (hours), m (minutes), Use 0 to never clean up temporary files, can be empty. The default value is 24h.
+    # log_max_days: # Expired days of log file(delete after max days), can be empty. The default value is 7.
+    # domian: # The public facing domain name used to access grafana from a browser, can be empty. The default value is $server.ip.
+    # port: # The http port to use, can be empty. The default value is 3000.
+
+    # # list of datasources to insert/update depending on what's available in the database, can be empty.
+    # # For more parameter settings, please refer to https://grafana.com/docs/grafana/latest/administration/provisioning/#datasources
+    # datasources:
+    #   name: # name of the datasource. Required and should not be 'OB-Prometheus'
+    #   type: # datasource type. Required
+    #   access: # access mode. direct or proxy. Required
+    #   url: # the url of datasource
+
+    # list of dashboards providers that load dashboards into Grafana from the local filesystem, can be empty.
+    # For more information, please refer to https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
+    # providers:
+    #   name: # an unique provider name. Required and should not be 'OceanBase Metrics'
+    #   type: # provider type. Default to 'file'
+    #   options:
+    #     path: # path to dashboard files on disk. Required when using the 'file' type
+
+    # # customize your Grafana instance by adding/modifying the custom configuration as follows
+    # # for more information, please refer to https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#configure-grafana
+    # # Here, setting parameters is required for format conversion.
+    # # For example, if the original grafana configuration format is
+    # #
+    # # [section1.section2]
+    # # key1 = value1
+    # # key2 = value2
+    # #
+    # # Then when writing the configuration below, you need to write it as
+    # #
+    # # section1:
+    # #   section2:
+    # #     key1: value1
+    # #     key2: value2
+    # #
+    # # Here we only list one item, because there are more than 500 items. Please add them according to your own needs.
+    # customize_config:
+    #   # original grafana configuration format is
+    #   # [server]
+    #   # protocol = http
+    #   server:
+    #     protocol: http
 ocp-express:
   depends:
     - oceanbase-ce