From 68c8f5b3c692d27167fa9b605b35f03ff6f62172 Mon Sep 17 00:00:00 2001 From: Andrew Klychkov Date: Fri, 1 Nov 2024 08:29:02 +0100 Subject: [PATCH 1/3] clickhouse_cfg_info: add XML files support --- plugins/modules/clickhouse_cfg_info.py | 63 +- requirements.txt | 1 + .../clickhouse_cfg_info/files/config.xml | 1744 +++++++++++++++++ .../clickhouse_cfg_info/tasks/initial.yml | 22 +- .../clickhouse_cfg_info/tasks/main.yml | 7 + 5 files changed, 1822 insertions(+), 15 deletions(-) create mode 100644 tests/integration/targets/clickhouse_cfg_info/files/config.xml diff --git a/plugins/modules/clickhouse_cfg_info.py b/plugins/modules/clickhouse_cfg_info.py index c412900..aecb25a 100644 --- a/plugins/modules/clickhouse_cfg_info.py +++ b/plugins/modules/clickhouse_cfg_info.py @@ -17,7 +17,8 @@ description: - Retrieves ClickHouse config file content and returns it as JSON. - - Only config files in YAML format are supported at the moment. + - Supports config files in the YAML and XML formats. + - When the config file is XML, options values are returned as strings. - Does not change server state. attributes: @@ -36,11 +37,10 @@ required: true requirements: - - pyyaml + - pyyaml (for YAML config files) + - xmltodict (for XML config files) ''' -# TODO: it should also handle xml configs - EXAMPLES = r''' - name: Get server information register: result @@ -52,7 +52,15 @@ var: result ''' -RETURN = r''' # ''' +RETURN = r''' +config: + description: + - The content of the config file. + - When the file is XML, options values + are returned as strings. + returned: success + type: dict +''' try: import yaml @@ -60,23 +68,42 @@ except ImportError: HAS_PYYAML = False +try: + import xmltodict + HAS_XMLTODICT = True +except ImportError: + HAS_XMLTODICT = False + from ansible.module_utils.basic import ( AnsibleModule, missing_required_lib, ) -def load_from_yaml(module, path): +def load_config(module, load_func, path): try: f = open(path, 'r') - content = yaml.safe_load(f) + content = load_func(f) except Exception as e: - module.fail_json(msg="Could not open/load YAML from the file %s: %s" % (path, e)) + fail_msg = "Could not open/load from file %s: %s" % (path, e) + module.fail_json(msg=fail_msg) else: f.close() return content +def load_from_yaml(f): + return yaml.safe_load(f) + + +def load_from_xml(f): + return xmltodict.parse(f.read())['clickhouse'] + + +def is_xml(path): + return True if len(path) > 4 and path[-4:] == '.xml' else False + + def main(): argument_spec = {} argument_spec.update( @@ -89,13 +116,25 @@ def main(): supports_check_mode=True, ) - if not HAS_PYYAML: - module.fail_json(msg=missing_required_lib('pyyaml')) + path = module.params['path'] + + # When XML + if is_xml(path): + if not HAS_XMLTODICT: + module.fail_json(msg=missing_required_lib('xmltodict')) + load_func = load_from_xml + + # When YAML, the default + else: + if not HAS_PYYAML: + module.fail_json(msg=missing_required_lib('pyyaml')) + load_func = load_from_yaml - cfg_content = load_from_yaml(module, module.params['path']) + # Load content as dict + cfg_content = load_config(module, load_func, path) # Users will get this in JSON output after execution - module.exit_json(changed=False, **cfg_content) + module.exit_json(changed=False, config=cfg_content) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index d8bcaef..7e09aa5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ clickhouse_driver pyyaml +xmltodict diff --git a/tests/integration/targets/clickhouse_cfg_info/files/config.xml b/tests/integration/targets/clickhouse_cfg_info/files/config.xml new file mode 100644 index 0000000..9807f8c --- /dev/null +++ b/tests/integration/targets/clickhouse_cfg_info/files/config.xml @@ -0,0 +1,1744 @@ + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + + + + + + + https://{bucket}.s3.amazonaws.com + + + https://storage.googleapis.com/{bucket} + + + https://{bucket}.oss.aliyuncs.com + + + + + +
+ Access-Control-Allow-Origin + * +
+
+ Access-Control-Allow-Headers + origin, x-requested-with, x-clickhouse-format, x-clickhouse-user, x-clickhouse-key, Authorization +
+
+ Access-Control-Allow-Methods + POST, GET, OPTIONS +
+
+ Access-Control-Max-Age + 86400 +
+
+ + + + + + 8123 + + + 9000 + + + + + + 9004 + + + 9005 + + + + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4096 + + + 10 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + + + + none + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 0 + 2 + + + 1000 + + + 0 + + + + 10000 + + + + + + true + + + 0.9 + + + 4194304 + + + 0 + + + + + + 8589934592 + + + + + + + + + + + + + + + + + + + + + /var/lib/clickhouse/caches/ + + false + + + /var/lib/clickhouse/ + + + + + + + /var/lib/clickhouse/tmp/ + + + 1 + 1 + 1 + + + sha256_password + + + 12 + + + + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + + true + + + true + + + true + + + true + + + true + + + false + + + 600 + + + + default + + + SQL_ + + + + + + + + + default + + + + + + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc-bridge/ + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + + + + + + + + + + + + + + + + localhost + 9000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + + + + + + + 7500 + + 1048576 + + 8192 + + 524288 + + false + + + +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + + false +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false +
+ + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + part_log
+ toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false +
+ + + + system + text_log
+ 7500 + 1048576 + 8192 + 524288 + false + trace +
+ + + + system + metric_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false +
+ + + + system + error_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false +
+ + + + system + query_metric_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false +
+ + + + system + asynchronous_metric_log
+ 7000 + 1048576 + 8192 + 524288 + false +
+ + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + 1048576 + 8192 + 524288 + false +
+ + + + + system + crash_log
+ + + 1000 + 1024 + 1024 + 512 + true +
+ + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false +
+ + + + system + asynchronous_insert_log
+ + 7500 + 1048576 + 8192 + 524288 + false + event_date + event_date + INTERVAL 3 DAY +
+ + + + system + backup_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + s3queue_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + blob_storage_log
+ toYYYYMM(event_date) + 7500 + event_date + INTERVAL 30 DAY +
+ + + + + + + + + + + + + *_dictionary.*ml + + + true + + + true + + + *_function.*ml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /clickhouse/task_queue/ddl + + /clickhouse/task_queue/replicas + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + /usr/share/clickhouse/protos/ + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + false + + + + + + + + + + + + + + + + + + + + + + + + + backups + + + true + + + + + + + + + + + +
diff --git a/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml b/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml index 16e4c26..4619741 100644 --- a/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml +++ b/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml @@ -8,6 +8,11 @@ src: config.yaml dest: /tmp/config.yaml +- name: Put config.yaml file in place + ansible.builtin.copy: + src: config.xml + dest: /tmp/config.xml + - name: Retrieve info from config.yaml register: result community.clickhouse.clickhouse_cfg_info: @@ -16,8 +21,19 @@ - name: Check the content ansible.builtin.assert: that: - - result['http_port'] == 8123 - - result['grpc']['compression_level'] == 'medium' + - result['config']['http_port'] == 8123 + - result['config']['tcp_port'] == 9000 + +- name: Retrieve info from config.xml + register: result + community.clickhouse.clickhouse_cfg_info: + path: /tmp/config.xml + +- name: Check the content + ansible.builtin.assert: + that: + - result['config']['http_port'] == '8123' + - result['config']['tcp_port'] == '9000' - name: Try to retreive info from a non-existing file register: result @@ -29,4 +45,4 @@ ansible.builtin.assert: that: - result is failed - - result.msg is search('Could not open/load YAML from the file') + - result.msg is search('Could not open/load from file') diff --git a/tests/integration/targets/clickhouse_cfg_info/tasks/main.yml b/tests/integration/targets/clickhouse_cfg_info/tasks/main.yml index c8ea342..17127f5 100644 --- a/tests/integration/targets/clickhouse_cfg_info/tasks/main.yml +++ b/tests/integration/targets/clickhouse_cfg_info/tasks/main.yml @@ -3,5 +3,12 @@ # and should not be used as examples of how to write Ansible roles # #################################################################### +- name: Install Python packages + ansible.builtin.pip: + name: + - pyyaml + - xmltodict + state: present + # Initial CI tests of clickhouse_info module - import_tasks: initial.yml From a5022e6c1ccab232abf088bab256e2e1e77f2bbf Mon Sep 17 00:00:00 2001 From: Andrew Klychkov Date: Fri, 1 Nov 2024 10:20:07 +0100 Subject: [PATCH 2/3] Fix typo --- tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml b/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml index 4619741..9865fa1 100644 --- a/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml +++ b/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml @@ -8,7 +8,7 @@ src: config.yaml dest: /tmp/config.yaml -- name: Put config.yaml file in place +- name: Put config.xml file in place ansible.builtin.copy: src: config.xml dest: /tmp/config.xml From 2ff921bc823299c31e2b0eaabc94c8e86d8c2f70 Mon Sep 17 00:00:00 2001 From: Andrew Klychkov Date: Fri, 1 Nov 2024 15:20:05 +0100 Subject: [PATCH 3/3] Add values conversion --- plugins/modules/clickhouse_cfg_info.py | 43 +++++++++++++++++-- .../clickhouse_cfg_info/tasks/initial.yml | 12 +++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/plugins/modules/clickhouse_cfg_info.py b/plugins/modules/clickhouse_cfg_info.py index aecb25a..1c033ac 100644 --- a/plugins/modules/clickhouse_cfg_info.py +++ b/plugins/modules/clickhouse_cfg_info.py @@ -18,7 +18,6 @@ description: - Retrieves ClickHouse config file content and returns it as JSON. - Supports config files in the YAML and XML formats. - - When the config file is XML, options values are returned as strings. - Does not change server state. attributes: @@ -56,8 +55,6 @@ config: description: - The content of the config file. - - When the file is XML, options values - are returned as strings. returned: success type: dict ''' @@ -97,13 +94,51 @@ def load_from_yaml(f): def load_from_xml(f): - return xmltodict.parse(f.read())['clickhouse'] + content = xmltodict.parse(f.read())['clickhouse'] + # This lib loads all values including boolean + # or numerical as strings. Let's convert if possible. + return convert_str_vals_in_dict(content) def is_xml(path): return True if len(path) > 4 and path[-4:] == '.xml' else False +def convert_str_vals_in_dict(d): + """Recursively traverse a dict and covert + string values to appropirate types. + """ + for key, val in d.items(): + if isinstance(val, dict): + convert_str_vals_in_dict(val) + + elif isinstance(val, list): + for i, v in enumerate(val): + if isinstance(v, dict): + convert_str_vals_in_dict(val[i]) + else: + d[key] = convert(val) + + return d + + +def convert(val): + # Try to convert or just return it back + try: + if val == 'false': + val = False + elif val == 'true': + val = True + elif val.isnumeric(): + val = int(val) + else: + val = float(val) + except Exception: + return val + + return val + + def main(): argument_spec = {} argument_spec.update( diff --git a/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml b/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml index 9865fa1..97fb12e 100644 --- a/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml +++ b/tests/integration/targets/clickhouse_cfg_info/tasks/initial.yml @@ -23,6 +23,10 @@ that: - result['config']['http_port'] == 8123 - result['config']['tcp_port'] == 9000 + - result['config']['error_log']['collect_interval_milliseconds'] == 1000 + - result['config']['graphite_rollup_example']['default']['retention'][0]['age'] == 0 + - result['config']['crash_log']['partition_by'] == '' + - result['config']['grpc']['enable_ssl'] == false - name: Retrieve info from config.xml register: result @@ -32,8 +36,12 @@ - name: Check the content ansible.builtin.assert: that: - - result['config']['http_port'] == '8123' - - result['config']['tcp_port'] == '9000' + - result['config']['http_port'] == 8123 + - result['config']['tcp_port'] == 9000 + - result['config']['error_log']['collect_interval_milliseconds'] == 1000 + - result['config']['graphite_rollup_example']['default']['retention'][0]['age'] == 0 + - result['config']['crash_log']['partition_by'] == none + - result['config']['grpc']['enable_ssl'] == false - name: Try to retreive info from a non-existing file register: result