From a8929b2319d7f428f3f6fa9b1943e7619b31ab81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 11 Nov 2024 20:10:36 +0100 Subject: [PATCH] Update to the clickhouse multi-host config --- ansible/group_vars/clickhouse/vars.yml | 85 +++++++++++++ ansible/inventory | 5 + ansible/playbook.yml | 43 +++---- ansible/requirements.yml | 2 +- ansible/roles/bootstrap/tasks/main.yml | 8 ++ ansible/roles/bootstrap/templates/bashrc | 113 ++++++++++++++++++ ansible/roles/nftables/tasks/main.yml | 2 +- ansible/roles/oonidata/templates/tmp.sql | 5 + .../oonidata_clickhouse/defaults/main.yml | 69 ----------- .../oonidata_clickhouse/meta/requirements.yml | 9 +- .../roles/oonidata_clickhouse/tasks/main.yml | 26 ++++ tf/environments/prod/dns_records.tf | 42 ++++++- 12 files changed, 306 insertions(+), 103 deletions(-) create mode 100644 ansible/group_vars/clickhouse/vars.yml create mode 100644 ansible/roles/bootstrap/templates/bashrc create mode 100644 ansible/roles/oonidata/templates/tmp.sql diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml new file mode 100644 index 00000000..728fceb2 --- /dev/null +++ b/ansible/group_vars/clickhouse/vars.yml @@ -0,0 +1,85 @@ +clickhouse_version: "24.3.13.40" + +nftables_clickhouse_allow: + - fqdn: data1.htz-fsn.prod.ooni.nu + ip: 142.132.254.225 + - fqdn: data2.htz-fsn.prod.ooni.nu + ip: 88.198.54.12 + - fqdn: data3.htz-fsn.prod.ooni.nu + ip: 168.119.7.188 + +nftables_zookeeper_allow: + - fqdn: data1.htz-fsn.prod.ooni.nu + ip: 142.132.254.225 + - fqdn: data2.htz-fsn.prod.ooni.nu + ip: 88.198.54.12 + - fqdn: data3.htz-fsn.prod.ooni.nu + ip: 168.119.7.188 + +clickhouse_keeper: + tcp_port: 9181 + log_storage_path: /var/lib/clickhouse/coordination/log + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + coordination_settings: + operation_timeout_ms: 10000 + session_timeout_ms: 30000 + raft_logs_level: trace + keeper_servers: + - keeper_server: + server: data1.htz-fsn.prod.ooni.nu + id: 1 + hostname: clickhouse1.prod.ooni.io + port: 9234 + + - keeper_server: + server: data2.htz-fsn.prod.ooni.nu + id: 2 + hostname: clickhouse2.prod.ooni.io + port: 9234 + + - keeper_server: + server: data3.htz-fsn.prod.ooni.nu + id: 3 + hostname: clickhouse3.prod.ooni.io + port: 9234 + +clickhouse_remote_servers: + - server: + servername: oonidata_cluster + secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}" + shards: + - shard: + replicas: + - replica: + host: clickhouse1.prod.ooni.io + port: 9000 + - replica: + host: clickhouse2.prod.ooni.io + port: 9000 + - replica: + host: clickhouse3.prod.ooni.io + port: 9000 + +clickhouse_default_profiles: + default: + readonly: 1 + write: + readonly: 0 + +clickhouse_listen_host: "::" + +clickhouse_default_users: + - user: + name: default + password: + networks: + - 0.0.0.0 + profile: default + quota: default + - user: + name: write + password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}" + networks: + - 127.0.0.1 + profile: write + quota: default diff --git a/ansible/inventory b/ansible/inventory index cc15d418..6134062b 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -14,3 +14,8 @@ data3.htz-fsn.prod.ooni.nu [dev] oonidatatest.ooni.nu + +[clickhouse] +data1.htz-fsn.prod.ooni.nu +data2.htz-fsn.prod.ooni.nu +data3.htz-fsn.prod.ooni.nu diff --git a/ansible/playbook.yml b/ansible/playbook.yml index b71d2212..f8ba6ace 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -4,20 +4,8 @@ become: yes roles: - bootstrap - -- name: ClickHouse servers - hosts: clickhouse_servers - user: admin - become: true - vars: - clickhouse_reader_password: "{{ lookup('env', 'CLICKHOUSE_READER_PASSWORD') }}" - roles: - - clickhouse - handlers: - - name: Restart clickhouse-server - ansible.builtin.service: - name: clickhouse-server - state: restarted + tags: + - bootstrap - name: Update monitoring config hosts: monitoring.ooni.org @@ -27,14 +15,6 @@ - prometheus_blackbox_exporter - prometheus_alertmanager -- name: Deploy data.ooni.org host - hosts: data.ooni.org - become: true - roles: - #- clickhouse - - ssh_users - #- jupyterhub - - name: Setup OpenVPN server hosts: openvpn-server1.ooni.io become: true @@ -42,13 +22,24 @@ roles: - ssh_users -- name: Deploy oonidata hosts - hosts: oonidata.ooni.org +- name: Deploy oonidata clickhouse hosts (ssd backed) + hosts: + - data1.htz-fsn.prod.ooni.nu + - data2.htz-fsn.prod.ooni.nu become: true roles: - - oonidata + - oonidata_clickhouse + +- name: Deploy oonidata clickhouse hosts (hdd backed) + hosts: + - data3.htz-fsn.prod.ooni.nu + become: true + roles: + - oonidata_clickhouse + vars: + clickhouse_data_directory: /data/clickhouse -- name: Deploy notebook hosts +- name: Deploy notebook host hosts: notebook.ooni.org become: true vars: diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 3b4d5ae0..1bd4575b 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,4 +1,4 @@ - src: willshersystems.sshd - src: nginxinc.nginx - src: geerlingguy.certbot -- src: geerlingguy.node_exporter \ No newline at end of file +- src: geerlingguy.node_exporter diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index c2d5839b..fc2e3812 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -1,3 +1,11 @@ +- name: write bashrc template + ansible.builtin.template: + src: bashrc + dest: /etc/skel/.bashrc + owner: root + group: root + mode: u=rw,g=r,o=r + - ansible.builtin.include_role: name: ssh_users tags: diff --git a/ansible/roles/bootstrap/templates/bashrc b/ansible/roles/bootstrap/templates/bashrc new file mode 100644 index 00000000..4d34923b --- /dev/null +++ b/ansible/roles/bootstrap/templates/bashrc @@ -0,0 +1,113 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. +# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) +# for examples + +# If not running interactively, don't do anything +case $- in + *i*) ;; + *) return;; +esac + +# don't put duplicate lines or lines starting with space in the history. +# See bash(1) for more options +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) +HISTSIZE=1000 +HISTFILESIZE=2000 + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# If set, the pattern "**" used in a pathname expansion context will +# match all files and zero or more directories and subdirectories. +#shopt -s globstar + +# make less more friendly for non-text input files, see lesspipe(1) +#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +#force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +if [ "$color_prompt" = yes ]; then + PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\H\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +else + PS1='${debian_chroot:+($debian_chroot)}\u@\H:\w\$ ' +fi +unset color_prompt force_color_prompt + +# If this is an xterm set the title to user@host:dir +case "$TERM" in +xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; +*) + ;; +esac + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + #alias grep='grep --color=auto' + #alias fgrep='fgrep --color=auto' + #alias egrep='egrep --color=auto' +fi + +# colored GCC warnings and errors +#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01' + +# some more ls aliases +#alias ll='ls -l' +#alias la='ls -A' +#alias l='ls -CF' + +# Alias definitions. +# You may want to put all your additions into a separate file like +# ~/.bash_aliases, instead of adding them here directly. +# See /usr/share/doc/bash-doc/examples in the bash-doc package. + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if ! shopt -oq posix; then + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi +fi diff --git a/ansible/roles/nftables/tasks/main.yml b/ansible/roles/nftables/tasks/main.yml index 7c4258d2..5946772b 100644 --- a/ansible/roles/nftables/tasks/main.yml +++ b/ansible/roles/nftables/tasks/main.yml @@ -16,7 +16,7 @@ tags: - nftables -- name: "write {{ item.name }}.nft" +- name: "write nft config for item" ansible.builtin.template: src: "rule.nft.j2" dest: "/etc/ooni/nftables/tcp/{{ item.name }}.nft" diff --git a/ansible/roles/oonidata/templates/tmp.sql b/ansible/roles/oonidata/templates/tmp.sql new file mode 100644 index 00000000..54b1feac --- /dev/null +++ b/ansible/roles/oonidata/templates/tmp.sql @@ -0,0 +1,5 @@ +CREATE TABLE default.obs_web (`measurement_uid` String, `observation_idx` UInt16, `input` Nullable(String), `report_id` String, `measurement_start_time` DateTime64(3, 'UTC'), `software_name` String, `software_version` String, `test_name` String, `test_version` String, `bucket_date` String, `probe_asn` UInt32, `probe_cc` String, `probe_as_org_name` String, `probe_as_cc` String, `probe_as_name` String, `network_type` String, `platform` String, `origin` String, `engine_name` String, `engine_version` String, `architecture` String, `resolver_ip` String, `resolver_asn` UInt32, `resolver_cc` String, `resolver_as_org_name` String, `resolver_as_cc` String, `resolver_is_scrubbed` UInt8, `resolver_asn_probe` UInt32, `resolver_as_org_name_probe` String, `created_at` Nullable(DateTime('UTC')), `target_id` Nullable(String), `hostname` Nullable(String), `transaction_id` Nullable(UInt16), `ip` Nullable(String), `port` Nullable(UInt16), `ip_asn` Nullable(UInt32), `ip_as_org_name` Nullable(String), `ip_as_cc` Nullable(String), `ip_cc` Nullable(String), `ip_is_bogon` Nullable(UInt8), `dns_query_type` Nullable(String), `dns_failure` Nullable(String), `dns_engine` Nullable(String), `dns_engine_resolver_address` Nullable(String), `dns_answer_type` Nullable(String), `dns_answer` Nullable(String), `dns_answer_asn` Nullable(UInt32), `dns_answer_as_org_name` Nullable(String), `dns_t` Nullable(Float64), `tcp_failure` Nullable(String), `tcp_success` Nullable(UInt8), `tcp_t` Nullable(Float64), `tls_failure` Nullable(String), `tls_server_name` Nullable(String), `tls_version` Nullable(String), `tls_cipher_suite` Nullable(String), `tls_is_certificate_valid` Nullable(UInt8), `tls_end_entity_certificate_fingerprint` Nullable(String), `tls_end_entity_certificate_subject` Nullable(String), `tls_end_entity_certificate_subject_common_name` Nullable(String), `tls_end_entity_certificate_issuer` Nullable(String), `tls_end_entity_certificate_issuer_common_name` Nullable(String), `tls_end_entity_certificate_san_list` Array(String), `tls_end_entity_certificate_not_valid_after` Nullable(DateTime64(3, 'UTC')), `tls_end_entity_certificate_not_valid_before` Nullable(DateTime64(3, 'UTC')), `tls_certificate_chain_length` Nullable(UInt16), `tls_certificate_chain_fingerprints` Array(String), `tls_handshake_read_count` Nullable(UInt16), `tls_handshake_write_count` Nullable(UInt16), `tls_handshake_read_bytes` Nullable(UInt32), `tls_handshake_write_bytes` Nullable(UInt32), `tls_handshake_last_operation` Nullable(String), `tls_handshake_time` Nullable(Float64), `tls_t` Nullable(Float64), `http_request_url` Nullable(String), `http_network` Nullable(String), `http_alpn` Nullable(String), `http_failure` Nullable(String), `http_request_body_length` Nullable(UInt32), `http_request_method` Nullable(String), `http_runtime` Nullable(Float64), `http_response_body_length` Nullable(Int32), `http_response_body_is_truncated` Nullable(UInt8), `http_response_body_sha1` Nullable(String), `http_response_status_code` Nullable(UInt16), `http_response_header_location` Nullable(String), `http_response_header_server` Nullable(String), `http_request_redirect_from` Nullable(String), `http_request_body_is_truncated` Nullable(UInt8), `http_t` Nullable(Float64), `probe_analysis` Nullable(String)) +ENGINE = ReplicatedReplacingMergeTree +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time, probe_cc, probe_asn) SETTINGS index_granularity = 8192 \ No newline at end of file diff --git a/ansible/roles/oonidata_clickhouse/defaults/main.yml b/ansible/roles/oonidata_clickhouse/defaults/main.yml index 7547638a..e69de29b 100644 --- a/ansible/roles/oonidata_clickhouse/defaults/main.yml +++ b/ansible/roles/oonidata_clickhouse/defaults/main.yml @@ -1,69 +0,0 @@ -clickhouse_version: "24.3.13.40" - -clickhouse_default_profiles: - default: - readonly: 1 - write: - readonly: 0 - -clickhouse_default_users: - - user: - name: default - password: - networks: - - 0.0.0.0 - profile: default - quota: default - - user: - name: write - password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}" - networks: - - 127.0.0.1 - profile: write - quota: default - -clickhouse_keeper: - tcp_port: 9181 - log_storage_path: /var/lib/clickhouse/coordination/log - snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots - coordination_settings: - operation_timeout_ms: 10000 - session_timeout_ms: 30000 - raft_logs_level: trace - keeper_servers: - - keeper_server: - server: data1.htz-fsn.prod.ooni.nu - id: 1 - hostname: clickhouse1.prod.ooni.io - port: 9234 - - - keeper_server: - server: data2.htz-fsn.prod.ooni.nu - id: 2 - hostname: clickhouse2.prod.ooni.io - port: 9234 - - - keeper_server: - server: data3.htz-fsn.prod.ooni.nu - id: 3 - hostname: clickhouse3.prod.ooni.io - port: 9234 - -clickhouse_remote_servers: - - server: - servername: oonidata_cluster - secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}" - shards: - - shard: - replicas: - - replica: - host: clickhouse1.prod.ooni.io - port: 9000 - - shard: - - replica: - host: clickhouse2.prod.ooni.io - port: 9000 - - shard: - - replica: - host: clickhouse3.prod.ooni.io - port: 9000 diff --git a/ansible/roles/oonidata_clickhouse/meta/requirements.yml b/ansible/roles/oonidata_clickhouse/meta/requirements.yml index e9e6cdd8..53d654a5 100644 --- a/ansible/roles/oonidata_clickhouse/meta/requirements.yml +++ b/ansible/roles/oonidata_clickhouse/meta/requirements.yml @@ -1,5 +1,4 @@ ---- -dependencies: - - src: idealista.clickhouse_role - scm: git - version: 3.5.1 +- src: https://github.com/idealista/clickhouse_role + scm: git + version: 3.5.1 + name: idealista.clickhouse_role diff --git a/ansible/roles/oonidata_clickhouse/tasks/main.yml b/ansible/roles/oonidata_clickhouse/tasks/main.yml index f16ce772..a54d1084 100644 --- a/ansible/roles/oonidata_clickhouse/tasks/main.yml +++ b/ansible/roles/oonidata_clickhouse/tasks/main.yml @@ -3,3 +3,29 @@ tags: - oonidata - clickhouse + +- name: Create nftables rule for clickhouse native port + set_fact: + ch_nft_rule: "{{ ch_nft_rule | default([]) + ['add rule inet filter input ip saddr ' + item.ip + ' tcp dport 9000 counter accept comment \"incoming clickhouse from ' + item.fqdn + '\"'] }}" + loop: "{{ nftables_clickhouse_allow | rejectattr('fqdn', 'eq', inventory_hostname) | list }}" + +- name: Create nftables rule for zookeeper + set_fact: + zk_nft_rule: "{{ zk_nft_rule | default([]) + ['add rule inet filter input ip saddr ' + item.ip + ' tcp dport 9181 counter accept comment \"incoming zookeeper from ' + item.fqdn + '\"'] }}" + loop: "{{ nftables_zookeeper_allow | rejectattr('fqdn', 'eq', inventory_hostname) | list }}" + +- name: Create nftables rule for raft port + set_fact: + raft_nft_rule: "{{ raft_nft_rule | default([]) + ['add rule inet filter input ip saddr ' + item.ip + ' tcp dport 9234 counter accept comment \"incoming raft from ' + item.fqdn + '\"'] }}" + loop: "{{ nftables_zookeeper_allow | rejectattr('fqdn', 'eq', inventory_hostname) | list }}" + +- ansible.builtin.include_role: + name: nftables + vars: + nft_rules_tcp: + - name: 9000 + rules: "{{ ch_nft_rule }}" + - name: 9181 + rules: "{{ zk_nft_rule }}" + - name: 9234 + rules: "{{ raft_nft_rule }}" diff --git a/tf/environments/prod/dns_records.tf b/tf/environments/prod/dns_records.tf index 8ce681ab..fd77fa54 100644 --- a/tf/environments/prod/dns_records.tf +++ b/tf/environments/prod/dns_records.tf @@ -1006,10 +1006,50 @@ resource "aws_route53_record" "notebook-ooni-org-_A_" { zone_id = local.dns_root_zone_ooni_org } -resource "aws_route53_record" "data1-htz-fsn-prod-ooni-nu-_A_" { +resource "aws_route53_record" "data1-htz-fsn-prod-ooni-nu-_a_" { name = "data1.htz-fsn.prod.ooni.nu" records = ["142.132.254.225"] ttl = "60" type = "A" zone_id = local.dns_root_zone_ooni_nu } + +resource "aws_route53_record" "data2-htz-fsn-prod-ooni-nu-_A_" { + name = "data2.htz-fsn.prod.ooni.nu" + records = ["88.198.54.12"] + ttl = "60" + type = "A" + zone_id = local.dns_root_zone_ooni_nu +} + +resource "aws_route53_record" "data3-htz-fsn-prod-ooni-nu-_A_" { + name = "data3.htz-fsn.prod.ooni.nu" + records = ["168.119.7.188"] + ttl = "60" + type = "A" + zone_id = local.dns_root_zone_ooni_nu +} + +resource "aws_route53_record" "clickhouse1-prod-ooni-io-_a_" { + name = "clickhouse1.prod.ooni.io" + records = ["142.132.254.225"] + ttl = "60" + type = "A" + zone_id = local.dns_root_zone_ooni_io +} + +resource "aws_route53_record" "clickhouse2-prod-ooni-io-_A_" { + name = "clickhouse2.prod.ooni.io" + records = ["88.198.54.12"] + ttl = "60" + type = "A" + zone_id = local.dns_root_zone_ooni_io +} + +resource "aws_route53_record" "clickhouse3-prod-ooni-io-_A_" { + name = "clickhouse3.prod.ooni.io" + records = ["168.119.7.188"] + ttl = "60" + type = "A" + zone_id = local.dns_root_zone_ooni_io +}