Update to the clickhouse multi-host config

ansible/group_vars/clickhouse/vars.yml

@@ -0,0 +1,85 @@
+clickhouse_version: "24.3.13.40"
+
+nftables_clickhouse_allow:
+  - fqdn: data1.htz-fsn.prod.ooni.nu
+    ip: 142.132.254.225
+  - fqdn: data2.htz-fsn.prod.ooni.nu
+    ip: 88.198.54.12
+  - fqdn: data3.htz-fsn.prod.ooni.nu
+    ip: 168.119.7.188
+
+nftables_zookeeper_allow:
+  - fqdn: data1.htz-fsn.prod.ooni.nu
+    ip: 142.132.254.225
+  - fqdn: data2.htz-fsn.prod.ooni.nu
+    ip: 88.198.54.12
+  - fqdn: data3.htz-fsn.prod.ooni.nu
+    ip: 168.119.7.188
+
+clickhouse_keeper:
+  tcp_port: 9181
+  log_storage_path: /var/lib/clickhouse/coordination/log
+  snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots
+  coordination_settings:
+    operation_timeout_ms: 10000
+    session_timeout_ms: 30000
+    raft_logs_level: trace
+  keeper_servers:
+  - keeper_server:
+    server: data1.htz-fsn.prod.ooni.nu
+    id: 1
+    hostname: clickhouse1.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: data2.htz-fsn.prod.ooni.nu
+    id: 2
+    hostname: clickhouse2.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: data3.htz-fsn.prod.ooni.nu
+    id: 3
+    hostname: clickhouse3.prod.ooni.io
+    port: 9234
+
+clickhouse_remote_servers:
+  - server:
+    servername: oonidata_cluster
+    secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}"
+    shards:
+      - shard:
+        replicas:
+          - replica:
+            host: clickhouse1.prod.ooni.io
+            port: 9000
+          - replica:
+            host: clickhouse2.prod.ooni.io
+            port: 9000
+          - replica:
+            host: clickhouse3.prod.ooni.io
+            port: 9000
+
+clickhouse_default_profiles:
+  default:
+    readonly: 1
+  write:
+    readonly: 0
+
+clickhouse_listen_host: "::"
+
+clickhouse_default_users:
+  - user:
+    name: default
+    password:
+    networks:
+      - 0.0.0.0
+    profile: default
+    quota: default
+  - user:
+    name: write
+    password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}"
+    networks:
+      - 127.0.0.1
+    profile: write
+    quota: default

ansible/inventory

@@ -14,3 +14,8 @@ data3.htz-fsn.prod.ooni.nu
 
 [dev]
 oonidatatest.ooni.nu
+
+[clickhouse]
+data1.htz-fsn.prod.ooni.nu
+data2.htz-fsn.prod.ooni.nu
+data3.htz-fsn.prod.ooni.nu

ansible/playbook.yml

@@ -4,20 +4,8 @@
   become: yes
   roles:
     - bootstrap
-
-- name: ClickHouse servers
-  hosts: clickhouse_servers
-  user: admin
-  become: true
-  vars:
-    clickhouse_reader_password: "{{ lookup('env', 'CLICKHOUSE_READER_PASSWORD') }}"
-  roles:
-    - clickhouse
-  handlers:
-    - name: Restart clickhouse-server
-      ansible.builtin.service:
-        name: clickhouse-server
-        state: restarted
+  tags:
+    - bootstrap
 
 - name: Update monitoring config
   hosts: monitoring.ooni.org
@@ -27,28 +15,31 @@
     - prometheus_blackbox_exporter
     - prometheus_alertmanager
 
-- name: Deploy data.ooni.org host
-  hosts: data.ooni.org
-  become: true
-  roles:
-    #- clickhouse
-    - ssh_users
-    #- jupyterhub
-
 - name: Setup OpenVPN server
   hosts: openvpn-server1.ooni.io
   become: true
   remote_user: root
   roles:
     - ssh_users
 
-- name: Deploy oonidata hosts
-  hosts: oonidata.ooni.org
+- name: Deploy oonidata clickhouse hosts (ssd backed)
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+    - data2.htz-fsn.prod.ooni.nu
   become: true
   roles:
-    - oonidata
+    - oonidata_clickhouse
+
+- name: Deploy oonidata clickhouse hosts (hdd backed)
+  hosts:
+    - data3.htz-fsn.prod.ooni.nu
+  become: true
+  roles:
+    - oonidata_clickhouse
+  vars:
+    clickhouse_data_directory: /data/clickhouse
 
-- name: Deploy notebook hosts
+- name: Deploy notebook host
   hosts: notebook.ooni.org
   become: true
   vars:

ansible/requirements.yml

@@ -1,4 +1,4 @@
 - src: willshersystems.sshd
 - src: nginxinc.nginx
 - src: geerlingguy.certbot
-- src: geerlingguy.node_exporter
+- src: geerlingguy.node_exporter

ansible/roles/bootstrap/tasks/main.yml

@@ -1,3 +1,11 @@
+- name: write bashrc template
+  ansible.builtin.template:
+    src: bashrc
+    dest: /etc/skel/.bashrc
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+
 - ansible.builtin.include_role:
     name: ssh_users
   tags:

ansible/roles/bootstrap/templates/bashrc

@@ -0,0 +1,113 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\H\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\H:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    #alias grep='grep --color=auto'
+    #alias fgrep='fgrep --color=auto'
+    #alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+#alias ll='ls -l'
+#alias la='ls -A'
+#alias l='ls -CF'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi

ansible/roles/nftables/tasks/main.yml

@@ -16,7 +16,7 @@
   tags:
     - nftables
 
-- name: "write {{ item.name }}.nft"
+- name: "write nft config for item"
   ansible.builtin.template:
     src: "rule.nft.j2"
     dest: "/etc/ooni/nftables/tcp/{{ item.name }}.nft"

ansible/roles/oonidata/templates/tmp.sql

@@ -0,0 +1,5 @@
+CREATE TABLE default.obs_web (`measurement_uid` String, `observation_idx` UInt16, `input` Nullable(String), `report_id` String, `measurement_start_time` DateTime64(3, 'UTC'), `software_name` String, `software_version` String, `test_name` String, `test_version` String, `bucket_date` String, `probe_asn` UInt32, `probe_cc` String, `probe_as_org_name` String, `probe_as_cc` String, `probe_as_name` String, `network_type` String, `platform` String, `origin` String, `engine_name` String, `engine_version` String, `architecture` String, `resolver_ip` String, `resolver_asn` UInt32, `resolver_cc` String, `resolver_as_org_name` String, `resolver_as_cc` String, `resolver_is_scrubbed` UInt8, `resolver_asn_probe` UInt32, `resolver_as_org_name_probe` String, `created_at` Nullable(DateTime('UTC')), `target_id` Nullable(String), `hostname` Nullable(String), `transaction_id` Nullable(UInt16), `ip` Nullable(String), `port` Nullable(UInt16), `ip_asn` Nullable(UInt32), `ip_as_org_name` Nullable(String), `ip_as_cc` Nullable(String), `ip_cc` Nullable(String), `ip_is_bogon` Nullable(UInt8), `dns_query_type` Nullable(String), `dns_failure` Nullable(String), `dns_engine` Nullable(String), `dns_engine_resolver_address` Nullable(String), `dns_answer_type` Nullable(String), `dns_answer` Nullable(String), `dns_answer_asn` Nullable(UInt32), `dns_answer_as_org_name` Nullable(String), `dns_t` Nullable(Float64), `tcp_failure` Nullable(String), `tcp_success` Nullable(UInt8), `tcp_t` Nullable(Float64), `tls_failure` Nullable(String), `tls_server_name` Nullable(String), `tls_version` Nullable(String), `tls_cipher_suite` Nullable(String), `tls_is_certificate_valid` Nullable(UInt8), `tls_end_entity_certificate_fingerprint` Nullable(String), `tls_end_entity_certificate_subject` Nullable(String), `tls_end_entity_certificate_subject_common_name` Nullable(String), `tls_end_entity_certificate_issuer` Nullable(String), `tls_end_entity_certificate_issuer_common_name` Nullable(String), `tls_end_entity_certificate_san_list` Array(String), `tls_end_entity_certificate_not_valid_after` Nullable(DateTime64(3, 'UTC')), `tls_end_entity_certificate_not_valid_before` Nullable(DateTime64(3, 'UTC')), `tls_certificate_chain_length` Nullable(UInt16), `tls_certificate_chain_fingerprints` Array(String), `tls_handshake_read_count` Nullable(UInt16), `tls_handshake_write_count` Nullable(UInt16), `tls_handshake_read_bytes` Nullable(UInt32), `tls_handshake_write_bytes` Nullable(UInt32), `tls_handshake_last_operation` Nullable(String), `tls_handshake_time` Nullable(Float64), `tls_t` Nullable(Float64), `http_request_url` Nullable(String), `http_network` Nullable(String), `http_alpn` Nullable(String), `http_failure` Nullable(String), `http_request_body_length` Nullable(UInt32), `http_request_method` Nullable(String), `http_runtime` Nullable(Float64), `http_response_body_length` Nullable(Int32), `http_response_body_is_truncated` Nullable(UInt8), `http_response_body_sha1` Nullable(String), `http_response_status_code` Nullable(UInt16), `http_response_header_location` Nullable(String), `http_response_header_server` Nullable(String), `http_request_redirect_from` Nullable(String), `http_request_body_is_truncated` Nullable(UInt8), `http_t` Nullable(Float64), `probe_analysis` Nullable(String)) 
+ENGINE = ReplicatedReplacingMergeTree 
+PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2))
+PRIMARY KEY (measurement_uid, observation_idx) 
+ORDER BY (measurement_uid, observation_idx, measurement_start_time, probe_cc, probe_asn) SETTINGS index_granularity = 8192

ansible/roles/oonidata_clickhouse/defaults/main.yml

@@ -1,69 +0,0 @@
-clickhouse_version: "24.3.13.40"
-
-clickhouse_default_profiles:
-  default:
-    readonly: 1
-  write:
-    readonly: 0
-
-clickhouse_default_users:
-  - user:
-    name: default
-    password:
-    networks:
-      - 0.0.0.0
-    profile: default
-    quota: default
-  - user:
-    name: write
-    password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}"
-    networks:
-      - 127.0.0.1
-    profile: write
-    quota: default
-
-clickhouse_keeper:
-  tcp_port: 9181
-  log_storage_path: /var/lib/clickhouse/coordination/log
-  snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots
-  coordination_settings:
-    operation_timeout_ms: 10000
-    session_timeout_ms: 30000
-    raft_logs_level: trace
-  keeper_servers:
-  - keeper_server:
-    server: data1.htz-fsn.prod.ooni.nu
-    id: 1
-    hostname: clickhouse1.prod.ooni.io
-    port: 9234
-
-  - keeper_server:
-    server: data2.htz-fsn.prod.ooni.nu
-    id: 2
-    hostname: clickhouse2.prod.ooni.io
-    port: 9234
-
-  - keeper_server:
-    server: data3.htz-fsn.prod.ooni.nu
-    id: 3
-    hostname: clickhouse3.prod.ooni.io
-    port: 9234
-
-clickhouse_remote_servers:
-  - server:
-    servername: oonidata_cluster
-    secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}"
-    shards:
-      - shard:
-        replicas:
-          - replica:
-            host: clickhouse1.prod.ooni.io
-            port: 9000
-      - shard:
-          - replica:
-            host: clickhouse2.prod.ooni.io
-            port: 9000
-      - shard:
-          - replica:
-            host: clickhouse3.prod.ooni.io
-            port: 9000

ansible/roles/oonidata_clickhouse/meta/requirements.yml

@@ -1,5 +1,4 @@
----
-dependencies:
-  - src: idealista.clickhouse_role
-    scm: git
-    version: 3.5.1
+- src: https://github.com/idealista/clickhouse_role
+  scm: git
+  version: 3.5.1
+  name: idealista.clickhouse_role