Work towards production deployment of data pipeline v5

* Make use of clickhouse idealista role
* Refactor nftables role
* Tighten up ssh users related configuration
* Improve the bootstrap host playbook

ansible/group_vars/all/vars.yml

@@ -23,7 +23,7 @@ ssh_users:
     keys:
       - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBXprrutdT6AhrV9hWBKjyzq6RqGmCBWpWxi3qwJyRcBJfkiEYKV9QWl3H0g/Sg9JzLd9lWG2yfAai7cyBAT4Ih0+OhwQ0V7wkhBn4YkNjs7d4BGPHjuLIywS9VtmiyH7VafikMjmqPLL/uPBIbRrx9RuSfLkAuN9XFZpVmqzWY8ePpcRCvnG6ucPxEY8o+4j5nfTrgxSaIT31kH16/PFJe07tn1SZjxZE4sZTz/p9xKt6s8HXmlP3RdnXSpXWmH8ZwYDrNhkcH8m6mC3giiqSKThFdwvQVflRRvn9pAlUOhy6KIBtAt1KobVJtOCPrrkcLhQ1C+2P9wKhfYspCGrScFGnrUqumLxPpwlqILxJvmgqGAtkm8Ela9f2D9sEv8CUv5x9XptZKlyRhtOLixvLYoJlwfXXnmXa8T1pg8+4063BhHUOu/bg0InpSp3hdscOfk0R8FtDlXnn6COwbPXynIt4PxzIxD/WQhP0ymgH3ky6ClB5wRBVhOqYvxQw32n2QFS9A5ocga+nATiOE7BTOufgmDCA/OIXfJ/GukXRaMCBsvlx7tObHS1LOMt0I+WdoOEjI0ARUrFzwoiTrs9QYmd922e7S35EnheT3JjnCTjebJrCNtwritUy8vjsN/M27wJs7MAXleT7drwXXnm+3xYrH+4KQ+ru0dxMe1zfBw== [email protected]"
 
-admin_usernames: [ art, majakomel, mehul, norbel ]
+admin_usernames: [ art, mehul ]
 root_usernames: [ art, mehul ]
-non_admin_usernames: [ agrabeli ]
-deactivated_usernames: [ sbs, federico, sarath ]
+non_admin_usernames: [ ]
+deactivated_usernames: [ sbs, federico, sarath ]

ansible/group_vars/dev/vars.yml

@@ -1 +1,3 @@
-prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"
+admin_usernames: [ art, mehul, norbel, majakomel ]
+non_admin_usernames: [ agrabeli ]

ansible/group_vars/prod/vars.yml

@@ -1 +1,3 @@
-prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"
+admin_usernames: [ art, mehul ]
+non_admin_usernames: [ ]

ansible/host_vars/data3.htz-fsn.prod.ooni.nu

@@ -0,0 +1 @@
+non_admin_usernames: [ ]

ansible/inventory

@@ -8,6 +8,9 @@ oonidata.ooni.org
 monitoring.ooni.org
 openvpn-server1.ooni.io
 notebook.ooni.org
+data1.htz-fsn.prod.ooni.nu
+data2.htz-fsn.prod.ooni.nu
+data3.htz-fsn.prod.ooni.nu
 
 [dev]
 oonidatatest.ooni.nu

ansible/playbook-bootstrap.yml

@@ -4,5 +4,4 @@
   hosts: all
   remote_user: root
   roles:
-    - ssh_users
     - bootstrap

ansible/roles/bootstrap/tasks/main.yml

@@ -22,6 +22,7 @@
       - man-db
       - mtr
       - net-tools
+      - nvme-cli
       - openssl
       - python3-passlib
       - rsync

ansible/roles/nftables/defaults/main.yml

@@ -0,0 +1,10 @@
+nft_rules_tcp:
+  - name: 22
+    rules:
+      - add rule inet filter input tcp dport 22 counter accept comment "Incoming SSH"
+  #- name: 80
+  #  rules:
+  #    - add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP"
+  #- name: 443
+  #  rules:
+  #    - add rule inet filter input tcp dport 443 counter accept comment "incoming HTTPS"

ansible/roles/nftables/tasks/main.yml

@@ -16,12 +16,15 @@
   tags:
     - nftables
 
-- name: allow SSH
-  ansible.builtin.blockinfile:
-    path: /etc/ooni/nftables/tcp/22.nft
-    create: yes
-    block: |
-      add rule inet filter input tcp dport 22 counter accept comment "Incoming SSH"
+- name: "write {{ item.name }}.nft"
+  ansible.builtin.template:
+    src: "rule.nft.j2"
+    dest: "/etc/ooni/nftables/tcp/{{ item.name }}.nft"
+  vars:
+    rules: "{{ item.rules }}"
+  loop: "{{ nft_rules_tcp }}"
+  notify:
+    - Reload nftables
   tags:
     - nftables
 

ansible/roles/nftables/templates/nftables.conf

@@ -38,4 +38,3 @@ include "/etc/ooni/nftables/tcp/*.nft"
 
 # Configure any other rule
 include "/etc/ooni/nftables/*.nft"
-

ansible/roles/nftables/templates/rule.nft.j2

@@ -0,0 +1,4 @@
+{{ ansible_managed | comment }}
+{% for entry in rules %}
+{{ entry }}
+{% endfor %}

ansible/roles/nginx/handlers/main.yml

@@ -13,9 +13,3 @@
   service:
     name: nginx
     state: reloaded
-
-- name: reload nftables
-  tags: nftables
-  ansible.builtin.systemd_service:
-    name: nftables
-    state: reloaded

ansible/roles/nginx/tasks/main.yml

@@ -1,4 +1,18 @@
 ---
+- ansible.builtin.include_role:
+    name: nftables
+  vars:
+    nft_rules_tcp:
+      - name: 80
+        rules:
+          - add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP"
+      - name: 443
+        rules:
+          - add rule inet filter input tcp dport 443 counter accept comment "incoming HTTPS"
+  tags:
+    - nginx
+    - nftables
+
 - name: install nginx
   include_role:
     name: nginxinc.nginx
@@ -47,25 +61,3 @@
     mode: 0755
   tags:
     - nftables
-
-- name: allow incoming TCP connections to Nginx port 80
-  blockinfile:
-    path: /etc/ooni/nftables/tcp/80.nft
-    create: yes
-    block: |
-      add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP"
-  notify:
-    - reload nftables
-  tags:
-    - nginx
-
-- name: allow incoming TCP connections to Nginx port 443
-  blockinfile:
-    path: /etc/ooni/nftables/tcp/443.nft
-    create: yes
-    block: |
-      add rule inet filter input tcp dport 443 counter accept comment "incoming HTTP"
-  notify:
-    - reload nftables
-  tags:
-    - nginx

ansible/roles/oonidata/defaults/main.yml

@@ -7,6 +7,7 @@ admin_group_name: admin
 enable_oonipipeline_worker: true
 enable_oonidata_proxy: false
 enable_jupyterhub: true
+enable_clickhouse: true
 conda_forge_packages:
   - seaborn
   - dask
@@ -21,3 +22,78 @@ pip_packages:
   - "clickhouse-driver"
   - pomegranate
   - pgmpy
+apt_packages:
+  - net-tools
+  - curl
+  - git
+  - socat
+
+clickhouse_version: "24.3.13.40"
+
+clickhouse_default_profiles:
+  default:
+    readonly: 1
+  write:
+    readonly: 0
+
+clickhouse_default_users:
+  - user:
+    name: default
+    password:
+    networks:
+      - 0.0.0.0
+    profile: default
+    quota: default
+  - user:
+    name: write
+    password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}"
+    networks:
+      - 127.0.0.1
+    profile: write
+    quota: default
+
+clickhouse_keeper:
+  tcp_port: 9181
+  log_storage_path: /var/lib/clickhouse/coordination/log
+  snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots
+  coordination_settings:
+    operation_timeout_ms: 10000
+    session_timeout_ms: 30000
+    raft_logs_level: trace
+  keeper_servers:
+  - keeper_server:
+    server: data1.htz-fsn.prod.ooni.nu
+    id: 1
+    hostname: clickhouse1.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: data2.htz-fsn.prod.ooni.nu
+    id: 2
+    hostname: clickhouse2.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: data3.htz-fsn.prod.ooni.nu
+    id: 3
+    hostname: clickhouse3.prod.ooni.io
+    port: 9234
+
+clickhouse_remote_servers:
+  - server:
+    servername: oonidata_cluster
+    secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}"
+    shards:
+      - shard:
+        replicas:
+          - replica:
+            host: clickhouse1.prod.ooni.io
+            port: 9000
+      - shard:
+          - replica:
+            host: clickhouse2.prod.ooni.io
+            port: 9000
+      - shard:
+          - replica:
+            host: clickhouse3.prod.ooni.io
+            port: 9000

ansible/roles/oonidata/meta/requirements.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - geerlingguy.certbot
+  - src: idealista.clickhouse_role
+    scm: git
+    version: 3.5.1

ansible/roles/oonidata/tasks/clickhouse.yml

@@ -0,0 +1,5 @@
+- ansible.builtin.include_role:
+    name: idealista.clickhouse_role
+  tags:
+    - oonidata
+    - clickhouse

ansible/roles/oonidata/tasks/jupyterhub.yml

@@ -89,3 +89,24 @@
     - oonidata
     - jupyterhub
     - config
+
+- ansible.builtin.include_role:
+    name: nginx
+  tags:
+    - oonidata
+    - nginx
+
+- ansible.builtin.include_role:
+    name: geerlingguy.certbot
+  tags:
+    - oonidata
+    - certbot
+  vars:
+    certbot_admin_email: [email protected]
+    certbot_create_extra_args: ""
+    certbot_create_if_missing: true
+    certbot_create_standalone_stop_services:
+      - nginx
+    certbot_certs:
+      - domains:
+          - "{{ inventory_hostname }}"

ansible/roles/oonidata/tasks/main.yml

@@ -22,34 +22,9 @@
     - oonidata
     - jupyterhub
 
-- ansible.builtin.include_role:
-    name: nginx
-  tags:
-    - oonidata
-    - nginx
-
-- ansible.builtin.include_role:
-    name: geerlingguy.certbot
-  tags:
-    - oonidata
-    - certbot
-  vars:
-    certbot_admin_email: [email protected]
-    certbot_create_extra_args: ""
-    certbot_create_if_missing: true
-    certbot_create_standalone_stop_services:
-      - nginx
-    certbot_certs:
-      - domains:
-          - "{{ inventory_hostname }}"
-
-- name: Install oonipipeline requirements
+- name: Install apt packages
   ansible.builtin.apt:
-    name:
-      - net-tools
-      - curl
-      - git
-      - socat
+    name: "{{ apt_packages }}"
   tags:
     - oonidata
     - oonipipeline
@@ -87,7 +62,14 @@
     - oonidata
     - oonipipeline
 
+- ansible.builtin.import_tasks: clickhouse.yml
+  when: enable_clickhouse
+  tags:
+    - oonidata
+    - clickhouse
+
 - name: Write oonidataproxy service
+  when: enable_oonidata_proxy
   ansible.builtin.template:
     src: oonidata-proxy.service.j2
     dest: "/etc/systemd/system/oonidata-proxy.service"

ansible/roles/prometheus_node_exporter/handlers/main.yml

@@ -13,9 +13,3 @@
   ansible.builtin.systemd_service:
     name: nginx
     state: restarted
-
-- name: Reload nftables
-  tags: nftables
-  ansible.builtin.systemd_service:
-    name: nftables
-    state: reloaded

ansible/roles/prometheus_node_exporter/tasks/main.yml

@@ -49,14 +49,13 @@
     - node_exporter
     - config
 
-- name: Allow prometheus monitoring
-  ansible.builtin.blockinfile:
-    path: /etc/ooni/nftables/tcp/9100.nft
-    create: yes
-    block: |
-      add rule inet filter input tcp dport 9100 counter accept comment "Incoming prometheus monitoring"
-  notify:
-    - Reload nftables
+- ansible.builtin.include_role:
+    name: nftables
+  vars:
+    nft_rules_tcp:
+      - name: 9100
+        rules:
+          - add rule inet filter input tcp dport 9100 counter accept comment "Incoming prometheus monitoring"
   tags:
     - monitoring
     - node_exporter

ansible/roles/ssh_users/tasks/main.yml

@@ -66,6 +66,21 @@
     force: yes
   with_items: "{{ deactivated_usernames }}"
 
+- name: configure sshd
+  include_role:
+    name: willshersystems.sshd
+  vars:
+    sshd_skip_defaults: false
+    sshd:
+      AllowUsers: "{{ admin_usernames | union(non_admin_usernames) | sort | join(' ') }}"
+
+- name: Enesure sudoers dir exists
+  ansible.builtin.file:
+    path: /etc/sudoers.d
+    state: directory
+    owner: root
+    group: root
+
 - name: sudoers.d/80-admins
   template:
     src: sudoers
@@ -79,11 +94,3 @@
   ansible.builtin.file:
     path: /etc/sudoers.d/adm
     state: absent
-
-- name: configure sshd
-  include_role:
-    name: willshersystems.sshd
-  vars:
-    sshd_skip_defaults: false
-    sshd:
-      AllowUsers: "{{ admin_usernames | union(non_admin_usernames) | sort | join(' ') }}"