refactor: make backend-hel functional as the test server

ansible/deploy-clickhouse-proxy.yml

@@ -0,0 +1,13 @@
+---
+- name: Deploy clickhouse proxy
+  hosts:
+    - clickhouseproxy.dev.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: nginx
+      tags: nginx
+    - role: clickhouse_proxy
+      vars: 
+        clickhouse_url: "clickhouse3.prod.ooni.io"
+        clickhouse_port: 9000

ansible/deploy-clickhouse.yml

@@ -3,7 +3,7 @@
   hosts:
     - notebook.ooni.org
     - data1.htz-fsn.prod.ooni.nu
-    #- data2.htz-fsn.prod.ooni.nu
+    - data2.htz-fsn.prod.ooni.nu
     - data3.htz-fsn.prod.ooni.nu
   become: true
   tags:

ansible/deploy-ooni-backend.yml

@@ -19,3 +19,42 @@
     - role: ooni-backend
       vars: 
         ssl_domain: backend-hel.ooni.org
+        collector_id: 2
+        clickhouse_url: "" # fetch from aws secrets
+        bucket_name: ooni-data-eu-fra-test
+        collectors: 
+          - "backend-hel.ooni.org"
+        fastpath_version: 0.86~pr831-395
+        analysis_version: 1.12~pr836-413
+        deb_bucket_name: ooni-internal-deb
+        deb_server_name: deb-cli.ooni.org
+
+# - hosts: backend-fsn.ooni.org
+  # roles:
+    # - role: bootstrap
+    # - role: base-backend
+    # - role: nftables
+    # - role: nginx
+      # tags: nginx
+      # vars:
+        # nginx_user: "www-data"
+    # - role: dehydrated
+      # tags: dehydrated
+      # expand: yes
+      # vars: 
+        # ssl_domains:
+          # # with dehydrated the first entry is the cert FQDN
+          # # and the other ones are alternative names
+          # - "backend-hel.ooni.org"
+    # - role: ooni-backend
+      # vars: 
+        # ssl_domain: backend-fsn.ooni.org
+        # collector_id: 1
+        # clickhouse_url: "" # fetch from aws secrets
+        # bucket_name: ooni-data-eu-fra
+        # collectors: 
+          # - "backend-fsn.ooni.org"
+        # fastpath_version: 0.86~pr831-395
+        # analysis_version: 1.12~pr836-413
+        # deb_bucket_name: ooni.deb
+        # deb_server_name: deb.ooni.org

ansible/deploy-tier0.yml

@@ -5,6 +5,9 @@
 - name: Include ooni-backend playbook
   ansible.builtin.import_playbook: deploy-ooni-backend.yml
 
+- name: Include ooni-clickhouse-proxy playbook
+  ansible.builtin.import_playbook: deploy-clickhouse-proxy.yml
+
 - name: Include clickhouse playbook
   ansible.builtin.import_playbook: deploy-clickhouse.yml
 

ansible/group_vars/clickhouse/vars.yml

@@ -7,6 +7,8 @@ nftables_clickhouse_allow:
     ip: 168.119.7.188
   - fqdn: notebook.ooni.org
     ip: 138.201.19.39
+  - fqdn: backend-hel.ooni.org
+    ip: 65.108.192.151
   - fqdn: clickhouseproxy.dev.ooni.io
     ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}"
 

ansible/host_vars/backend-fsn.ooni.org

@@ -0,0 +1,10 @@
+s3_ooni_open_data_access_key: "{{ vault_s3_ooni_open_data_access_key }}"
+amspg_ooni_org_onion_key: "{{ vault_amspg_ooni_org_onion_key }}"
+jwt_encryption_key: "{{ vault_jwt_encryption_key }}"
+account_id_hashing_key: "{{ vault_account_id_hashing_key }}"
+github_token: "{{ vault_github_token }}"
+mail_smtp_password: "{{ vault_mail_smtp_password }}"
+base_url: "https://backend-fsn.ooni.org"
+tor_targets: "{{ vault_tor_targets }}"
+psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}"
+digital_ocean_token: "{{ vault_digital_ocean_token }}"

ansible/host_vars/backend-hel.ooni.org

@@ -0,0 +1,8 @@
+s3_ooni_open_data_access_key: "{{ vault_s3_ooni_open_data_access_key }}"
+jwt_encryption_key: "{{ vault_jwt_encryption_key }}"
+account_id_hashing_key: "{{ vault_account_id_hashing_key }}"
+github_token: "{{ vault_github_token }}"
+mail_smtp_password: "{{ vault_mail_smtp_password }}"
+base_url: "https://backend-hel.ooni.org"
+tor_targets: "{{ vault_tor_targets }}"
+psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}"

ansible/host_vars/data3.htz-fsn.prod.ooni.nu

@@ -1,2 +1,3 @@
-non_admin_usernames: [ ]
+non_admin_usernames: []
 clickhouse_base_path: /data/clickhouse
+

ansible/inventory

@@ -22,3 +22,6 @@ data3.htz-fsn.prod.ooni.nu
 
 [ghs-ams]
 openvpn-server1.ooni.io
+
+[aws-proxy]
+clickhouseproxy.dev.ooni.io

ansible/roles/bootstrap/tasks/main.yml

@@ -68,6 +68,7 @@
 
 - name: install systemd-resolved
   tags: resolved
+  ignore_errors: yes # some ubuntu hosts do not have the systemd-resolved package
   ansible.builtin.apt:
     install_recommends: no
     cache_valid_time: 86400

ansible/roles/clickhouse_proxy/handlers/main.yml

@@ -0,0 +1,15 @@
+- name: test nginx config
+  command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen:
+    - restart nginx
+    - reload nginx
+
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded

ansible/roles/clickhouse_proxy/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Create the modules-enabled directory if not exists
+  tags: webserv
+  ansible.builtin.file:
+    path: /etc/nginx/modules-enabled
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Add stream nginx config
+  tags: webserv
+  template:
+    src: templates/99-stream.conf
+    dest: /etc/nginx/modules-enabled/99-stream.conf
+    mode: 0755
+    owner: root
+  notify:
+    - reload nginx
+    - restart nginx

ansible/roles/clickhouse_proxy/templates/99-stream.conf

@@ -0,0 +1,13 @@
+stream {
+    upstream clickhouse_backend {
+        server {{ clickhouse_url }}:{{ clickhouse_port }};
+    }
+
+    server {
+        listen 9000;
+
+        proxy_pass clickhouse_backend; 
+    }
+
+    error_log /var/log/nginx/error.log;
+}

ansible/roles/ooni-backend/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+ssl_domain: backend-hel.ooni.org
+collector_id: 2
+clickhouse_url: "" # fetch from aws secrets
+bucket_name: ooni-data-eu-fra-test
+collectors: 
+  - "backend-hel.ooni.org"
+fastpath_version: 0.86~pr831-395
+analysis_version: 1.12~pr836-413
+deb_bucket_name: ooni-internal-deb
+deb_server_name: deb-ci.ooni.org

ansible/roles/ooni-backend/handlers/main.yml

@@ -1,6 +1,22 @@
 ---
 - name: reload nftables
+  tags: api, webserv
   service: name=nftables state=reloaded
 
-- name: restart clickhouse
-  service: name=clickhouse-server state=restarted
+- name: test nginx config
+  command: nginx -t
+  listen:
+    - restart nginx
+    - reload nginx
+
+- name: restart nginx
+  tags: webserv
+  service:
+    name: nginx
+    state: restarted
+
+- name: reload nginx
+  tags: api, deb_ooni, webserv
+  service:
+    name: nginx
+    state: reloaded