refactor: make backend-hel functional as the test server

ansible/ansible-playbook

@@ -8,5 +8,5 @@ ANSIBLE_SLACK_CMD=`printf "%q " "$0" "$@"`
 ANSIBLE_SLACK_CMD="${ANSIBLE_SLACK_CMD% }" # strip trailing whitespace
 export ANSIBLE_SLACK_CMD
 
-ansible localhost --module-name include_role --args name=notify-slack
+ansible localhost --module-name include_role --args name=notify-slack --vault-password-file password-pipe
 ansible-playbook "$@"

ansible/deploy-clickhouse-proxy.yml

@@ -0,0 +1,13 @@
+---
+- name: Deploy clickhouse proxy
+  hosts:
+    - clickhouseproxy.dev.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: nginx
+      tags: nginx
+    - role: clickhouse_proxy
+      vars: 
+        clickhouse_url: "clickhouse3.prod.ooni.io"
+        clickhouse_port: 9000

ansible/deploy-clickhouse.yml

@@ -3,7 +3,7 @@
   hosts:
     - notebook.ooni.org
     - data1.htz-fsn.prod.ooni.nu
-    #- data2.htz-fsn.prod.ooni.nu
+    - data2.htz-fsn.prod.ooni.nu
     - data3.htz-fsn.prod.ooni.nu
   become: true
   tags:

ansible/deploy-ooni-backend.yml

@@ -1,7 +1,11 @@
 ---
-- hosts: backend-hel.ooni.org
+- name: Deploy ooni backend services
+  hosts: backend-hel.ooni.org
+  become: true
   roles:
     - role: bootstrap
+      vars:
+        admin_group_name: adm
     - role: base-backend
     - role: nftables
     - role: nginx
@@ -19,3 +23,13 @@
     - role: ooni-backend
       vars: 
         ssl_domain: backend-hel.ooni.org
+        collector_id: 2
+        clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse3.prod.ooni.io/oonitest"
+        bucket_name: ooni-data-eu-fra-test
+        public_bucket_name: ooni-data-eu-fra-test
+        collectors: 
+          - "backend-hel.ooni.org"
+        fastpath_version: 0.86~pr831-395
+        analysis_version: 1.12~pr836-412
+        deb_bucket_name: ooni-internal-deb
+        deb_server_name: deb-cli.ooni.org

ansible/deploy-tier0.yml

@@ -5,6 +5,9 @@
 - name: Include ooni-backend playbook
   ansible.builtin.import_playbook: deploy-ooni-backend.yml
 
+- name: Include ooni-clickhouse-proxy playbook
+  ansible.builtin.import_playbook: deploy-clickhouse-proxy.yml
+
 - name: Include clickhouse playbook
   ansible.builtin.import_playbook: deploy-clickhouse.yml
 

ansible/group_vars/clickhouse/vars.yml

@@ -7,6 +7,8 @@ nftables_clickhouse_allow:
     ip: 168.119.7.188
   - fqdn: notebook.ooni.org
     ip: 138.201.19.39
+  - fqdn: backend-hel.ooni.org
+    ip: 65.108.192.151
   - fqdn: clickhouseproxy.dev.ooni.io
     ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}"
 

ansible/host_vars/backend-fsn.ooni.org/vars.yml

@@ -0,0 +1,10 @@
+s3_ooni_open_data_access_key: "{{ vault_s3_ooni_open_data_access_key }}"
+amspg_ooni_org_onion_key: "{{ vault_amspg_ooni_org_onion_key }}"
+jwt_encryption_key: "{{ vault_jwt_encryption_key }}"
+account_id_hashing_key: "{{ vault_account_id_hashing_key }}"
+github_token: "{{ vault_github_token }}"
+mail_smtp_password: "{{ vault_mail_smtp_password }}"
+base_url: "https://backend-fsn.ooni.org"
+tor_targets: "{{ vault_tor_targets }}"
+psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}"
+digital_ocean_token: "{{ vault_digital_ocean_token }}"

ansible/host_vars/backend-fsn.ooni.org/vault

@@ -0,0 +1,78 @@
+$ANSIBLE_VAULT;1.1;AES256
+39653265353731373139626362626462376535306531333636396461376435323632323665623532
+6566663563616635636663653438616530326333366532640a376632393532613338386635393662
+63626637303861653236353137303637323930396132646632626163303739353931303464633137
+3661306234363035310a303739306561653531646437633036653639323337303561343339626530
+34646636666538376264366361383137653665373730373136353461626163346366356436336461
+36653266366363633538623330363062393562363864663765636665636135626563373238663239
+63613730386537663166386663343935303064323064303438383635363238623866663031316339
+36346236336265336662396130373333643235313237393539633066396538633661666630616637
+31333935633165333536356665363837393736306361643864323662666461313764363364373737
+37326137633631653332656461636534313035336132613239613037323364626234613136643232
+34326164326339303439323865303836313666613136383431343235653739323361366163643566
+30643463663162363830333264376138323339356430656265626565383236383430636632616365
+32343665633965636566333162386135323638383964396633326139623039313434333836303132
+66323461666335643363646265643862346332653536623433326364353766323337313836356534
+35336563373535653436333735313836636336353433623132653136663530646230363931323762
+33313861663965626465323665656431386134326465393033346239613862643236336230336632
+66336662643435333563643437363762333030326333383738366439653233393132306136326561
+65383239663434626639376632306166333934366235656438653261646163383164323533636430
+39636133366131323635623039396236343634613964636237633036343034363038336237323736
+37616434633231323866643464626437343461663537646563303437653830653934656162343939
+35316233316661356161303937633238633733633030396464636230353530353031313437366231
+32343730663065656439356639316261623132656365663730333634363164343836653032393536
+65313935313763346362623365646135333363303230363262313334306439333139306264313533
+32326564353563656235383034633131316238623735383438633864663661633632323863613961
+37376432646337633432623837383131363131363238336239353833336135343031616335633961
+34303737393864346132636336366331643634363532373763316631383439383437316536623137
+36643763653734666665363663643066333539653635643565306139383336383338353634613563
+31316437333733636534316338353837626638636530313865386630623665663630643630356430
+66313637383536386663666532306530343931616139316232633437376532626431383562333861
+31636161323431323864636265323337626236383632373436393236363334313031313733613532
+64346137636535373061313336333661363162663734643862663338353839326436653631663265
+36646565363934356636316433393166396638643563663637363232323461623833633565663935
+32326561323535636630396362653839343835316265623935393135633038323737663030333733
+31383264353265343132613764356234666235316537323539393366316334346261663639663564
+37343737306330613962346261373963643162353739366361663965666338373063323462646435
+32646439323738396134373030343539623264616164656136393136613434666332356533333031
+61383866616361313666323238623464363162353634643630323663386263623764623234613062
+30643565396431653736666633313461353535616335663262343038383634353362363962663131
+37313331663266353133396166323232643361343236363937306437636337653061356130313461
+30396465653461653766326236346236623361393636353731666532333332633430383732656131
+64373931306362333237383635343437323139386539356136363532343439373731333939393063
+33653238626465383134613066306337313930653036623337363863333330303163363338373138
+30623635643638383165393134613834346435333665343531376464646663623733623834333161
+35643732366636656262323931383236333134623931343038353531643234336665663261306231
+66313735306332613962613936366535646432363633656430666262346362376566333163353333
+63646630346536653230313361353131663037393965656466333561393935386132316166313064
+63613334626339396266336364656161623862313234313135626335643434346638346632353265
+32383639393230373464323038623462376362663538373933393863303132653536633665646662
+64353333303037666339356262646536613838626633343635663632653733363036636265396637
+64396461636435336235303261353636353839356265373035643934383034616161326130363363
+37306431353639363537393061326163326561313839356465393665626666653634656265306533
+62343838623866663766386131313839363636656635613639323437646335396534653535356364
+64303239366361623830353231656332653838313866366339333939373862653866633639333935
+37613663636130313538346161393830613139333937383239623437616465396536633037303534
+31303562386666383464393638666631613265653331366135316434653431356361646163323061
+64613963353733363138393661653966303633663638663037656566343333316435366337353362
+31346463343239636132653163653631353063616362346236326331366562653230633132643363
+37376236656261663436323230393634386266343761343165343965613532656238353663326237
+36336435333162616465346362393264616635316362366364653537663031656231643566373733
+61303964626166333535306530633136363461623034636661313161343961386432626231363335
+36316233653734366462323639633034353165363163626363323330343136303235326531353030
+62653131373533656466613165386139663836373062313632363137393130353965303633656636
+31376264313431656338356330653133663339653337326331633733653762663266383634666261
+34353832616265643436323237396638393966643135343539333332653137363863653232636435
+36643266623139313238336333336637323139633565393633346663666666613338353561363962
+36353663313934643034386661383730343836653262333832353263636161633232393431383465
+65323233343362363931303339386639643930313433656165666436393766666435653230393733
+38643466333365306566373631333261363461336462613134366138383764633031623633343461
+39616561306366343339383766623265643364623532653866316131363234653632653765343936
+65363131383630363163396433613537646634613135353639386462333962623333386461633863
+33663963323632303961323138666634356134373035663335663263303333343036356335396532
+35303164623234353566313237313363373630363536303766353561653935336466336263333534
+36366230303230373137663266343166626562313835386164613362386436313737633536353165
+39376164653366373161326434623639626337366434626564663433656231316334653336313630
+65326265336634623965666333303132643564663838636662333664633864336163313537313262
+65666366636430663235333766356437653436303936363366303734303661633431343366306530
+33646635343336386337323561316137666530653337663365323939613264363935

ansible/host_vars/backend-hel.ooni.org/vars.yml

@@ -0,0 +1,7 @@
+s3_ooni_open_data_access_key: "{{ vault_s3_ooni_open_data_access_key }}"
+jwt_encryption_key: "{{ vault_jwt_encryption_key }}"
+account_id_hashing_key: "{{ vault_account_id_hashing_key }}"
+github_token: "{{ vault_github_token }}"
+mail_smtp_password: "{{ vault_mail_smtp_password }}"
+base_url: "https://backend-hel.ooni.org"
+psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}"

ansible/host_vars/backend-hel.ooni.org/vault

@@ -0,0 +1,30 @@
+$ANSIBLE_VAULT;1.1;AES256
+32343262656431363066623434666166643434303861636663643039383630316330656132643735
+3838653761363233623336373439646362643232613564620a393739633764393762306365666265
+39656633323133353231353732643433353662656666626232663234313732363739356466303432
+6564353734376231330a383236346534653166396237353430663631316265643764623233323436
+33373331626632613361633166313163393235623634633965343531653831343138343862366461
+34656233306532353038316636636538393436356264333338353636376231343531633566356665
+33616230393565353030336236303964666463623163633764626139363236663433393366653362
+35393539623434643232373636326232393738653330363364393031626238653735663134623431
+31393463633635623636663638333962343530303635653862616231663465303864326531643961
+35363032356265343734636432386265393530613438386636353664636338633266636130616264
+62663831353735663732343637306435323163303865623964633662306163643838336437333962
+33343534393135306165373534343631343864353561333133653765313032326531363237306564
+65323939626436373839306536616630393633613835633632326530366333663562383562353663
+30396135326239343263323666376632303835366139393137643263326338303365613039313630
+61346137646231353463396566376634666562323539623833323334306663623332626538616132
+32363765306234363938656364643364333061616364653932363237376337366632663533616634
+37633262653130313334643239616535613365353261353336396464316532366134376461376232
+35336435653233643634646166333137656632386361623533313133336638393361373762356637
+30373264626366306333643932313834613263616161656630646666626233393330346236333032
+35653531313534373565663938316637336333366633323863396431623966313038353162333534
+65343266346538656535353132376636336337626630653933313361623736623934376361623865
+64373365396436653762393438666236336433653334343035336265396239303938646231333738
+38653334613335363132343739343565376665636136356534636331326535326264343062346433
+64623038626536346435323031383934303138396363666633393465636232356463383231663937
+30383263396565313136316131386535316337333166303133613430626332363537326539353138
+33303565323031396261336533616662346663636132653739663965303930396466353339383836
+35376137333962333331613736623337656664663565326435643164356166336566653665623036
+30363739363064653231343165363966333238366637626166633165383535326130343135663931
+6636

ansible/host_vars/data3.htz-fsn.prod.ooni.nu

@@ -1,2 +1,2 @@
-non_admin_usernames: [ ]
+non_admin_usernames: []
 clickhouse_base_path: /data/clickhouse

ansible/inventory

@@ -27,3 +27,9 @@ db-1.proteus.ooni.io
 ams-slack-1.ooni.org
 #mia-echoth.ooni.nu
 #mia-httpth.ooni.nu
+
+[aws-proxy]
+clickhouseproxy.dev.ooni.io
+
+[backend]
+backend-hel.ooni.org

ansible/password-pipe

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec gpg --quiet --decrypt --batch <~/.ssh/ooni-sysadmin.vaultpw.gpg

ansible/roles/base-backend/tasks/main.yml

@@ -94,8 +94,8 @@
     create: yes
     block: |
       add rule inet filter input ip saddr {{ lookup('dig', 'prometheus.ooni.org/A') }} tcp dport 19999 counter accept comment "netdata.service"
-    notify:
-      - reload nftables
+  notify:
+    - reload nftables
 
 - name: configure netdata.service
   tags: netdata

ansible/roles/bootstrap/tasks/main.yml

@@ -68,6 +68,7 @@
 
 - name: install systemd-resolved
   tags: resolved
+  ignore_errors: yes # some ubuntu hosts do not have the systemd-resolved package
   ansible.builtin.apt:
     install_recommends: no
     cache_valid_time: 86400

ansible/roles/clickhouse_proxy/handlers/main.yml

@@ -0,0 +1,15 @@
+- name: test nginx config
+  command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen:
+    - restart nginx
+    - reload nginx
+
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded

ansible/roles/clickhouse_proxy/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Create the modules-enabled directory if not exists
+  tags: webserv
+  ansible.builtin.file:
+    path: /etc/nginx/modules-enabled
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Add stream nginx config
+  tags: webserv
+  template:
+    src: templates/99-stream.conf
+    dest: /etc/nginx/modules-enabled/99-stream.conf
+    mode: 0755
+    owner: root
+  notify:
+    - reload nginx
+    - restart nginx

ansible/roles/clickhouse_proxy/templates/99-stream.conf

@@ -0,0 +1,13 @@
+stream {
+    upstream clickhouse_backend {
+        server {{ clickhouse_url }}:{{ clickhouse_port }};
+    }
+
+    server {
+        listen 9000;
+
+        proxy_pass clickhouse_backend; 
+    }
+
+    error_log /var/log/nginx/error.log;
+}

ansible/roles/dehydrated/handlers/main.yml

@@ -0,0 +1,15 @@
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
+
+- name: reload nftables
+  service: 
+    name: nftables
+    state: reloaded
+
+- name: restart dehydrated
+  service:
+    name: dehydrated
+    state: restarted
+    enabled: yes

ansible/roles/dehydrated/tasks/main.yml

@@ -42,10 +42,8 @@
     dest: /var/lib/dehydrated/acme-challenges/ooni-acme-canary
     mode: 0644
     owner: root
-
-- name: reload nginx
-  tags: dehydrated
-  shell: systemctl reload nginx.service
+  notify: 
+    - reload nginx
 
 - name: allow incoming TCP connections to Nginx on port 80
   tags: dehydrated
@@ -54,10 +52,8 @@
     create: yes
     block: |
       add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP"
-
-- name: reload nftables service
-  tags: dehydrated
-  shell: systemctl reload nftables.service
+  notify:
+    - reload nftables
 
 - name: Configure domains {{ ssl_domains }}
   # https://github.com/dehydrated-io/dehydrated/blob/master/docs/domains_txt.md
@@ -93,16 +89,8 @@
     name: dehydrated.timer
     state: started
     enabled: yes
-
-- name: Run dehydrated service immediately
-  # creates:
-  # /var/lib/dehydrated/certs/<name>/chain.pem cert.pem privkey.pem fullchain.pem
-  tags: dehydrated
-  systemd:
-    name: dehydrated.service
-    state: started
-    enabled: yes
-
-- name: reload nginx
-  tags: dehydrated
-  shell: systemctl reload nginx.service
+  notify:
+    # creates:
+    # /var/lib/dehydrated/certs/<name>/chain.pem cert.pem privkey.pem fullchain.pem
+    - restart dehydrated
+    - reload nginx

ansible/roles/ooni-backend/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+ssl_domain: backend-hel.ooni.org
+collector_id: 2
+clickhouse_url: "" # fetch from aws secrets
+bucket_name: ooni-data-eu-fra-test
+public_bucket_name: ooni-data-eu-fra-test
+collectors: 
+  - "backend-hel.ooni.org"
+fastpath_version: 0.86~pr831-395
+analysis_version: 1.12~pr836-413
+deb_bucket_name: ooni-internal-deb
+deb_server_name: deb-ci.ooni.org
+apt_cache_valid_time: 28800 # 8h