From 3ed88c00fd9e64ec71ec853dc4c6669863f9945f Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 4 Dec 2024 17:36:08 -0500 Subject: [PATCH] refactor: add base files for backend-hel deployment --- ansible/playbook-backend.yml | 16 + ansible/roles/base-bookworm/README.adoc | 1 + ansible/roles/base-bookworm/meta/main.yml | 6 + ansible/roles/base-bookworm/tasks/main.yml | 221 +++++ .../base-bookworm/templates/internal-deb.gpg | 14 + .../base-bookworm/templates/journald.conf | 8 + .../base-bookworm/templates/netdata.conf | 32 + .../templates/ooni_internal.sources | 7 + .../base-bookworm/templates/resolved.conf | 9 + .../base-bookworm/templates/sources.list | 6 + ansible/roles/dehydrated/README.adoc | 10 + ansible/roles/dehydrated/meta/main.yml | 5 + ansible/roles/dehydrated/tasks/main.yml | 108 +++ .../dehydrated/templates/dehydrated.service | 13 + .../dehydrated/templates/dehydrated.timer | 9 + .../roles/dehydrated/templates/domains.txt.j2 | 1 + ansible/roles/dehydrated/templates/hook.sh | 20 + .../dehydrated/templates/letsencrypt-http | 13 + ansible/roles/nftables-sysadmin/README.adoc | 25 + .../roles/nftables-sysadmin/tasks/main.yml | 49 ++ .../nftables-sysadmin/templates/nftables.conf | 41 + .../nginx-buster/files/ffdhe2048_dhparam.pem | 8 + .../nginx-buster/files/ssl_intermediate.conf | 3 + .../roles/nginx-buster/files/ssl_modern.conf | 4 + ansible/roles/nginx-buster/handlers/main.yml | 6 + ansible/roles/nginx-buster/tasks/main.yml | 33 + .../roles/nginx-buster/templates/nginx.conf | 68 ++ ansible/roles/ooni-backend/handlers/main.yml | 6 + ansible/roles/ooni-backend/meta/main.yml | 3 + ansible/roles/ooni-backend/tasks/main.yml | 822 ++++++++++++++++++ ansible/roles/ooni-backend/templates/444.nft | 2 + .../ooni-backend/templates/analysis.conf | 9 + .../ooni-backend/templates/api-uploader.conf | 9 + ansible/roles/ooni-backend/templates/api.conf | 60 ++ .../ooni-backend/templates/api.gunicorn.py | 12 + .../templates/clickhouse_config.xml | 41 + .../templates/clickhouse_readonly.xml | 9 + .../templates/clickhouse_users.xml | 31 + .../ooni-backend/templates/db-backup.conf | 17 + .../templates/deb_ooni_org.nginx.conf | 64 ++ .../ooni-backend/templates/dehydrated.config | 7 + .../templates/dehydrated_haproxy_hook.sh | 23 + .../ooni-backend/templates/fastpath.conf | 15 + .../roles/ooni-backend/templates/haproxy.cfg | 122 +++ .../templates/nginx-api-ams-pg.conf | 297 +++++++ .../ooni-backend/templates/nginx-api-fsn.conf | 260 ++++++ .../templates/nginx-api-test.conf | 157 ++++ .../templates/rotation_nginx_conf | 70 ++ .../ooni-backend/templates/rotation_setup.sh | 114 +++ .../ooni-backend/templates/tor_targets.json | 304 +++++++ 50 files changed, 3190 insertions(+) create mode 100644 ansible/playbook-backend.yml create mode 100644 ansible/roles/base-bookworm/README.adoc create mode 100644 ansible/roles/base-bookworm/meta/main.yml create mode 100644 ansible/roles/base-bookworm/tasks/main.yml create mode 100644 ansible/roles/base-bookworm/templates/internal-deb.gpg create mode 100644 ansible/roles/base-bookworm/templates/journald.conf create mode 100644 ansible/roles/base-bookworm/templates/netdata.conf create mode 100644 ansible/roles/base-bookworm/templates/ooni_internal.sources create mode 100644 ansible/roles/base-bookworm/templates/resolved.conf create mode 100644 ansible/roles/base-bookworm/templates/sources.list create mode 100644 ansible/roles/dehydrated/README.adoc create mode 100644 ansible/roles/dehydrated/meta/main.yml create mode 100644 ansible/roles/dehydrated/tasks/main.yml create mode 100644 ansible/roles/dehydrated/templates/dehydrated.service create mode 100644 ansible/roles/dehydrated/templates/dehydrated.timer create mode 100644 ansible/roles/dehydrated/templates/domains.txt.j2 create mode 100644 ansible/roles/dehydrated/templates/hook.sh create mode 100644 ansible/roles/dehydrated/templates/letsencrypt-http create mode 100644 ansible/roles/nftables-sysadmin/README.adoc create mode 100644 ansible/roles/nftables-sysadmin/tasks/main.yml create mode 100755 ansible/roles/nftables-sysadmin/templates/nftables.conf create mode 100644 ansible/roles/nginx-buster/files/ffdhe2048_dhparam.pem create mode 100644 ansible/roles/nginx-buster/files/ssl_intermediate.conf create mode 100644 ansible/roles/nginx-buster/files/ssl_modern.conf create mode 100644 ansible/roles/nginx-buster/handlers/main.yml create mode 100644 ansible/roles/nginx-buster/tasks/main.yml create mode 100644 ansible/roles/nginx-buster/templates/nginx.conf create mode 100644 ansible/roles/ooni-backend/handlers/main.yml create mode 100644 ansible/roles/ooni-backend/meta/main.yml create mode 100644 ansible/roles/ooni-backend/tasks/main.yml create mode 100644 ansible/roles/ooni-backend/templates/444.nft create mode 100644 ansible/roles/ooni-backend/templates/analysis.conf create mode 100644 ansible/roles/ooni-backend/templates/api-uploader.conf create mode 100644 ansible/roles/ooni-backend/templates/api.conf create mode 100644 ansible/roles/ooni-backend/templates/api.gunicorn.py create mode 100644 ansible/roles/ooni-backend/templates/clickhouse_config.xml create mode 100644 ansible/roles/ooni-backend/templates/clickhouse_readonly.xml create mode 100644 ansible/roles/ooni-backend/templates/clickhouse_users.xml create mode 100644 ansible/roles/ooni-backend/templates/db-backup.conf create mode 100644 ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf create mode 100644 ansible/roles/ooni-backend/templates/dehydrated.config create mode 100644 ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh create mode 100644 ansible/roles/ooni-backend/templates/fastpath.conf create mode 100644 ansible/roles/ooni-backend/templates/haproxy.cfg create mode 100644 ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf create mode 100644 ansible/roles/ooni-backend/templates/nginx-api-fsn.conf create mode 100644 ansible/roles/ooni-backend/templates/nginx-api-test.conf create mode 100644 ansible/roles/ooni-backend/templates/rotation_nginx_conf create mode 100644 ansible/roles/ooni-backend/templates/rotation_setup.sh create mode 100644 ansible/roles/ooni-backend/templates/tor_targets.json diff --git a/ansible/playbook-backend.yml b/ansible/playbook-backend.yml new file mode 100644 index 00000000..c6b36579 --- /dev/null +++ b/ansible/playbook-backend.yml @@ -0,0 +1,16 @@ +--- +- hosts: backend-hel.ooni.org + roles: + - role: base-bookworm + - role: nftables + - role: nginx-buster + tags: nginx + - role: dehydrated + tags: dehydrated + expand: yes + ssl_domains: + # with dehydrated the first entry is the cert FQDN + # and the other ones are alternative names + - "backend-hel.ooni.org" + - role: ooni-backend + ssl_domain: backend-hel.ooni.org diff --git a/ansible/roles/base-bookworm/README.adoc b/ansible/roles/base-bookworm/README.adoc new file mode 100644 index 00000000..be579005 --- /dev/null +++ b/ansible/roles/base-bookworm/README.adoc @@ -0,0 +1 @@ +Configure base host based on Bookworm diff --git a/ansible/roles/base-bookworm/meta/main.yml b/ansible/roles/base-bookworm/meta/main.yml new file mode 100644 index 00000000..5de9bc56 --- /dev/null +++ b/ansible/roles/base-bookworm/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: adm + become: false + remote_user: root + gather_facts: false diff --git a/ansible/roles/base-bookworm/tasks/main.yml b/ansible/roles/base-bookworm/tasks/main.yml new file mode 100644 index 00000000..c9c3b1da --- /dev/null +++ b/ansible/roles/base-bookworm/tasks/main.yml @@ -0,0 +1,221 @@ +--- +- name: motd + shell: echo "" > /etc/motd + +- name: Set hostname + ansible.builtin.hostname: + name: "{{ inventory_hostname }}" + +- name: Remove apt repo + tags: apt + file: + path: /etc/apt/sources.list.d/ftp_nl_debian_org_debian.list + state: absent + +- name: Remove apt repo + tags: apt + file: + path: /etc/apt/sources.list.d/security_debian_org.list + state: absent + +- name: Create internal-deb repo GPG pubkey + tags: apt + template: + src: templates/internal-deb.gpg + dest: /etc/ooni/internal-deb.gpg + mode: 0644 + owner: root + +- name: Set apt repos + tags: apt + template: + src: templates/sources.list + dest: /etc/apt/sources.list + mode: 0644 + owner: root + +- name: Install gpg + tags: base-packages + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - gpg + - gpg-agent + +- name: Update apt cache + tags: apt + apt: + update_cache: yes + +- name: Installs base packages + tags: base-packages + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - bash-completion + - byobu + - chrony + - etckeeper + - fail2ban + - git + - iotop + - jupyter-notebook + - manpages + - ncdu + - netdata-core + - netdata-plugins-bash + - netdata-plugins-python + - netdata-web + - nftables + - nullmailer + - prometheus-node-exporter + - pv + # needed by ansible + - python3-apt + - rsync + - ssl-cert + - strace + - tcpdump + - tmux + - vim + +- name: Configure journald + tags: journald + template: + src: templates/journald.conf + dest: /etc/systemd/journald.conf + mode: 0644 + owner: root + +- name: enable and restart journald + tags: journald + systemd: + name: systemd-journald.service + state: restarted + enabled: yes + daemon_reload: yes + +- name: Autoremove + tags: autoremove + apt: + autoremove: yes + +- name: Clean cache + tags: apt + apt: + autoclean: yes + +- name: allow netdata.service + tags: netdata + blockinfile: + path: /etc/ooni/nftables/tcp/19999.nft + create: yes + block: | + add rule inet filter input ip saddr {{ lookup('dig', 'prometheus.ooni.org/A') }} tcp dport 19999 counter accept comment "netdata.service" + +#- name: reload nftables service +# systemd: +# name: nftables.service +# state: reloaded +# enabled: yes +# daemon_reload: yes + +- name: reload nftables service + service: name=nftables state=restarted + +- name: configure netdata.service + tags: netdata + template: + src: netdata.conf + dest: /etc/netdata/netdata.conf + +- name: disable netdata emails + tags: netdata + blockinfile: + path: /etc/netdata/conf.d/health_alarm_notify.conf + create: yes + block: | + # Managed by ansible, see roles/base-bookworm/tasks/main.yml + SEND_EMAIL="NO" + +- name: Set timezone + tags: timezone + timezone: + name: Etc/UTC + +- name: restart chrony service + tags: timezone + systemd: + name: chrony.service + state: restarted + +- name: configure netdata chrony + tags: netdata, timezone + blockinfile: + path: /etc/netdata/python.d/chrony.conf + create: yes + block: | + # Managed by ansible, see roles/base-bookworm/tasks/main.yml + update_every: 5 + local: + command: 'chronyc -n tracking' + +- name: configure netdata chrony + tags: netdata, timezone + lineinfile: + path: /usr/lib/netdata/conf.d/python.d.conf + regexp: '^chrony:' + line: 'chrony: yes' + +#- name: configure netdata nginx +# blockinfile: +# path: /etc/netdata/python.d/nginx.conf +# create: yes +# block: | +# # Managed by ansible, see roles/base-bookworm/tasks/main.yml +# update_every: 5 +# nginx_log: +# name : 'nginx_log' +# path : '/var/log/nginx/access.log' + +#- name: configure netdata haproxy +# blockinfile: +# path: /etc/netdata/python.d/haproxy.conf +# block: | +# # Managed by ansible, see roles/base-bookworm/tasks/main.yml +# update_every: 5 +# via_url: +# url: 'http://127.0.0.1:7000/haproxy_stats;csv;norefresh' + +- name: restart netdata service + tags: netdata, timezone + systemd: + name: netdata.service + state: restarted + + +- name: install systemd-resolved + tags: resolved + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - systemd-resolved + +- name: configure systemd-resolved + tags: resolved + template: + src: resolved.conf + dest: /etc/systemd/resolved.conf + +- name: restart systemd-resolved + tags: resolved + systemd: + name: systemd-resolved.service + state: restarted + +- name: test systemd-resolved + tags: resolved + shell: resolvectl query go.dnscheck.tools --cache=no diff --git a/ansible/roles/base-bookworm/templates/internal-deb.gpg b/ansible/roles/base-bookworm/templates/internal-deb.gpg new file mode 100644 index 00000000..28126a36 --- /dev/null +++ b/ansible/roles/base-bookworm/templates/internal-deb.gpg @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ +JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh +dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb +AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb +gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo +Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX +HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh +tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy +6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s +oN7jfdbznrWVigE= +=PtYb +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/base-bookworm/templates/journald.conf b/ansible/roles/base-bookworm/templates/journald.conf new file mode 100644 index 00000000..d7ae85e1 --- /dev/null +++ b/ansible/roles/base-bookworm/templates/journald.conf @@ -0,0 +1,8 @@ +[Journal] +Storage=persistent +Compress=yes +#RateLimitIntervalSec=30s +#RateLimitBurst=10000 +SystemMaxFileSize=200M +RuntimeMaxFileSize=1G +ForwardToSyslog=no diff --git a/ansible/roles/base-bookworm/templates/netdata.conf b/ansible/roles/base-bookworm/templates/netdata.conf new file mode 100644 index 00000000..e2bef302 --- /dev/null +++ b/ansible/roles/base-bookworm/templates/netdata.conf @@ -0,0 +1,32 @@ +# Managed by ansible, see roles/base-bookworm/tasks/main.yml +[global] + run as user = netdata + web files owner = root + web files group = root + bind socket to IP = 0.0.0.0 + +[plugins] + python.d = yes + + +[statsd] + enabled = yes + # decimal detail = 1000 + update every (flushInterval) = 1 + # udp messages to process at once = 10 + # create private charts for metrics matching = * + max private charts allowed = 10000 + max private charts hard limit = 10000 + private charts memory mode = ram + private charts history = 300 + # histograms and timers percentile (percentThreshold) = 95.00000 + # add dimension for number of events received = no + # gaps on gauges (deleteGauges) = no + # gaps on counters (deleteCounters) = no + # gaps on meters (deleteMeters) = no + # gaps on sets (deleteSets) = no + # gaps on histograms (deleteHistograms) = no + # gaps on timers (deleteTimers) = no + # listen backlog = 4096 + # default port = 8125 + # bind to = udp:localhost:8125 tcp:localhost:8125 diff --git a/ansible/roles/base-bookworm/templates/ooni_internal.sources b/ansible/roles/base-bookworm/templates/ooni_internal.sources new file mode 100644 index 00000000..f85bc625 --- /dev/null +++ b/ansible/roles/base-bookworm/templates/ooni_internal.sources @@ -0,0 +1,7 @@ +Architectures: amd64 +Suites: unstable +Uris: https://ooni-internal-deb.s3.eu-central-1.amazonaws.com +Types: deb +Components: main +Enabled: yes +Signed-By: /etc/ooni/internal-deb.gpg diff --git a/ansible/roles/base-bookworm/templates/resolved.conf b/ansible/roles/base-bookworm/templates/resolved.conf new file mode 100644 index 00000000..dd937e3c --- /dev/null +++ b/ansible/roles/base-bookworm/templates/resolved.conf @@ -0,0 +1,9 @@ +# Deployed by ansible +# See roles/base-bookworm/templates/resolved.conf + +[Resolve] +## https://meta.wikimedia.org/wiki/Wikimedia_DNS +DNS=185.71.138.138 +DNSOverTLS=opportunistic +DNSSEC=allow-downgrade +Cache=yes diff --git a/ansible/roles/base-bookworm/templates/sources.list b/ansible/roles/base-bookworm/templates/sources.list new file mode 100644 index 00000000..7432ddad --- /dev/null +++ b/ansible/roles/base-bookworm/templates/sources.list @@ -0,0 +1,6 @@ +# Managed by ansible +# roles/base-bookworm/templates/sources.list + +deb http://deb.debian.org/debian bookworm main contrib non-free-firmware +deb http://deb.debian.org/debian-security/ bookworm-security main contrib non-free-firmware +deb http://deb.debian.org/debian bookworm-backports main diff --git a/ansible/roles/dehydrated/README.adoc b/ansible/roles/dehydrated/README.adoc new file mode 100644 index 00000000..477601de --- /dev/null +++ b/ansible/roles/dehydrated/README.adoc @@ -0,0 +1,10 @@ + +Configure dehydrated to generate certificates (locally to each server) + +- listen on port 443 for ACME challenge + +- ansible --diff is supported + +- generate certificate expirations metrics for node exporter + +- changes to /etc are also tracked locally by etckeeper diff --git a/ansible/roles/dehydrated/meta/main.yml b/ansible/roles/dehydrated/meta/main.yml new file mode 100644 index 00000000..e7e996b0 --- /dev/null +++ b/ansible/roles/dehydrated/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - nginx-buster +... + diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml new file mode 100644 index 00000000..0bfaf7c3 --- /dev/null +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -0,0 +1,108 @@ +--- +- name: Installs packages + tags: dehydrated + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - dehydrated + +#- name: create dehydrated hook file +# # This hook is called after getting a new cert to deploy it +# template: +# src: templates/hook.sh +# dest: /etc/dehydrated/hook.sh +# mode: 0755 +# owner: root +# +# +#- name: set dehydrated hook +# blockinfile: +# path: /etc/dehydrated/config +# block: | +# HOOK="/etc/dehydrated/hook.sh" + +- name: Add ACME dedicated sites-enabled file + tags: dehydrated + template: + src: templates/letsencrypt-http + # the server block matches all SSL FQDNs and must be + # parsed first, hence 00- + dest: /etc/nginx/sites-enabled/00-letsencrypt-http + mode: 0644 + owner: root + +- name: Add canary file to ensure /.well-known/acme-challenge is reachable by let's encrypt + tags: dehydrated + copy: + content: | + Generated by ansible using ansible/roles/dehydrated/tasks/main.yml. + + Also, meow!!! + dest: /var/lib/dehydrated/acme-challenges/ooni-acme-canary + mode: 0644 + owner: root + +- name: reload nginx + tags: dehydrated + shell: systemctl reload nginx.service + +- name: allow incoming TCP connections to Nginx on port 80 + tags: dehydrated + blockinfile: + path: /etc/ooni/nftables/tcp/80.nft + create: yes + block: | + add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP" + +- name: reload nftables service + tags: dehydrated + shell: systemctl reload nftables.service + +- name: Configure domains {{ ssl_domains }} + # https://github.com/dehydrated-io/dehydrated/blob/master/docs/domains_txt.md + tags: dehydrated + template: + src: templates/domains.txt.j2 + dest: /etc/dehydrated/domains.txt + +- name: Register account if needed + tags: dehydrated + ansible.builtin.shell: + cmd: "test -d /var/lib/dehydrated/accounts || dehydrated --register --accept-terms" + +- name: Install dehydrated.service + tags: dehydrated + template: + src: templates/dehydrated.service + dest: /etc/systemd/system/dehydrated.service + mode: 0644 + owner: root + +- name: Install dehydrated.timer + tags: dehydrated + template: + src: templates/dehydrated.timer + dest: /etc/systemd/system/dehydrated.timer + mode: 0644 + owner: root + +- name: Ensure timer runs + tags: dehydrated + systemd: + name: dehydrated.timer + state: started + enabled: yes + +- name: Run dehydrated service immediately + # creates: + # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem + tags: dehydrated + systemd: + name: dehydrated.service + state: started + enabled: yes + +- name: reload nginx + tags: dehydrated + shell: systemctl reload nginx.service diff --git a/ansible/roles/dehydrated/templates/dehydrated.service b/ansible/roles/dehydrated/templates/dehydrated.service new file mode 100644 index 00000000..50ffdc46 --- /dev/null +++ b/ansible/roles/dehydrated/templates/dehydrated.service @@ -0,0 +1,13 @@ +[Unit] +Description=Run dehydrated certificate refresh + +[Service] +Type=oneshot +#User=dehydrated +#Group=dehydrated +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/dehydrated +PrivateTmp=yes +ExecStart=/usr/bin/dehydrated --cron +ExecStartPost=+/bin/systemctl reload nginx.service diff --git a/ansible/roles/dehydrated/templates/dehydrated.timer b/ansible/roles/dehydrated/templates/dehydrated.timer new file mode 100644 index 00000000..5e6ea784 --- /dev/null +++ b/ansible/roles/dehydrated/templates/dehydrated.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Run dehydrated certificate refresh + +[Timer] +OnCalendar=Mon 13:00 + +[Install] +WantedBy=timers.target + diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 new file mode 100644 index 00000000..5850d203 --- /dev/null +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -0,0 +1 @@ +{% for d in ssl_domains %}{{ d }} {% endfor %} diff --git a/ansible/roles/dehydrated/templates/hook.sh b/ansible/roles/dehydrated/templates/hook.sh new file mode 100644 index 00000000..26193aeb --- /dev/null +++ b/ansible/roles/dehydrated/templates/hook.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Deployed by ansible +# see ansible/roles/dehydrated/templates/hook.sh +# +deploy_cert() { + local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" + # This hook is called once for each certificate that has been produced. + # Parameters: + # - DOMAIN The primary domain name, i.e. the certificate common name (CN). + # - KEYFILE The path of the file containing the private key. + # - CERTFILE The path of the file containing the signed certificate. + # - FULLCHAINFILE The path of the file containing the full certificate chain. + # - CHAINFILE The path of the file containing the intermediate certificate(s). + # - TIMESTAMP Timestamp when the specified certificate was created. + + logger "Deploying SSL certificate $DOMAIN $KEYFILE $CERTFILE $FULLCHAINFILE $CHAINFILE $TIMESTAMP" + # cp ... + #systemctl reload nginx +} diff --git a/ansible/roles/dehydrated/templates/letsencrypt-http b/ansible/roles/dehydrated/templates/letsencrypt-http new file mode 100644 index 00000000..41fda273 --- /dev/null +++ b/ansible/roles/dehydrated/templates/letsencrypt-http @@ -0,0 +1,13 @@ +# Generated by ansible +# roles/dehydrated/templates/letsencrypt-http + +server { + # Listen on port 80 for *any* domain + listen 80; + server_name _; + + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } +} diff --git a/ansible/roles/nftables-sysadmin/README.adoc b/ansible/roles/nftables-sysadmin/README.adoc new file mode 100644 index 00000000..e3bef58f --- /dev/null +++ b/ansible/roles/nftables-sysadmin/README.adoc @@ -0,0 +1,25 @@ +Install nftables based firewall + +Set up /etc/ooni/nftables/ + +Rules for specific services are *not* configured by this role + +When creating rules to accept TCP traffic from any IPv4/6 address, +files are named with the port number to detect collisions. + +Example (also see roles/nftables/tasks/main.yml): + +/etc/ooni/nftables/tcp/8080.nft + +``` +add rule inet filter input tcp dport 8080 counter accept comment "MyService" +``` + + +Otherwise: + +/etc/ooni/nftables/tcp/5432_postgres_internal.nft + +``` +add rule inet filter input ip saddr { 10.0.0.0/8, 192.168.0.0/16 } tcp dport 5432 counter accept comment "Internal PostgreSQL" +``` diff --git a/ansible/roles/nftables-sysadmin/tasks/main.yml b/ansible/roles/nftables-sysadmin/tasks/main.yml new file mode 100644 index 00000000..886c7ab0 --- /dev/null +++ b/ansible/roles/nftables-sysadmin/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: Install nftables + tags: nftables + apt: + cache_valid_time: 86400 + name: nftables + +- name: create config dir + tags: nftables + file: + path: /etc/ooni/nftables/tcp + state: directory + owner: root + group: root + mode: 0755 + +- name: allow SSH + tags: nftables + blockinfile: + path: /etc/ooni/nftables/tcp/22.nft + create: yes + block: | + add rule inet filter input tcp dport 22 counter accept comment "Incoming SSH" + +- name: Overwrite nftables.conf + tags: nftables + template: + src: templates/nftables.conf + dest: /etc/nftables.conf + mode: 0755 + owner: root + +- name: enable nftables service + tags: nftables + shell: systemctl enable nftables.service + +- name: enable nftables service + tags: nftables + shell: systemctl start nftables.service + +#- name: Enable and start nftables service +# systemd: +# name: nftables.service +# state: reloaded +# enabled: yes + +- name: reload nftables service + tags: nftables + shell: systemctl reload nftables.service diff --git a/ansible/roles/nftables-sysadmin/templates/nftables.conf b/ansible/roles/nftables-sysadmin/templates/nftables.conf new file mode 100755 index 00000000..5f7b50cc --- /dev/null +++ b/ansible/roles/nftables-sysadmin/templates/nftables.conf @@ -0,0 +1,41 @@ +#!/usr/sbin/nft -f +# +# Nftables configuration script +# +# Managed by ansible +# roles/nftables/templates/nftables.conf +# +# The ruleset is applied atomically + +flush ruleset + +table inet filter { + chain input { + type filter hook input priority 0; + policy drop; + iif lo accept comment "Accept incoming traffic from localhost" + ct state invalid drop + ct state established,related accept comment "Accept traffic related to outgoing connections" + icmp type echo-request accept + icmpv6 type echo-request counter packets 0 bytes 0 accept + icmpv6 type { nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 1 accept + icmpv6 type { nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 255 counter packets 1 bytes 72 accept + } + + chain forward { + type filter hook forward priority 0; + policy accept; + } + + chain output { + type filter hook output priority 0; + policy accept; + } +} + +# Configure TCP traffic rules +include "/etc/ooni/nftables/tcp/*.nft" + +# Configure any other rule +include "/etc/ooni/nftables/*.nft" + diff --git a/ansible/roles/nginx-buster/files/ffdhe2048_dhparam.pem b/ansible/roles/nginx-buster/files/ffdhe2048_dhparam.pem new file mode 100644 index 00000000..9b182b72 --- /dev/null +++ b/ansible/roles/nginx-buster/files/ffdhe2048_dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS----- diff --git a/ansible/roles/nginx-buster/files/ssl_intermediate.conf b/ansible/roles/nginx-buster/files/ssl_intermediate.conf new file mode 100644 index 00000000..96d2e6e2 --- /dev/null +++ b/ansible/roles/nginx-buster/files/ssl_intermediate.conf @@ -0,0 +1,3 @@ +# Oldest compatible clients: Firefox 1, Chrome 1, IE 7, Opera 5, Safari 1, Windows XP IE8, Android 2.3, Java 7 +ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE +ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; diff --git a/ansible/roles/nginx-buster/files/ssl_modern.conf b/ansible/roles/nginx-buster/files/ssl_modern.conf new file mode 100644 index 00000000..9ad7c11d --- /dev/null +++ b/ansible/roles/nginx-buster/files/ssl_modern.conf @@ -0,0 +1,4 @@ +# Oldest compatible clients: Firefox 27, Chrome 30, IE 11 on Windows 7, Edge, Opera 17, Safari 9, Android 5.0, and Java 8 +ssl_protocols TLSv1.2; +ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; +# NB: technically, it does not require ssl_dhparam as it has no DHE, only ECDHE. diff --git a/ansible/roles/nginx-buster/handlers/main.yml b/ansible/roles/nginx-buster/handlers/main.yml new file mode 100644 index 00000000..b180da14 --- /dev/null +++ b/ansible/roles/nginx-buster/handlers/main.yml @@ -0,0 +1,6 @@ +- name: restart nginx + service: name=nginx state=restarted +- name: start nginx + service: name=nginx state=started +- name: reload nginx + service: name=nginx state=reloaded diff --git a/ansible/roles/nginx-buster/tasks/main.yml b/ansible/roles/nginx-buster/tasks/main.yml new file mode 100644 index 00000000..30f37e57 --- /dev/null +++ b/ansible/roles/nginx-buster/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: install stable nginx + apt: + name: nginx + cache_valid_time: 86400 + notify: start nginx + +# https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4 +# +# Guide https://wiki.mozilla.org/Security/Server_Side_TLS#Pre-defined_DHE_groups +# suggests ffdhe2048 instead of `openssl dhparam` to avoid https://weakdh.org/ +- name: copy nginx configuration snippets + copy: src={{item}} dest=/etc/nginx/{{ item }} mode=0444 owner=root group=root + with_items: + - ffdhe2048_dhparam.pem # ffdhe2048 Diffie-Hellman parameters + - ssl_intermediate.conf + - ssl_modern.conf + +- name: remove `default` vhost + file: path={{item}} state=absent + #notify: reload nginx + with_items: + - /etc/nginx/conf.d/default.conf + - /etc/nginx/sites-available/default + - /etc/nginx/sites-enabled/default + +- name: set nginx.conf + template: + src=nginx.conf + dest=/etc/nginx/nginx.conf + mode=0444 + #notify: reload nginx +... diff --git a/ansible/roles/nginx-buster/templates/nginx.conf b/ansible/roles/nginx-buster/templates/nginx.conf new file mode 100644 index 00000000..f9e742a1 --- /dev/null +++ b/ansible/roles/nginx-buster/templates/nginx.conf @@ -0,0 +1,68 @@ + +# Managed by ansible +# roles/nginx-buster/templates/nginx.conf +# +# Generated with: +# https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4 +# + +user www-data; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + + # Basic Settings + + sendfile on; + tcp_nopush on; # TCP_CORK HTTP headers with sendfile() body into single packet + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Logging Settings + + # anonymize ipaddr + map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; + } + + # log anonymized ipaddr and caching status + log_format ooni_nginx_fmt '$remote_addr_anon $upstream_cache_status [$time_local] ' + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'; + + + access_log syslog:server=unix:/dev/log ooni_nginx_fmt; + error_log syslog:server=unix:/dev/log; + + # Gzip Settings + + gzip on; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + # Virtual Host Configs + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; + +} + diff --git a/ansible/roles/ooni-backend/handlers/main.yml b/ansible/roles/ooni-backend/handlers/main.yml new file mode 100644 index 00000000..84d0f4f1 --- /dev/null +++ b/ansible/roles/ooni-backend/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: reload nftables + service: name=nftables state=reloaded + +- name: restart clickhouse + service: name=clickhouse-server state=restarted diff --git a/ansible/roles/ooni-backend/meta/main.yml b/ansible/roles/ooni-backend/meta/main.yml new file mode 100644 index 00000000..c82f9e2d --- /dev/null +++ b/ansible/roles/ooni-backend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: nftables diff --git a/ansible/roles/ooni-backend/tasks/main.yml b/ansible/roles/ooni-backend/tasks/main.yml new file mode 100644 index 00000000..55c56bf6 --- /dev/null +++ b/ansible/roles/ooni-backend/tasks/main.yml @@ -0,0 +1,822 @@ +--- + +## API ## + +- name: install API if not present + # do not update package if present + tags: api + apt: + cache_valid_time: '{{ apt_cache_valid_time }}' + name: ooni-api + state: present + update_cache: yes + +- name: create Nginx cache dir + file: + path: /var/cache/nginx/ooni-api + state: directory + +- name: configure test api + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: api + template: + src: api.conf + dest: /etc/ooni/api.conf + owner: ooniapi + group: ooniapi + mode: 0640 + vars: + collectors: [] + # bucket_name and collector_id must match the uploader + collector_id: 2 + bucket_name: ooni-data-eu-fra-test + github_push_repo: "ooni-bot/test-lists" + github_origin_repo: "ooni/test-lists" + login_base_url: "https://test-lists.test.ooni.org/login" + pg_uri: "" + clickhouse_url: clickhouse://api:api@localhost/default + # mail_smtp_password: "DISABLED" + # jwt_encryption_key and account_id_hashing_key are taken from the vault + +- name: configure backend-hel api + when: inventory_hostname == 'backend-hel.ooni.org' + tags: api + template: + src: api.conf + dest: /etc/ooni/api.conf + owner: ooniapi + group: ooniapi + mode: 0640 + vars: + collectors: ['backend-hel.ooni.org',] + # bucket_name and collector_id must match the uploader + collector_id: 3 + # test bucket + bucket_name: ooni-data-eu-fra-test + # test GH repo + github_push_repo: "ooni-bot/test-lists" + github_origin_repo: "citizenlab/test-lists" + login_base_url: "https://test-lists.ooni.org/login" + pg_uri: "" + clickhouse_url: clickhouse://api:api@localhost/default + base_url: "https://backend-hel.ooni.org" + +- name: configure backend-fsn api + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: api + template: + src: api.conf + dest: /etc/ooni/api.conf + owner: ooniapi + group: ooniapi + mode: 0640 + vars: + collectors: ['backend-fsn.ooni.org', 'ams-pg.ooni.org'] + # bucket_name and collector_id must match the uploader + collector_id: 1 + bucket_name: ooni-data-eu-fra + github_push_repo: "ooni/test-lists" + github_origin_repo: "citizenlab/test-lists" + login_base_url: "https://test-lists.ooni.org/login" + pg_uri: "" + clickhouse_url: clickhouse://api:api@localhost/default + base_url: "https://api.ooni.io" + +- name: configure prod api + when: inventory_hostname == 'ams-pg.ooni.org' + tags: api + template: + src: api.conf + dest: /etc/ooni/api.conf + owner: ooniapi + group: ooniapi + mode: 0640 + vars: + collectors: ['backend-fsn.ooni.org', 'ams-pg.ooni.org'] + # collector_id must match the uploader + collector_id: 0 + bucket_name: ooni-data-eu-fra + github_push_repo: "ooni/test-lists" + github_origin_repo: "citizenlab/test-lists" + login_base_url: "https://test-lists.ooni.org/login" + pg_uri: "postgresql://shovel:yEqgNr2eXvgG255iEBxVeP@localhost/metadb" + clickhouse_url: "" + +- name: create Psiphon conffile + tags: api + copy: + content: "{{ psiphon_config }}" + dest: /etc/ooni/psiphon_config.json + +- name: Write Tor targets conffile + tags: api + template: + src: tor_targets.json + dest: /etc/ooni/tor_targets.json + +- name: configure api uploader using test bucket + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: api + template: + src: templates/api-uploader.conf + dest: /etc/ooni/api-uploader.conf + vars: + # bucket_name and collector_id must match the API + bucket_name: ooni-data-eu-fra-test + collector_id: 2 + +- name: configure FSN api uploader using PROD bucket + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: api + template: + src: templates/api-uploader.conf + dest: /etc/ooni/api-uploader.conf + vars: + # bucket_name and collector_id must match the API + bucket_name: ooni-data-eu-fra + collector_id: 1 + +# - name: configure HEL api uploader using test bucket +# when: inventory_hostname == 'backend-hel.ooni.org' +# tags: api +# template: +# src: templates/api-uploader.conf +# dest: /etc/ooni/api-uploader.conf +# vars: +# # bucket_name and collector_id must match the API +# bucket_name: ooni-data-eu-fra-test +# collector_id: 3 + + +## Haproxy and nginx ## + +- name: configure api uploader using PROD bucket + when: inventory_hostname == 'ams-pg.ooni.org' + tags: api + template: + src: templates/api-uploader.conf + dest: /etc/ooni/api-uploader.conf + vars: + # bucket_name and collector_id must match the API + bucket_name: ooni-data-eu-fra + collector_id: 0 + +- name: Overwrite API nginx test conf + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: api, webserv + template: + src: templates/nginx-api-test.conf + dest: /etc/nginx/sites-available/ooni-api.conf + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: Overwrite API nginx HEL conf + when: inventory_hostname == 'backend-hel.ooni.org' + tags: api, webserv + template: + src: templates/nginx-api-test.conf + dest: /etc/nginx/sites-available/ooni-api.conf + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: install haproxy if not present + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: webserv + apt: + cache_valid_time: 86400 + name: haproxy + state: present + +- name: Deploy haproxy conf + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: api, webserv + template: + src: templates/haproxy.cfg + dest: /etc/haproxy/haproxy.cfg + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: Delete old files + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: api, webserv + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/nginx/sites-enabled/00-letsencrypt-http + - /etc/nginx/sites-enabled/deb_ooni_org + - /etc/nginx/sites-enabled/deb_ooni_org_http + +- name: Deploy dehydrated conf + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: api, webserv + template: + src: templates/dehydrated.config + dest: /etc/dehydrated/config + mode: 0755 + owner: root + +- name: Deploy dehydrated conf + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: api, webserv + template: + src: templates/dehydrated.config + dest: /etc/dehydrated/config + mode: 0755 + owner: root + +- name: Deploy dehydrated haproxy hook + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: api, webserv + template: + src: templates/dehydrated_haproxy_hook.sh + dest: /etc/dehydrated/haproxy_hook.sh + mode: 0755 + owner: root + +- name: Overwrite API nginx FSN conf + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: api, webserv + template: + src: templates/nginx-api-fsn.conf + dest: /etc/nginx/sites-available/ooni-api.conf + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: Overwrite API nginx prod conf + when: inventory_hostname == 'ams-pg.ooni.org' + tags: api, webserv + template: + src: templates/nginx-api-ams-pg.conf + dest: /etc/nginx/sites-available/ooni-api.conf + mode: 0755 + owner: root + vars: + certpath: /etc/letsencrypt/live/ + +- name: Deploy API gunicorn conf + tags: api + template: + src: api.gunicorn.py + dest: /etc/ooni/api.gunicorn.py + owner: ooniapi + group: ooniapi + mode: 0640 + +- name: Create symlink for API nginx conf + tags: api + file: + src=/etc/nginx/sites-available/ooni-api.conf + dest=/etc/nginx/sites-enabled/ooni-api.conf + state=link + +- name: Configure deb.ooni.org forwarder on FSN host + when: inventory_hostname in ('backend-fsn.ooni.org', ) + tags: deb_ooni_org + # Uses dehydrated + template: + src: deb_ooni_org.nginx.conf + dest: /etc/nginx/sites-enabled/deb_ooni_org + +- name: Configure deb-ci.ooni.org forwarder on test host + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: deb_ooni_org + blockinfile: + path: /etc/nginx/sites-enabled/deb_ooni_org_http + create: yes + block: | + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + server { + listen 80; + server_name deb-ci.ooni.org; + location / { + proxy_pass https://ooni-internal-deb.s3.eu-central-1.amazonaws.com/; + } + } + +- name: create badges dir + tags: api + file: + path: /var/www/package_badges/ + state: directory + +- name: Safely reload Nginx + # TODO remove restart after transition to haproxy + tags: api, deb_ooni_org, webserv + shell: nginx -t && systemctl reload nginx + +- name: Restart Nginx + tags: webserv + shell: nginx -t && systemctl restart nginx + +- name: Restart haproxy + # reload is not enough + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: api, deb_ooni_org, webserv + shell: systemctl restart haproxy + +- name: allow incoming TCP connections to API + tags: api + blockinfile: + path: /etc/ooni/nftables/tcp/443.nft + create: yes + block: | + add rule inet filter input tcp dport 443 counter accept comment "incoming HTTPS" + +- name: allow incoming TCP connections to haproxy metrics + tags: webserv + template: + src: 444.nft + dest: /etc/ooni/nftables/tcp/444.nft + +#- name: reload nftables service +# tags: api +# systemd: +# name: nftables.service +# state: reloaded + +- name: reload nftables service + tags: api, webserv + shell: systemctl reload nftables.service + + +## Fastpath ## + +- name: install fastpath if not present + # do not update package if present + when: inventory_hostname != 'backend-fsn.ooni.org' + tags: fastpath + apt: + cache_valid_time: 86400 + name: fastpath + state: present + +- name: configure fastpath on test + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: fastpath + template: + src: fastpath.conf + dest: /etc/ooni/fastpath.conf + owner: fastpath + group: fastpath + mode: 0640 + vars: + clickhouse_url: clickhouse://fastpath:fastpath@localhost/default + +- name: configure fastpath on FSN + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: fastpath + template: + src: fastpath.conf + dest: /etc/ooni/fastpath.conf + owner: fastpath + group: fastpath + mode: 0640 + vars: + clickhouse_url: clickhouse://fastpath:fastpath@localhost/default + + +- name: configure fastpath on HEL + when: inventory_hostname == 'backend-hel.ooni.org' + tags: fastpath + template: + src: fastpath.conf + dest: /etc/ooni/fastpath.conf + owner: fastpath + group: fastpath + mode: 0640 + vars: + clickhouse_url: clickhouse://fastpath:fastpath@localhost/default +- name: configure fastpath on ams-pg + when: inventory_hostname == 'ams-pg.ooni.org' + tags: fastpath + template: + src: fastpath.conf + dest: /etc/ooni/fastpath.conf + owner: fastpath + group: fastpath + mode: 0640 + vars: + clickhouse_url: + psql_uri: postgresql://shovel:yEqgNr2eXvgG255iEBxVeP@localhost/metadb + + +## Event detector ## + +#- name: install detector +# tags: detector +# apt: +# cache_valid_time: 86400 +# name: detector +# +#- name: configure detector +# tags: detector +# blockinfile: +# path: /etc/ooni/detector.conf +# create: yes +# block: | +# # Managed by ansible, see roles/ooni-backend/tasks/main.yml + + +## Analysis daemon ## + +- name: install analysis + # do not update package if present + when: inventory_hostname != 'backend-fsn.ooni.org' + tags: analysis + apt: + cache_valid_time: 86400 + name: analysis=1.4~pr408-209 + force: True + state: present + +- name: configure analysis + tags: analysis-conf + template: + src: analysis.conf + dest: /etc/ooni/analysis.conf + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + + +## Test helper rotation ## + +- name: configure test helper rotation + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + blockinfile: + path: /etc/ooni/rotation.conf + create: yes + mode: 0400 + block: | + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + [DEFAULT] + # Digital Ocean token + token = {{ digital_ocean_token }} + active_droplets_count = 4 + size_slug = s-1vcpu-1gb + image_name = debian-11-x64 + draining_time_minutes = 1440 + dns_zone = th.ooni.org + +- name: configure test helper rotation certbot + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + blockinfile: + path: /etc/ooni/certbot-digitalocean + create: yes + mode: 0400 + block: | + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + dns_digitalocean_token = {{ digital_ocean_token }} + +- name: configure test helper rotation setup script + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + template: + src: rotation_setup.sh + dest: /etc/ooni/rotation_setup.sh + +- name: create test helper rotation nginx template + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + template: + src: rotation_nginx_conf + dest: /etc/ooni/rotation_nginx_conf + +- name: generate test helper rotation SSH keypair + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + openssh_keypair: + path: /etc/ooni/testhelper_ssh_key + owner: root + group: root + mode: 0400 + type: ed25519 + register: pubkey + +- name: print SSH pubkey + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + debug: msg={{ pubkey.public_key }} + +- name: Enable and start rotation service + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + systemd: + daemon_reload: yes + enabled: yes + name: ooni-rotation.timer + state: started + + +## Tor daemon and onion service ## + + +- name: configure tor onion service hostname + when: inventory_hostname == 'ams-pg.ooni.org' + tags: tor + blockinfile: + path: /var/lib/tor/ooni_onion_service/hostname + create: yes + owner: debian-tor + group: debian-tor + mode: 0644 + block: guegdifjy7bjpequ.onion + +- name: configure tor onion service private_key + when: inventory_hostname == 'ams-pg.ooni.org' + tags: tor + blockinfile: + path: /var/lib/tor/ooni_onion_service/private_key + create: yes + owner: debian-tor + group: debian-tor + mode: 0600 + block: "{{ amspg_ooni_org_onion_key }}" + +- name: set tor onion service directory + when: inventory_hostname == 'ams-pg.ooni.org' + tags: tor + shell: | + chown debian-tor:debian-tor /var/lib/tor/ooni_onion_service + chmod 0700 /var/lib/tor/ooni_onion_service + + +# # Clickhouse # # + +- name: install APT HTTPS support + # do not update package if present + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + apt: + cache_valid_time: 86400 + state: present + name: + - apt-transport-https + - ca-certificates + - dirmngr + +- name: install clickhouse keys + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + command: apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 + +- name: set clickhouse repos + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/apt/sources.list.d/clickhouse.list + create: yes + block: | + deb https://packages.clickhouse.com/deb lts main + +- name: pin clickhouse release train + when: inventory_hostname in ('backend-fsn.ooni.org', ) + tags: clickhouse + blockinfile: + path: /etc/apt/preferences.d/clickhouse-server + create: yes + block: | + Package: clickhouse-server + Pin: version 21.8.12.* + Pin-Priority: 999 + +- name: pin clickhouse release train + when: inventory_hostname in ('backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/apt/preferences.d/clickhouse-server + create: yes + block: | + Package: clickhouse-server + Pin: version 23.8.2.* + Pin-Priority: 999 + +- name: install clickhouse on backend-fsn + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: clickhouse + apt: + # refresh cache + cache_valid_time: 0 + name: + - clickhouse-server={{ clickhouse_pkg_ver }} + - clickhouse-client={{ clickhouse_pkg_ver }} + - clickhouse-common-static={{ clickhouse_pkg_ver }} + vars: + clickhouse_pkg_ver: 21.8.12.* + +- name: install clickhouse on backend-hel + when: inventory_hostname == 'backend-hel.ooni.org' + tags: clickhouse + apt: + # refresh cache + cache_valid_time: 0 + name: + - clickhouse-server={{ clickhouse_pkg_ver }} + - clickhouse-client={{ clickhouse_pkg_ver }} + vars: + clickhouse_pkg_ver: 23.8.2.* + +- name: install clickhouse on ams-pg-test.ooni.org + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: clickhouse + apt: + # refresh cache + cache_valid_time: 0 + name: + - clickhouse-server={{ clickhouse_pkg_ver }} + - clickhouse-client={{ clickhouse_pkg_ver }} + - clickhouse-common-static={{ clickhouse_pkg_ver }} + vars: + clickhouse_pkg_ver: 23.8.2.* + +- name: install clickhouse conf override + when: inventory_hostname in ('backend-fsn.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + template: + src: clickhouse_config.xml + dest: /etc/clickhouse-server/config.d/ooni_conf.xml + owner: clickhouse + group: clickhouse + mode: 0400 + notify: restart clickhouse + +- name: allow incoming TCP connections from monitoring to Clickhouse prometheus interface + when: inventory_hostname in ('backend-fsn.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/ooni/nftables/tcp/9363.nft + create: yes + block: | + add rule inet filter input ip saddr 5.9.112.244 tcp dport 9363 counter accept comment "clickhouse prometheus from monitoring.ooni.org" + notify: reload nftables + +- name: allow incoming TCP connections from jupiter on monitoring.ooni.org to Clickhouse + when: inventory_hostname in ('backend-fsn.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/ooni/nftables/tcp/9000.nft + create: yes + block: | + add rule inet filter input ip saddr 5.9.112.244 tcp dport 9000 counter accept comment "clickhouse from monitoring.ooni.org" + notify: reload nftables + +- name: Run clickhouse + when: inventory_hostname in ('backend-fsn.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + systemd: + name: clickhouse-server.service + state: started + enabled: yes + +## Clickhouse access control ## +# https://clickhouse.com/docs/en/operations/access-rights/#enabling-access-control + +- name: Clickhouse - test admin user - failure is ok to ignore + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse-users + command: clickhouse-client -u admin --password admin -q 'select 1' + ignore_errors: true + register: admin_check + +- name: install tor python3-lxml + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + apt: + cache_valid_time: 86400 + name: python3-lxml + +- name: Clickhouse - set flag + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + # The users.xml file itself needs to be edited for this to work + xml: + path: /etc/clickhouse-server/users.xml + backup: yes + xpath: /clickhouse/users/default/{{ item }} + value: "1" + loop: + - access_management + - named_collection_control + - show_named_collections + - show_named_collections_secrets + register: users_xml + +- name: Clickhouse - restart immediately if needed + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + systemd: + name: clickhouse-server + state: restarted + +- name: Clickhouse - create admin + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + command: clickhouse-client -q "CREATE USER OR REPLACE admin IDENTIFIED WITH sha256_password BY 'admin' HOST LOCAL GRANTEES ANY" + # The server might be still starting: retry as needed + retries: 10 + delay: 5 + register: result + until: result.rc == 0 + +- name: Clickhouse - grant admin rights + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + command: clickhouse-client -q 'GRANT ALL ON *.* TO admin WITH GRANT OPTION' + +- name: Clickhouse - create readonly profile + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + template: + src: clickhouse_readonly.xml + dest: /etc/clickhouse-server/users.d/make_default_readonly.xml + owner: clickhouse + group: clickhouse + mode: 0640 + + #- name: Clickhouse - restore users.xml + # when: admin_check is defined and admin_check is failed + # tags: clickhouse-users + # command: mv {{ users_xml.backup_file }} /etc/clickhouse-server/users.xml + +- name: Clickhouse - restart immediately if needed + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + systemd: + name: clickhouse-server + state: restarted + +- name: Clickhouse - setup users and permissions + tags: clickhouse-users + command: clickhouse-client -u admin --password admin -q "{{ item }}" + loop: + - "CREATE USER OR REPLACE api IDENTIFIED WITH sha256_password BY 'api' HOST LOCAL" + - "GRANT ALL ON *.* TO api" + - "CREATE USER OR REPLACE fastpath IDENTIFIED WITH sha256_password BY 'fastpath' HOST LOCAL" + - "GRANT ALL ON *.* TO fastpath" + +## end of Clickhouse access control ## + + + +- name: Run feeder on ams-pg-test + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: clickhouse + blockinfile: + path: /etc/ooni/clickhouse_feeder.conf + create: yes + block: | + [DEFAULT] + pg_dbuser = readonly + pg_dbhost = localhost + +- name: run feeder on backend-fsn + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: clickhouse + blockinfile: + path: /etc/ooni/clickhouse_feeder.conf + create: yes + block: | + [DEFAULT] + pg_dbuser = readonly + pg_dbhost = ams-pg.ooni.org + +- name: Run feeder + when: inventory_hostname in ('backend-fsn.ooni.org', 'ams-pg-test.ooni.org') + tags: clickhouse + systemd: + name: ooni-clickhouse-feeder.service + state: started + enabled: yes + +- name: Run DB backup on ams-pg-test + when: inventory_hostname == 'ams-pg-test.ooni.org' + tags: dbbackup + template: + src: db-backup.conf + dest: /etc/ooni/db-backup.conf + mode: 0600 + vars: + public_bucket_name: ooni-data-eu-fra-test + +- name: Run DB backup on FSN + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: dbbackup + template: + src: db-backup.conf + dest: /etc/ooni/db-backup.conf + mode: 0600 + vars: + public_bucket_name: ooni-data-eu-fra diff --git a/ansible/roles/ooni-backend/templates/444.nft b/ansible/roles/ooni-backend/templates/444.nft new file mode 100644 index 00000000..03f5106f --- /dev/null +++ b/ansible/roles/ooni-backend/templates/444.nft @@ -0,0 +1,2 @@ +# roles/ooni-backend/templates/444.nft +add rule inet filter input tcp dport 444 counter accept comment "incoming haproxy metrics" diff --git a/ansible/roles/ooni-backend/templates/analysis.conf b/ansible/roles/ooni-backend/templates/analysis.conf new file mode 100644 index 00000000..4df8a8ae --- /dev/null +++ b/ansible/roles/ooni-backend/templates/analysis.conf @@ -0,0 +1,9 @@ +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# [s3bucket] +# bucket_name = ooni-data-eu-fra-test +# aws_access_key_id = +# aws_secret_access_key = + +[backup] +# space separated +table_names = citizenlab fastpath jsonl diff --git a/ansible/roles/ooni-backend/templates/api-uploader.conf b/ansible/roles/ooni-backend/templates/api-uploader.conf new file mode 100644 index 00000000..2de0e399 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/api-uploader.conf @@ -0,0 +1,9 @@ +# OONI API measurement uploader - Python ini format +# Deployed by ansible, see roles/ooni-backend/templates/api-uploader.conf +[DEFAULT] +# arn:aws:iam::676739448697:user/ooni-pipeline, AWS: OONI Open Data +aws_access_key_id = AKIAJURD7T4DTN5JMJ5Q +aws_secret_access_key = {{ s3_ooni_open_data_access_key }} +bucket_name = {{ bucket_name }} +msmt_spool_dir = /var/lib/ooniapi/measurements +collector_id = {{ collector_id }} diff --git a/ansible/roles/ooni-backend/templates/api.conf b/ansible/roles/ooni-backend/templates/api.conf new file mode 100644 index 00000000..25d1d0c6 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/api.conf @@ -0,0 +1,60 @@ +# Deployed by ansible +# See ooni-backend/tasks/main.yml ooni-backend/templates/api.conf +# Syntax: treat it as a Python file, but only uppercase variables are used +COLLECTORS = {{ collectors }} +COLLECTOR_ID = {{ collector_id }} + +# Read-only database access +# The password is already made public +DATABASE_URI_RO = "{{ pg_uri }}" + +DATABASE_STATEMENT_TIMEOUT = 20 + +{% if clickhouse_url|length %} +USE_CLICKHOUSE = True +{% else %} +USE_CLICKHOUSE = False +{% endif %} + +CLICKHOUSE_URL = "{{ clickhouse_url }}" + + +BASE_URL = "{{ base_url }}" + +AUTOCLAVED_BASE_URL = "http://datacollector.infra.ooni.io/ooni-public/autoclaved/" +CENTRIFUGATION_BASE_URL = "http://datacollector.infra.ooni.io/ooni-public/centrifugation/" + +S3_ACCESS_KEY_ID = "AKIAJURD7T4DTN5JMJ5Q" +S3_BUCKET_NAME = "{{ bucket_name }}" +S3_SECRET_ACCESS_KEY = "CHANGEME" +S3_SESSION_TOKEN = "CHANGEME" +S3_ENDPOINT_URL = "CHANGEME" + +PSIPHON_CONFFILE = "/etc/ooni/psiphon_config.json" +TOR_TARGETS_CONFFILE = "/etc/ooni/tor_targets.json" + +JWT_ENCRYPTION_KEY = "{{ jwt_encryption_key }}" +ACCOUNT_ID_HASHING_KEY = "{{ account_id_hashing_key }}" + +SESSION_EXPIRY_DAYS = 180 +LOGIN_EXPIRY_DAYS = 365 + +# Registration email delivery +MAIL_SERVER = "mail.riseup.net" +MAIL_PORT = 465 +MAIL_USE_SSL = True +MAIL_USERNAME = "ooni-mailer" +MAIL_PASSWORD = "{{ mail_smtp_password }}" +MAIL_SOURCE_ADDRESS = "contact@ooni.org" +LOGIN_BASE_URL = "{{ login_base_url }}" + +GITHUB_WORKDIR = "/var/lib/ooniapi/citizenlab" +GITHUB_TOKEN = "{{ github_token }}" +GITHUB_USER = "ooni-bot" +GITHUB_ORIGIN_REPO = "{{ github_origin_repo }}" +GITHUB_PUSH_REPO = "{{ github_push_repo }}" + +# Measurement spool directory +MSMT_SPOOL_DIR = "/var/lib/ooniapi/measurements" +GEOIP_ASN_DB = "/var/lib/ooniapi/asn.mmdb" +GEOIP_CC_DB = "/var/lib/ooniapi/cc.mmdb" diff --git a/ansible/roles/ooni-backend/templates/api.gunicorn.py b/ansible/roles/ooni-backend/templates/api.gunicorn.py new file mode 100644 index 00000000..f86b6f67 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/api.gunicorn.py @@ -0,0 +1,12 @@ +# Gunicorn configuration file +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# and templates/api.gunicorn.py + +workers = 12 + +loglevel = "info" +proc_name = "ooni-api" +reuse_port = True +# Disabled statsd: https://github.com/benoitc/gunicorn/issues/2843 +#statsd_host = "127.0.0.1:8125" +#statsd_prefix = "ooni-api" diff --git a/ansible/roles/ooni-backend/templates/clickhouse_config.xml b/ansible/roles/ooni-backend/templates/clickhouse_config.xml new file mode 100644 index 00000000..e84e53ff --- /dev/null +++ b/ansible/roles/ooni-backend/templates/clickhouse_config.xml @@ -0,0 +1,41 @@ + + + + + information + + +{% if inventory_hostname == 'backend-fsn.ooni.org' %} + production + 20100100100 + +{% else %} + {{ inventory_hostname.replace(".ooni.org", "") }} +{% endif %} + +{% if inventory_hostname == 'ams-pg-test.ooni.org' %} + 500100100 + 3100100100 +{% endif %} + + + 0.0.0.0 + + + + + + + + + /metrics + 9363 + true + true + true + true + + diff --git a/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml b/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml new file mode 100644 index 00000000..73645616 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml @@ -0,0 +1,9 @@ + + + + + + readonly + + + diff --git a/ansible/roles/ooni-backend/templates/clickhouse_users.xml b/ansible/roles/ooni-backend/templates/clickhouse_users.xml new file mode 100644 index 00000000..49fd011a --- /dev/null +++ b/ansible/roles/ooni-backend/templates/clickhouse_users.xml @@ -0,0 +1,31 @@ + + + + + + + 1 + + + + + + + readonly + + 0.0.0.0 + + + + + + {{ clickhouse_writer_password|hash('sha256') }} + + 127.0.0.1 + + + + + + + diff --git a/ansible/roles/ooni-backend/templates/db-backup.conf b/ansible/roles/ooni-backend/templates/db-backup.conf new file mode 100644 index 00000000..4302f0ec --- /dev/null +++ b/ansible/roles/ooni-backend/templates/db-backup.conf @@ -0,0 +1,17 @@ +{ + "ver": 0, + "action": "export", + "public_aws_access_key_id": "AKIAJURD7T4DTN5JMJ5Q", + "public_aws_secret_access_key": "{{ s3_ooni_open_data_access_key }}", + "public_bucket_name": "{{ public_bucket_name }}", + "clickhouse_url": "clickhouse://localhost/default", + "__description": "tables can be backed up as: ignore, full, incremental, partition", + "backup_tables": { + "citizenlab": "ignore", + "fastpath": "ignore", + "jsonl": "ignore", + "msmt_feedback": "ignore", + "test_helper_instances": "ignore", + "url_priorities": "ignore" + } +} diff --git a/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf b/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf new file mode 100644 index 00000000..c069fd55 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf @@ -0,0 +1,64 @@ +# Managed by ansible, see roles/ooni-backend/tasks/main.yml + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# log anonymized ipaddr +log_format deb_ooni_org_logfmt '$remote_addr_anon [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + listen 80; + server_name deb.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info deb_ooni_org_logfmt; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + resolver 127.0.0.1; + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + location / { + proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name deb.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info deb_ooni_org_logfmt; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + location / { + proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + } +} diff --git a/ansible/roles/ooni-backend/templates/dehydrated.config b/ansible/roles/ooni-backend/templates/dehydrated.config new file mode 100644 index 00000000..7a0293a2 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/dehydrated.config @@ -0,0 +1,7 @@ +# Deployed by ansible +# See roles/ooni-backend/templates/dehydrated.config +CONFIG_D=/etc/dehydrated/conf.d +BASEDIR=/var/lib/dehydrated +WELLKNOWN="${BASEDIR}/acme-challenges" +DOMAINS_TXT="/etc/dehydrated/domains.txt" +HOOK="/etc/dehydrated/haproxy_hook.sh" diff --git a/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh b/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh new file mode 100644 index 00000000..0e5b41f3 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Deployed by ansible +# See roles/ooni-backend/templates/dehydrated_haproxy_hook.sh +# +# Deploys chained privkey and certificates for haproxy +# Reloads haproxy as needed + +deploy_cert() { + local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" + # Called once for each certificate + # /var/lib/dehydrated/certs/backend-hel.ooni.org/privkey.pem /var/lib/dehydrated/certs/backend-hel.ooni.org/cert.pem /var/lib/dehydrated/certs/backend-hel.ooni.org/fullchain.pem > /var/lib/dehydrated/certs/backend-hel.ooni.org/haproxy.pem + # cp "${KEYFILE}" "${FULLCHAINFILE}" /etc/nginx/ssl/; chown -R nginx: /etc/nginx/ssl + logger "deploy_cert hook reading ${KEYFILE} ${CERTFILE} ${FULLCHAINFILE}" + cat "${KEYFILE}" "${CERTFILE}" "${FULLCHAINFILE}" > "${KEYFILE}.haproxy" + logger "deploy_cert reloading haproxy" + systemctl reload haproxy.service +} + +HANDLER="$1"; shift +if [[ "${HANDLER}" =~ ^(deploy_cert)$ ]]; then + "$HANDLER" "$@" +fi diff --git a/ansible/roles/ooni-backend/templates/fastpath.conf b/ansible/roles/ooni-backend/templates/fastpath.conf new file mode 100644 index 00000000..031f49a0 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/fastpath.conf @@ -0,0 +1,15 @@ +# See roles/ooni-backend/tasks/main.yml +[DEFAULT] +collectors = localhost +{% if psql_uri is defined %} +# The password is already made public +db_uri = {{ psql_uri }} +{% else %} +db_uri = +{% endif %} +clickhouse_url = {{ clickhouse_url }} + +# S3 access credentials +# Currently unused +s3_access_key = +s3_secret_key = diff --git a/ansible/roles/ooni-backend/templates/haproxy.cfg b/ansible/roles/ooni-backend/templates/haproxy.cfg new file mode 100644 index 00000000..025a4fc2 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/haproxy.cfg @@ -0,0 +1,122 @@ +## Deployed by ansible, see roles/ooni-backend/templates/haproxy.cfg + +# Proxies to: +# - local nginx +# - remote test helpers +# See http://interactive.blockdiag.com/?compression=deflate&src=eJyFjjELwjAQhXd_xeFuEdpBEAURBwfBXSSk6ZkEr7mSZGgR_7tNXdoiuD2--7j3SmL1rKzU8FoAFEUOqz0Y2XhuuxSHICKLiCEKg9Sg3_bmSHHaujaxISRyuJ7hRrJEgh0slVTGOr28Txz2yvQvvYw44R617XGXMTubWU7HzXq26kfl8XISykgidBphVP-whLPuOtRRhIaZ_ogVlt8d7PVYDXkS3x_pgmPP + +global + log /dev/log local0 info alert + log /dev/log local1 notice alert + chroot /var/lib/haproxy + stats socket /run/haproxy/admin.sock mode 660 level admin + stats timeout 30s + user haproxy + group haproxy + daemon + + # Default SSL material locations + ca-base /etc/ssl/certs + crt-base /etc/ssl/private + + # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 50000 + timeout server 50000 + errorfile 400 /etc/haproxy/errors/400.http + errorfile 403 /etc/haproxy/errors/403.http + errorfile 408 /etc/haproxy/errors/408.http + errorfile 500 /etc/haproxy/errors/500.http + errorfile 502 /etc/haproxy/errors/502.http + errorfile 503 /etc/haproxy/errors/503.http + errorfile 504 /etc/haproxy/errors/504.http + + log-format "%[var(txn.src_ipaddr_masked)] %ft > %b > %s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" + +frontend haproxy_metrics + # Metrics exposed on TLS port 444 + # File generated by /etc/dehydrated/haproxy_hook.sh + bind :444 ssl crt /var/lib/dehydrated/certs/"{{ inventory_hostname }}"/privkey.pem.haproxy + + http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) + + # /__haproxy_stats stats page + stats enable + stats uri /__haproxy_stats + stats refresh 5s + + # /__haproxy_prom_metrics prometheus metrics + http-request use-service prometheus-exporter if { path /__haproxy_prom_metrics } + + +frontend public_tls + # TLS on port 443 + # File generated by /etc/dehydrated/haproxy_hook.sh + bind :443 ssl crt /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem.haproxy + + http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) + + # test helpers + default_backend lb_test_helpers + + # deb.ooni.org + acl ACL_deb_ooni_org hdr(host) -i deb.ooni.org + use_backend deb_ooni_org if ACL_deb_ooni_org + + # Nginx + use_backend nginx if !{ path / } || !{ method POST } + + +frontend public_80 + # Forwarded to Nginx for ACME and deb.ooni.org + bind :80 + + http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) + + # ACME + use_backend nginx if { path_beg /.well-known/acme-challenge } + + # deb.ooni.org + acl ACL_deb_ooni_org hdr(host) -i deb.ooni.org + use_backend deb_ooni_org if ACL_deb_ooni_org + + + +backend nginx + # Local Nginx is in front of the API and more. See diagram. + default-server check + option forwardfor + #option httpchk GET / + # forward to local nginx + server nginx localhost:17744 + + +backend lb_test_helpers + # Remote testn helpers + default-server check + option forwardfor + http-check send meth POST uri / hdr Content-Type application/json body "{}" + http-check send-state + http-check comment "TH POST with empty JSON" + + server th0 0.th.ooni.org:443 ssl verify none + server th1 1.th.ooni.org:443 ssl verify none + server th2 2.th.ooni.org:443 ssl verify none + server th3 3.th.ooni.org:443 ssl verify none + #option httpchk + + +backend deb_ooni_org + #default-server check + option forwardfor + server s3-ooni-deb ooni-deb.s3.eu-central-1.amazonaws.com ssl verify none + diff --git a/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf b/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf new file mode 100644 index 00000000..4e3cf934 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf @@ -0,0 +1,297 @@ +# Managed by ansible +# roles/ooni-backend/templates/nginx-api-ams-pg.conf + +# Use 2-level cache, 20MB of RAM + 5GB on disk, +proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M + max_size=5g inactive=24h use_temp_path=off; + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# log anonymized ipaddr and caching status +log_format ooni_api_fmt '$remote_addr_anon $upstream_cache_status [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests + # over Tor correctly land to the proper backend. We are listening on this custom port + # and we are configuring Tor such that it routes traffic to this port. + listen 127.0.0.1:17744; + + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name _; + access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; + gzip on; + + # TODO: we could use different client_max_body_size and SSL configurations for probe service paths + # and everyhing else + client_max_body_size 200M; # for measurement POST + + ssl_certificate {{ certpath }}{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key {{ certpath }}{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate {{ certpath }}{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + + # Registry + # Should match: + # - /api/v1/login + # - /api/v1/register + # - /api/v1/update + location ~^/api/v1/(login|register|update) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://registry.ooni.io:443; + } + + # Selectively route test-list/urls to the API + location ~^/api/v1/test-list/urls { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + } + + # Orchestrate + # Should match: + # - /api/v1/test-list + location ~^/api/v1/(test-list|urls) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://orchestrate.ooni.io:443; + } + + # Web Connectivity Test Helper + # Should match: + # - / + # - /status + # + # The fact that it responds to / means that we may have to differentiate + # via the Host record. + # TODO We should check if clients will respect a suffix added to by the + # bouncer in the returned field, otherwise new clients should use another + # form + location ~^/web-connectivity/(status) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://wcth.ooni.io; + } + + location /whoami { + return 200 "{{ inventory_hostname }}"; + } + + location /metrics { + return 200 ''; + } + + # Expose (only) Netdata badges + location ~ ^/netdata/badge { + rewrite ^/netdata/badge /api/v1/badge.svg break; + proxy_pass http://127.0.0.1:19999; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # Expose package version badges + location /package_badges { + root /var/www; + add_header Pragma "no-cache"; + add_header Cache-Control "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"; + } + + # Temporary redirection to backend-FSN + location ~ ^/api/v1/(aggregation|measurements|raw_measurement|measurement_meta) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/api/_/(asn_by_month|countries|countries_by_month|check_report_id|country_overview|global_overview|global_overview_by_month|im_networks|im_stats|network_stats) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/api/_/(test_coverage|website_networks|website_stats|website_urls|vanilla_tor_stats|test_names) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/_/circumvention_stats_by_country { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = / { + # match "/" strictly, not as a prefix + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/static/ { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + # open and close reports, submit msmt + location ~ ^/report/ { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # Auth, URL sumbission, URL priorities + location ~ ^/api/v1/(url-submission|get_account_role|set_account_role|set_session_expunge|user_login|user_register|user_logout) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/api/_/(url-priorities|account_metadata) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + location ~ ^/api/v1/(collectors|test-helpers|torsf_stats) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/(robots.txt|files) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/v1/test-list/tor-targets { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/v1/test-list/urls { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /bouncer/net-tests { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/v1/test-list/psiphon-config { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + #location ~ ^/api/_/(test_names) { + # proxy_pass https://backend-fsn.ooni.org; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + #} + ## /files* tree + #location ~ ^/files { + # proxy_pass https://backend-fsn.ooni.org; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + #} + #location ~ ^/(health) { + # proxy_pass https://backend-fsn.ooni.org; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + #} + + # Temporary redirect + location = /api/v1/check-in { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # new API + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + } + + # Expose the measurement spool directory + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + + gzip_comp_level 6; + gzip_min_length 1240; + gzip_proxied any; + gzip_types *; + gzip_vary on; + } +} diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf new file mode 100644 index 00000000..9d6e1451 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -0,0 +1,260 @@ +# Managed by ansible +# roles/ooni-backend/templates/nginx-api-fsn.conf + +# Use 2-level cache, 20MB of RAM + 5GB on disk, +proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M + max_size=5g inactive=24h use_temp_path=off; + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# anonymize forwarded ipaddr +map $http_x_forwarded_for $remote_fwd_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + + +# log anonymized ipaddr and caching status +log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests + # over Tor correctly land to the proper backend. We are listening on this custom port + # and we are configuring Tor such that it routes traffic to this port. + listen 127.0.0.1:17744; + + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + server_name _; + access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; + gzip on; + gzip_types text/plain application/xml application/json; + + # TODO: we could use different client_max_body_size and SSL configurations for probe service paths + # and everyhing else + client_max_body_size 200M; # for measurement POST + + ssl_certificate {{ certpath }}{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key {{ certpath }}{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate {{ certpath }}{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + + # Registry + # Should match: + # - /api/v1/login + # - /api/v1/register + # - /api/v1/update + location ~^/api/v1/(login|register|update) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://registry.ooni.io:443; + } + + # Selectively route test-list/urls to the API + location ~^/api/v1/test-list/urls { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # Orchestrate + # Should match: + # - /api/v1/test-list + location ~^/api/v1/(test-list|urls) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://orchestrate.ooni.io:443; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + } + + # Web Connectivity Test Helper + # Should match: + # - / + # - /status + # + # The fact that it responds to / means that we may have to differentiate + # via the Host record. + # TODO We should check if clients will respect a suffix added to by the + # bouncer in the returned field, otherwise new clients should use another + # form + location ~^/web-connectivity/(status) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://wcth.ooni.io; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + location /whoami { + return 200 "{{ inventory_hostname }}"; + } + + location /metrics { + return 200 ''; + } + + # Expose event detector RSS/atom feeds + location ~ ^/detector { + root /var/lib; + default_type application/xml; + } + + # Expose (only) Netdata badges + location ~ ^/netdata/badge { + rewrite ^/netdata/badge /api/v1/badge.svg break; + proxy_pass http://127.0.0.1:19999; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # Expose package version badges + location /package_badges { + root /var/www; + add_header Pragma "no-cache"; + add_header Cache-Control "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # 2022-09-01 20:08 CEST temporarily block a bot scraping /files/download/* + location ~^/files/download/ { + return 301 https://explorer.ooni.org/; + } + + # new API + location / { + + # Protect /apidocs invoked with url= and/or urls= args + if ($uri ~ "^/apidocs") { set $block_apidocs X; } + if ($args ~ "url=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($args ~ "urls=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($block_apidocs ~ "XY") { return 403; } # nested "if" are not supported + + deny 216.244.66.0/24; # DotBot/1.2 + deny 114.119.128.0/19; # PetalBot + allow all; + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + + # match test-helper POST to / and forward traffic to a TH + if ($request_uri = "/") { set $forward_to_th "YE"; } + if ($request_method = POST) { set $forward_to_th "${forward_to_th}S"; } + if ($forward_to_th = "YES") { + proxy_pass https://0.th.ooni.org; + } + + set $external_remote_addr $remote_addr; + if ($remote_addr = "188.166.93.143") { + # If remote_addr is ams-pg-test trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + if ($remote_addr = "142.93.237.101") { + # If remote_addr is ams-pg trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + proxy_set_header X-Real-IP $external_remote_addr; + + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # Expose the measurement spool directory + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + } +} + +# Used by Netdata to monitor Nginx +server { + listen 127.0.0.1:80; + server_name localhost; + location = /stub_status { + stub_status; + } +} diff --git a/ansible/roles/ooni-backend/templates/nginx-api-test.conf b/ansible/roles/ooni-backend/templates/nginx-api-test.conf new file mode 100644 index 00000000..092d40db --- /dev/null +++ b/ansible/roles/ooni-backend/templates/nginx-api-test.conf @@ -0,0 +1,157 @@ +# Managed by ansible +# roles/ooni-backend/templates/nginx-api-test.conf + +# Use 2-level cache, 20MB of RAM + 5GB on disk, +proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M + max_size=5g inactive=24h use_temp_path=off; + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# anonymize forwarded ipaddr +map $http_x_forwarded_for $remote_fwd_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + + +# log anonymized ipaddr and caching status +log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests + # over Tor correctly land to the proper backend. We are listening on this custom port + # and we are configuring Tor such that it routes traffic to this port. + listen 127.0.0.1:17744; + server_name _; + access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; + gzip on; + gzip_types text/plain application/xml application/json; + + # TODO: we could use different client_max_body_size and SSL configurations for probe service paths + # and everyhing else + client_max_body_size 200M; # for measurement POST + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + # use systemd-resolved + resolver 127.0.0.53; + + # Selectively route test-list/urls to the API + location ~^/api/v1/test-list/urls { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + location /whoami { + return 200 "{{ inventory_hostname }}"; + } + + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + + # 2022-09-01 20:08 CEST temporarily block a bot scraping /files/download/* + location ~^/files/download/ { + return 301 https://explorer.ooni.org/; + } + + # new API + location / { + + # Protect /apidocs invoked with url= and/or urls= args + if ($uri ~ "^/apidocs") { set $block_apidocs X; } + if ($args ~ "url=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($args ~ "urls=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($block_apidocs ~ "XY") { return 403; } # nested "if" are not supported + + deny 216.244.66.0/24; # DotBot/1.2 + deny 114.119.128.0/19; # PetalBot + allow all; + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + + set $external_remote_addr $remote_addr; + if ($remote_addr = "188.166.93.143") { + # If remote_addr is ams-pg-test trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + if ($remote_addr = "142.93.237.101") { + # If remote_addr is ams-pg trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + proxy_set_header X-Real-IP $external_remote_addr; + + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # Expose the measurement spool directory + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + } +} + +server { + # Forward deb.ooni.org to S3 + listen 17744; + server_name deb.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + resolver 127.0.0.53; + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + location / { + proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + } +} diff --git a/ansible/roles/ooni-backend/templates/rotation_nginx_conf b/ansible/roles/ooni-backend/templates/rotation_nginx_conf new file mode 100644 index 00000000..63255e51 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/rotation_nginx_conf @@ -0,0 +1,70 @@ +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# and roles/ooni-backend/templates/rotation_nginx_conf +# Deployed by rotation tool to the test-helper hosts +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M + max_size=5g inactive=24h use_temp_path=off; + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name _; + gzip on; + ssl_certificate /etc/ssl/private/th_fullchain.pem; + ssl_certificate_key /etc/ssl/private/th_privkey.pem; + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + add_header Strict-Transport-Security "max-age=63072000" always; + ssl_stapling on; + ssl_stapling_verify on; + resolver 127.0.0.1; + # local test helper + location / { + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + proxy_pass http://127.0.0.1:8080; + + proxy_cache thcache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + # Cache POST without headers set by the test helper! + proxy_cache_methods POST; + proxy_cache_key "$request_uri|$request_body"; + proxy_cache_valid 200 10m; + proxy_cache_valid any 0; + add_header X-Cache-Status $upstream_cache_status; + + } +} + +# Used by Netdata to monitor Nginx +server { + listen 127.0.0.1:80; + server_name localhost; + + allow 5.9.112.244; # monitoring host + deny all; + + location = /stub_status { + stub_status; + } +} + +# Used by Prometheus to reach the TH +server { + listen 9001; + server_name localhost; + + allow 5.9.112.244; # monitoring host + deny all; + + location = /metrics { + proxy_pass http://127.0.0.1:9091; + } +} diff --git a/ansible/roles/ooni-backend/templates/rotation_setup.sh b/ansible/roles/ooni-backend/templates/rotation_setup.sh new file mode 100644 index 00000000..5706150c --- /dev/null +++ b/ansible/roles/ooni-backend/templates/rotation_setup.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# +# Configure test-helper droplet +# This script is run remotely on newly spawned VM by https://github.com/ooni/backend/blob/master/analysis/rotation.py +# It runs as root and with CWD=/ +# +set -euo pipefail +exec 1>/var/log/vm_rotation_setup.log 2>&1 +echo > /etc/motd + +echo "Configuring APT" +echo "deb [trusted=yes] https://ooni-deb.s3.eu-central-1.amazonaws.com unstable main" > /etc/apt/sources.list.d/ooni.list +cat < /etc/apt/trusted.gpg.d/ooni.gpg +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ +JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh +dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb +AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb +gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo +Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX +HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh +tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy +6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s +oN7jfdbznrWVigE= +=PtYb +-----END PGP PUBLIC KEY BLOCK----- +EOF + +# Vector +cat < /etc/apt/trusted.gpg.d/vector.gpg +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2 + +mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ +jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d +2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy +Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL +Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz +629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj +a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE +EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor +xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X +muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE +RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA +gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm +0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g +3vq0NvX0k2CLi/W9mTiSdHXFChI15A== +=k36w +-----END PGP PUBLIC KEY BLOCK----- +EOF + +echo "deb https://repositories.timber.io/public/vector/deb/debian bullseye main" > /etc/apt/sources.list.d/vector.list + +echo "Installing packages" +export DEBIAN_FRONTEND=noninteractive +apt-get update -q +apt-get purge -qy unattended-upgrades rsyslog +apt-get upgrade -qy +apt-get install -qy --no-install-recommends chrony netdata oohelperd netdata-plugins-python + +systemctl daemon-reload +systemctl restart systemd-journald.service +logger start +systemctl restart systemd-journald.service + +apt-get install -qy --no-install-recommends vector + +echo "Configuring Vector" +# The certs are copied over by rotation.py +cat > /etc/vector/vector.toml < /etc/netdata/netdata.conf < /var/run/rotation_setup_completed diff --git a/ansible/roles/ooni-backend/templates/tor_targets.json b/ansible/roles/ooni-backend/templates/tor_targets.json new file mode 100644 index 00000000..933c4ede --- /dev/null +++ b/ansible/roles/ooni-backend/templates/tor_targets.json @@ -0,0 +1,304 @@ +{ + "128.31.0.39:9101": { + "address": "128.31.0.39:9101", + "fingerprint": "9695DFC35FFEB861329B9F1AB04C46397020CE31", + "name": "moria1", + "protocol": "or_port_dirauth" + }, + "128.31.0.39:9131": { + "address": "128.31.0.39:9131", + "fingerprint": "9695DFC35FFEB861329B9F1AB04C46397020CE31", + "name": "moria1", + "protocol": "dir_port" + }, + "131.188.40.189:443": { + "address": "131.188.40.189:443", + "fingerprint": "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281", + "name": "gabelmoo", + "protocol": "or_port_dirauth" + }, + "131.188.40.189:80": { + "address": "131.188.40.189:80", + "fingerprint": "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281", + "name": "gabelmoo", + "protocol": "dir_port" + }, + "154.35.175.225:443": { + "address": "154.35.175.225:443", + "fingerprint": "CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC", + "name": "Faravahar", + "protocol": "or_port_dirauth" + }, + "154.35.175.225:80": { + "address": "154.35.175.225:80", + "fingerprint": "CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC", + "name": "Faravahar", + "protocol": "dir_port" + }, + "171.25.193.9:443": { + "address": "171.25.193.9:443", + "fingerprint": "BD6A829255CB08E66FBE7D3748363586E46B3810", + "name": "maatuska", + "protocol": "dir_port" + }, + "171.25.193.9:80": { + "address": "171.25.193.9:80", + "fingerprint": "BD6A829255CB08E66FBE7D3748363586E46B3810", + "name": "maatuska", + "protocol": "or_port_dirauth" + }, + "193.23.244.244:443": { + "address": "193.23.244.244:443", + "fingerprint": "7BE683E65D48141321C5ED92F075C55364AC7123", + "name": "dannenberg", + "protocol": "or_port_dirauth" + }, + "193.23.244.244:80": { + "address": "193.23.244.244:80", + "fingerprint": "7BE683E65D48141321C5ED92F075C55364AC7123", + "name": "dannenberg", + "protocol": "dir_port" + }, + "199.58.81.140:443": { + "address": "199.58.81.140:443", + "fingerprint": "74A910646BCEEFBCD2E874FC1DC997430F968145", + "name": "longclaw", + "protocol": "or_port_dirauth" + }, + "199.58.81.140:80": { + "address": "199.58.81.140:80", + "fingerprint": "74A910646BCEEFBCD2E874FC1DC997430F968145", + "name": "longclaw", + "protocol": "dir_port" + }, + "204.13.164.118:443": { + "address": "204.13.164.118:443", + "fingerprint": "24E2F139121D4394C54B5BCC368B3B411857C413", + "name": "bastet", + "protocol": "or_port_dirauth" + }, + "204.13.164.118:80": { + "address": "204.13.164.118:80", + "fingerprint": "24E2F139121D4394C54B5BCC368B3B411857C413", + "name": "bastet", + "protocol": "dir_port" + }, + "2d7292b5163fb7de5b24cd04032c93a2d4c454431de3a00b5a6d4a3309529e49": { + "address": "193.11.166.194:27020", + "fingerprint": "86AC7B8D430DAC4117E9F42C9EAED18133863AAF", + "params": { + "cert": [ + "0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "3fa772a44e07856b4c70e958b2f6dc8a29450a823509d5dbbf8b884e7fb5bb9d": { + "address": "192.95.36.142:443", + "fingerprint": "CDF2E852BF539B82BD10E27E9115A31734E378C2", + "params": { + "cert": [ + "qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ" + ], + "iat-mode": [ + "1" + ] + }, + "protocol": "obfs4" + }, + "45.66.33.45:443": { + "address": "45.66.33.45:443", + "fingerprint": "7EA6EAD6FD83083C538F44038BBFA077587DD755", + "name": "dizum", + "protocol": "or_port_dirauth" + }, + "45.66.33.45:80": { + "address": "45.66.33.45:80", + "fingerprint": "7EA6EAD6FD83083C538F44038BBFA077587DD755", + "name": "dizum", + "protocol": "dir_port" + }, + "49116bf72d336bb8724fd3a06a5afa7bbd4e7baef35fbcdb9a98d13e702270ad": { + "address": "146.57.248.225:22", + "fingerprint": "10A6CD36A537FCE513A322361547444B393989F0", + "params": { + "cert": [ + "K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "4a330634c5d678887f0f7c299490af43a6ac9fa944a6cc2140ab264c9ec124a0": { + "address": "209.148.46.65:443", + "fingerprint": "74FAD13168806246602538555B5521A0383A1875", + "params": { + "cert": [ + "ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "548eebff71da6128321c3bc1c3ec12b5bfff277ef5cde32709a33e207b57f3e2": { + "address": "37.218.245.14:38224", + "fingerprint": "D9A82D2F9C2F65A18407B1D2B764F130847F8B5D", + "params": { + "cert": [ + "bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "5aeb9e43b43fc8a809b8d25aae968395a5ceea0e677caaf56e1c0a2ba002f5b5": { + "address": "193.11.166.194:27015", + "fingerprint": "2D82C2E354D531A68469ADF7F878FA6060C6BACA", + "params": { + "cert": [ + "4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "66.111.2.131:9001": { + "address": "66.111.2.131:9001", + "fingerprint": "BA44A889E64B93FAA2B114E02C2A279A8555C533", + "name": "Serge", + "protocol": "or_port_dirauth" + }, + "66.111.2.131:9030": { + "address": "66.111.2.131:9030", + "fingerprint": "BA44A889E64B93FAA2B114E02C2A279A8555C533", + "name": "Serge", + "protocol": "dir_port" + }, + "662218447d396b9d4f01b585457d267735601fedbeb9a19b86b942f238fe4e7b": { + "address": "51.222.13.177:80", + "fingerprint": "5EDAC3B810E12B01F6FD8050D2FD3E277B289A08", + "params": { + "cert": [ + "2uplIpLQ0q9+0qMFrK5pkaYRDOe460LL9WHBvatgkuRr/SL31wBOEupaMMJ6koRE6Ld0ew" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "75fe96d641a078fee06529af376d7f8c92757596e48558d5d02baa1e10321d10": { + "address": "45.145.95.6:27015", + "fingerprint": "C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C", + "params": { + "cert": [ + "TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "86.59.21.38:443": { + "address": "86.59.21.38:443", + "fingerprint": "847B1F850344D7876491A54892F904934E4EB85D", + "name": "tor26", + "protocol": "or_port_dirauth" + }, + "86.59.21.38:80": { + "address": "86.59.21.38:80", + "fingerprint": "847B1F850344D7876491A54892F904934E4EB85D", + "name": "tor26", + "protocol": "dir_port" + }, + "99e9adc8bba0d60982dbc655b5e8735d88ad788905c3713a39eff3224b617eeb": { + "address": "38.229.1.78:80", + "fingerprint": "C8CBDB2464FC9804A69531437BCF2BE31FDD2EE4", + "params": { + "cert": [ + "Hmyfd2ev46gGY7NoVxA9ngrPF2zCZtzskRTzoWXbxNkzeVnGFPWmrTtILRyqCTjHR+s9dg" + ], + "iat-mode": [ + "1" + ] + }, + "protocol": "obfs4" + }, + "9d735c6e70512123ab2c2fe966446b2345b352c512e9fb359f4b1673236e4d4a": { + "address": "38.229.33.83:80", + "fingerprint": "0BAC39417268B96B9F514E7F63FA6FBA1A788955", + "params": { + "cert": [ + "VwEFpk9F/UN9JED7XpG1XOjm/O8ZCXK80oPecgWnNDZDv5pdkhq1OpbAH0wNqOT6H6BmRQ" + ], + "iat-mode": [ + "1" + ] + }, + "protocol": "obfs4" + }, + "b7c0e3f183ad85a6686ec68344765cec57906b215e7b82a98a9ca013cb980efa": { + "address": "193.11.166.194:27025", + "fingerprint": "1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF", + "params": { + "cert": [ + "ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "b8de51da541ced804840b1d8fd24d5ff1cfdf07eae673dae38c2bc2cce594ddd": { + "address": "85.31.186.26:443", + "fingerprint": "91A6354697E6B02A386312F68D82CF86824D3606", + "params": { + "cert": [ + "PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "d2d6e34abeda851f7cd37138ffafcce992b2ccdb0f263eb90ab75d7adbd5eeba": { + "address": "85.31.186.98:443", + "fingerprint": "011F2599C0E9B27EE74B353155E244813763C3E5", + "params": { + "cert": [ + "ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "f855ba38d517d8589c16e1333ac23c6e516532cf036ab6f47b15030b40a3b6a6": { + "address": "[2a0c:4d80:42:702::1]:27015", + "fingerprint": "C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C", + "params": { + "cert": [ + "TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + } +} \ No newline at end of file