From 137ede2864f756d3d70079a7f5d570d46ea70477 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 14:30:14 -0500 Subject: [PATCH 01/20] cleanup: unrequired backend configs --- ansible/roles/ooni-backend/tasks/main.yml | 368 +----------------- .../templates/clickhouse_config.xml | 41 -- .../templates/clickhouse_readonly.xml | 9 - .../templates/clickhouse_users.xml | 31 -- .../templates/nginx-api-ams-pg.conf | 297 -------------- .../ooni-backend/templates/nginx-api-fsn.conf | 8 +- .../templates/nginx-api-test.conf | 8 - .../templates/rotation_nginx_conf | 70 ---- .../ooni-backend/templates/rotation_setup.sh | 114 ------ .../ooni-backend/templates/tor_targets.json | 46 +-- 10 files changed, 26 insertions(+), 966 deletions(-) delete mode 100644 ansible/roles/ooni-backend/templates/clickhouse_config.xml delete mode 100644 ansible/roles/ooni-backend/templates/clickhouse_readonly.xml delete mode 100644 ansible/roles/ooni-backend/templates/clickhouse_users.xml delete mode 100644 ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf delete mode 100644 ansible/roles/ooni-backend/templates/rotation_nginx_conf delete mode 100644 ansible/roles/ooni-backend/templates/rotation_setup.sh diff --git a/ansible/roles/ooni-backend/tasks/main.yml b/ansible/roles/ooni-backend/tasks/main.yml index a6ee12d..42da375 100644 --- a/ansible/roles/ooni-backend/tasks/main.yml +++ b/ansible/roles/ooni-backend/tasks/main.yml @@ -26,7 +26,7 @@ group: ooniapi mode: 0640 vars: - collectors: [] + collectors: ['backend-hel.ooni.org'] # bucket_name and collector_id must match the uploader collector_id: 2 bucket_name: ooni-data-eu-fra-test @@ -252,17 +252,10 @@ src: 444.nft dest: /etc/ooni/nftables/tcp/444.nft -#- name: reload nftables service -# tags: api -# systemd: -# name: nftables.service -# state: reloaded - - name: reload nftables service tags: api, webserv shell: systemctl reload nftables.service - ## Fastpath ## - name: install fastpath if not present @@ -298,25 +291,6 @@ vars: clickhouse_url: clickhouse://fastpath:fastpath@localhost/default - - -## Event detector ## - -#- name: install detector -# tags: detector -# apt: -# cache_valid_time: 86400 -# name: detector -# -#- name: configure detector -# tags: detector -# blockinfile: -# path: /etc/ooni/detector.conf -# create: yes -# block: | -# # Managed by ansible, see roles/ooni-backend/tasks/main.yml - - ## Analysis daemon ## - name: install analysis @@ -336,346 +310,6 @@ dest: /etc/ooni/analysis.conf # Managed by ansible, see roles/ooni-backend/tasks/main.yml - -## Test helper rotation ## - -- name: configure test helper rotation - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - blockinfile: - path: /etc/ooni/rotation.conf - create: yes - mode: 0400 - block: | - # Managed by ansible, see roles/ooni-backend/tasks/main.yml - [DEFAULT] - # Digital Ocean token - token = {{ digital_ocean_token }} - active_droplets_count = 4 - size_slug = s-1vcpu-1gb - image_name = debian-11-x64 - draining_time_minutes = 1440 - dns_zone = th.ooni.org - -- name: configure test helper rotation certbot - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - blockinfile: - path: /etc/ooni/certbot-digitalocean - create: yes - mode: 0400 - block: | - # Managed by ansible, see roles/ooni-backend/tasks/main.yml - dns_digitalocean_token = {{ digital_ocean_token }} - -- name: configure test helper rotation setup script - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - template: - src: rotation_setup.sh - dest: /etc/ooni/rotation_setup.sh - -- name: create test helper rotation nginx template - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - template: - src: rotation_nginx_conf - dest: /etc/ooni/rotation_nginx_conf - -- name: generate test helper rotation SSH keypair - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - openssh_keypair: - path: /etc/ooni/testhelper_ssh_key - owner: root - group: root - mode: 0400 - type: ed25519 - register: pubkey - -- name: print SSH pubkey - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - debug: msg={{ pubkey.public_key }} - -- name: Enable and start rotation service - tags: rotation - when: inventory_hostname == 'backend-fsn.ooni.org' - systemd: - daemon_reload: yes - enabled: yes - name: ooni-rotation.timer - state: started - - -## Tor daemon and onion service ## - -## TODO(decfox): get rid of this? -- name: configure tor onion service hostname - when: inventory_hostname == 'ams-pg.ooni.org' - tags: tor - blockinfile: - path: /var/lib/tor/ooni_onion_service/hostname - create: yes - owner: debian-tor - group: debian-tor - mode: 0644 - block: guegdifjy7bjpequ.onion - -- name: configure tor onion service private_key - when: inventory_hostname == 'ams-pg.ooni.org' - tags: tor - blockinfile: - path: /var/lib/tor/ooni_onion_service/private_key - create: yes - owner: debian-tor - group: debian-tor - mode: 0600 - block: "{{ amspg_ooni_org_onion_key }}" - -- name: set tor onion service directory - when: inventory_hostname == 'ams-pg.ooni.org' - tags: tor - shell: | - chown debian-tor:debian-tor /var/lib/tor/ooni_onion_service - chmod 0700 /var/lib/tor/ooni_onion_service - - -# # Clickhouse # # - -- name: install APT HTTPS support - # do not update package if present - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - apt: - cache_valid_time: 86400 - state: present - name: - - apt-transport-https - - ca-certificates - - dirmngr - -- name: install clickhouse keys - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - command: apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 - -- name: set clickhouse repos - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - blockinfile: - path: /etc/apt/sources.list.d/clickhouse.list - create: yes - block: | - deb https://packages.clickhouse.com/deb lts main - -- name: pin clickhouse release train - when: inventory_hostname in ('backend-fsn.ooni.org', ) - tags: clickhouse - blockinfile: - path: /etc/apt/preferences.d/clickhouse-server - create: yes - block: | - Package: clickhouse-server - Pin: version 21.8.12.* - Pin-Priority: 999 - -- name: pin clickhouse release train - when: inventory_hostname in ('backend-hel.ooni.org') - tags: clickhouse - blockinfile: - path: /etc/apt/preferences.d/clickhouse-server - create: yes - block: | - Package: clickhouse-server - Pin: version 23.8.2.* - Pin-Priority: 999 - -- name: install clickhouse on backend-fsn - when: inventory_hostname == 'backend-fsn.ooni.org' - tags: clickhouse - apt: - # refresh cache - cache_valid_time: 0 - name: - - clickhouse-server={{ clickhouse_pkg_ver }} - - clickhouse-client={{ clickhouse_pkg_ver }} - - clickhouse-common-static={{ clickhouse_pkg_ver }} - vars: - clickhouse_pkg_ver: 21.8.12.* - -- name: install clickhouse on backend-hel.ooni.org - when: inventory_hostname == 'backend-hel.ooni.org' - tags: clickhouse - apt: - # refresh cache - cache_valid_time: 0 - name: - - clickhouse-server={{ clickhouse_pkg_ver }} - - clickhouse-client={{ clickhouse_pkg_ver }} - - clickhouse-common-static={{ clickhouse_pkg_ver }} - vars: - clickhouse_pkg_ver: 23.8.2.* - -- name: install clickhouse conf override - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - template: - src: clickhouse_config.xml - dest: /etc/clickhouse-server/config.d/ooni_conf.xml - owner: clickhouse - group: clickhouse - mode: 0400 - notify: restart clickhouse - -- name: allow incoming TCP connections from monitoring to Clickhouse prometheus interface - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - blockinfile: - path: /etc/ooni/nftables/tcp/9363.nft - create: yes - block: | - add rule inet filter input ip saddr 5.9.112.244 tcp dport 9363 counter accept comment "clickhouse prometheus from monitoring.ooni.org" - notify: reload nftables - -- name: allow incoming TCP connections from jupiter on monitoring.ooni.org to Clickhouse - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - blockinfile: - path: /etc/ooni/nftables/tcp/9000.nft - create: yes - block: | - add rule inet filter input ip saddr 5.9.112.244 tcp dport 9000 counter accept comment "clickhouse from monitoring.ooni.org" - notify: reload nftables - -- name: Run clickhouse - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - systemd: - name: clickhouse-server.service - state: started - enabled: yes - -## Clickhouse access control ## -# https://clickhouse.com/docs/en/operations/access-rights/#enabling-access-control - -- name: Clickhouse - test admin user - failure is ok to ignore - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse-users - command: clickhouse-client -u admin --password admin -q 'select 1' - ignore_errors: true - register: admin_check - -- name: install tor python3-lxml - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - apt: - cache_valid_time: 86400 - name: python3-lxml - -- name: Clickhouse - set flag - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - # The users.xml file itself needs to be edited for this to work - xml: - path: /etc/clickhouse-server/users.xml - backup: yes - xpath: /clickhouse/users/default/{{ item }} - value: "1" - loop: - - access_management - - named_collection_control - - show_named_collections - - show_named_collections_secrets - register: users_xml - -- name: Clickhouse - restart immediately if needed - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - systemd: - name: clickhouse-server - state: restarted - -- name: Clickhouse - create admin - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - command: clickhouse-client -q "CREATE USER OR REPLACE admin IDENTIFIED WITH sha256_password BY 'admin' HOST LOCAL GRANTEES ANY" - # The server might be still starting: retry as needed - retries: 10 - delay: 5 - register: result - until: result.rc == 0 - -- name: Clickhouse - grant admin rights - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - command: clickhouse-client -q 'GRANT ALL ON *.* TO admin WITH GRANT OPTION' - -- name: Clickhouse - create readonly profile - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - template: - src: clickhouse_readonly.xml - dest: /etc/clickhouse-server/users.d/make_default_readonly.xml - owner: clickhouse - group: clickhouse - mode: 0640 - - #- name: Clickhouse - restore users.xml - # when: admin_check is defined and admin_check is failed - # tags: clickhouse-users - # command: mv {{ users_xml.backup_file }} /etc/clickhouse-server/users.xml - -- name: Clickhouse - restart immediately if needed - when: admin_check is defined and admin_check is failed - tags: clickhouse-users - systemd: - name: clickhouse-server - state: restarted - -- name: Clickhouse - setup users and permissions - tags: clickhouse-users - command: clickhouse-client -u admin --password admin -q "{{ item }}" - loop: - - "CREATE USER OR REPLACE api IDENTIFIED WITH sha256_password BY 'api' HOST LOCAL" - - "GRANT ALL ON *.* TO api" - - "CREATE USER OR REPLACE fastpath IDENTIFIED WITH sha256_password BY 'fastpath' HOST LOCAL" - - "GRANT ALL ON *.* TO fastpath" - -## end of Clickhouse access control ## - - - -- name: Run feeder on backend-hel - when: inventory_hostname == 'backend-hel.ooni.org' - tags: clickhouse - blockinfile: - path: /etc/ooni/clickhouse_feeder.conf - create: yes - block: | - [DEFAULT] - pg_dbuser = readonly - pg_dbhost = localhost - -- name: run feeder on backend-fsn - when: inventory_hostname == 'backend-fsn.ooni.org' - tags: clickhouse - blockinfile: - path: /etc/ooni/clickhouse_feeder.conf - create: yes - block: | - [DEFAULT] - pg_dbuser = readonly - pg_dbhost = backend-hel.ooni.org - -- name: Run feeder - when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') - tags: clickhouse - systemd: - name: ooni-clickhouse-feeder.service - state: started - enabled: yes - - name: Run DB backup on backend-hel when: inventory_hostname == 'backend-hel.ooni.org' tags: dbbackup diff --git a/ansible/roles/ooni-backend/templates/clickhouse_config.xml b/ansible/roles/ooni-backend/templates/clickhouse_config.xml deleted file mode 100644 index 548c2a8..0000000 --- a/ansible/roles/ooni-backend/templates/clickhouse_config.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - information - - -{% if inventory_hostname == 'backend-fsn.ooni.org' %} - production - 20100100100 - -{% else %} - {{ inventory_hostname.replace(".ooni.org", "") }} -{% endif %} - -{% if inventory_hostname == 'backend-hel.ooni.org' %} - 500100100 - 3100100100 -{% endif %} - - - 0.0.0.0 - - - - - - - - - /metrics - 9363 - true - true - true - true - - diff --git a/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml b/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml deleted file mode 100644 index 7364561..0000000 --- a/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - readonly - - - diff --git a/ansible/roles/ooni-backend/templates/clickhouse_users.xml b/ansible/roles/ooni-backend/templates/clickhouse_users.xml deleted file mode 100644 index 49fd011..0000000 --- a/ansible/roles/ooni-backend/templates/clickhouse_users.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - 1 - - - - - - - readonly - - 0.0.0.0 - - - - - - {{ clickhouse_writer_password|hash('sha256') }} - - 127.0.0.1 - - - - - - - diff --git a/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf b/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf deleted file mode 100644 index 4e3cf93..0000000 --- a/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf +++ /dev/null @@ -1,297 +0,0 @@ -# Managed by ansible -# roles/ooni-backend/templates/nginx-api-ams-pg.conf - -# Use 2-level cache, 20MB of RAM + 5GB on disk, -proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M - max_size=5g inactive=24h use_temp_path=off; - -# anonymize ipaddr -map $remote_addr $remote_addr_anon { - ~(?P\d+\.\d+\.\d+)\. $ip.0; - ~(?P[^:]+:[^:]+): $ip::; - default 0.0.0.0; -} - -# log anonymized ipaddr and caching status -log_format ooni_api_fmt '$remote_addr_anon $upstream_cache_status [$time_local] ' - '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; - -server { - # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests - # over Tor correctly land to the proper backend. We are listening on this custom port - # and we are configuring Tor such that it routes traffic to this port. - listen 127.0.0.1:17744; - - listen 443 ssl http2; - listen [::]:443 ssl http2; - server_name _; - access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; - gzip on; - - # TODO: we could use different client_max_body_size and SSL configurations for probe service paths - # and everyhing else - client_max_body_size 200M; # for measurement POST - - ssl_certificate {{ certpath }}{{ inventory_hostname }}/fullchain.pem; - ssl_certificate_key {{ certpath }}{{ inventory_hostname }}/privkey.pem; - ssl_trusted_certificate {{ certpath }}{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify - - # Use the intermediate configuration to support legacy probes - # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 - ssl_session_timeout 5m; - ssl_session_cache shared:MozSSL:30m; - ssl_session_tickets off; - - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; - ssl_prefer_server_ciphers off; - - # HSTS (ngx_http_headers_module is required) (63072000 seconds) - add_header Strict-Transport-Security "max-age=63072000" always; - - # OCSP stapling - ssl_stapling on; - ssl_stapling_verify on; - - # verify chain of trust of OCSP response using Root CA and Intermediate certs - #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; - - resolver 127.0.0.1; - - # Registry - # Should match: - # - /api/v1/login - # - /api/v1/register - # - /api/v1/update - location ~^/api/v1/(login|register|update) { - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - - proxy_pass https://registry.ooni.io:443; - } - - # Selectively route test-list/urls to the API - location ~^/api/v1/test-list/urls { - proxy_pass http://127.0.0.1:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_cache apicache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - proxy_cache_methods HEAD GET; - # Cache only 200, 301, and 302 by default and for very short. - # Overridden by the API using the Expires header - proxy_cache_valid 200 301 302 10s; - proxy_cache_valid any 0; - add_header x-cache-status $upstream_cache_status; - add_header X-Cache-Status $upstream_cache_status; - } - - # Orchestrate - # Should match: - # - /api/v1/test-list - location ~^/api/v1/(test-list|urls) { - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - - proxy_pass https://orchestrate.ooni.io:443; - } - - # Web Connectivity Test Helper - # Should match: - # - / - # - /status - # - # The fact that it responds to / means that we may have to differentiate - # via the Host record. - # TODO We should check if clients will respect a suffix added to by the - # bouncer in the returned field, otherwise new clients should use another - # form - location ~^/web-connectivity/(status) { - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - - proxy_pass https://wcth.ooni.io; - } - - location /whoami { - return 200 "{{ inventory_hostname }}"; - } - - location /metrics { - return 200 ''; - } - - # Expose (only) Netdata badges - location ~ ^/netdata/badge { - rewrite ^/netdata/badge /api/v1/badge.svg break; - proxy_pass http://127.0.0.1:19999; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - - # Expose package version badges - location /package_badges { - root /var/www; - add_header Pragma "no-cache"; - add_header Cache-Control "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"; - } - - # Temporary redirection to backend-FSN - location ~ ^/api/v1/(aggregation|measurements|raw_measurement|measurement_meta) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location ~ ^/api/_/(asn_by_month|countries|countries_by_month|check_report_id|country_overview|global_overview|global_overview_by_month|im_networks|im_stats|network_stats) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location ~ ^/api/_/(test_coverage|website_networks|website_stats|website_urls|vanilla_tor_stats|test_names) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location = /api/_/circumvention_stats_by_country { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location = / { - # match "/" strictly, not as a prefix - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location ~ ^/static/ { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - # open and close reports, submit msmt - location ~ ^/report/ { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - - # Auth, URL sumbission, URL priorities - location ~ ^/api/v1/(url-submission|get_account_role|set_account_role|set_session_expunge|user_login|user_register|user_logout) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location ~ ^/api/_/(url-priorities|account_metadata) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - - location ~ ^/api/v1/(collectors|test-helpers|torsf_stats) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location ~ ^/(robots.txt|files) { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location = /api/v1/test-list/tor-targets { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location = /api/v1/test-list/urls { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location = /bouncer/net-tests { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - location = /api/v1/test-list/psiphon-config { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - #location ~ ^/api/_/(test_names) { - # proxy_pass https://backend-fsn.ooni.org; - # proxy_set_header Host $host; - # proxy_set_header X-Real-IP $remote_addr; - #} - ## /files* tree - #location ~ ^/files { - # proxy_pass https://backend-fsn.ooni.org; - # proxy_set_header Host $host; - # proxy_set_header X-Real-IP $remote_addr; - #} - #location ~ ^/(health) { - # proxy_pass https://backend-fsn.ooni.org; - # proxy_set_header Host $host; - # proxy_set_header X-Real-IP $remote_addr; - #} - - # Temporary redirect - location = /api/v1/check-in { - proxy_pass https://backend-fsn.ooni.org; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - - # new API - location / { - proxy_pass http://127.0.0.1:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_cache apicache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - proxy_cache_methods HEAD GET; - # Cache only 200, 301, and 302 by default and for very short. - # Overridden by the API using the Expires header - proxy_cache_valid 200 301 302 10s; - proxy_cache_valid any 0; - add_header x-cache-status $upstream_cache_status; - add_header X-Cache-Status $upstream_cache_status; - } - - # Expose the measurement spool directory - location /measurement_spool/ { - alias /var/lib/ooniapi/measurements/incoming/; - autoindex off; - sendfile on; - tcp_nopush on; - if_modified_since off; - expires off; - etag off; - - gzip_comp_level 6; - gzip_min_length 1240; - gzip_proxied any; - gzip_types *; - gzip_vary on; - } -} diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index 9d6e145..1b8d300 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -210,12 +210,8 @@ server { } set $external_remote_addr $remote_addr; - if ($remote_addr = "188.166.93.143") { - # If remote_addr is ams-pg-test trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } - if ($remote_addr = "142.93.237.101") { - # If remote_addr is ams-pg trust the X-Real-IP header + if ($remote_addr ~ "^3.") { + # If remote_addr is AWS trust the X-Real-IP header set $external_remote_addr $http_x_real_ip; } proxy_set_header X-Real-IP $external_remote_addr; diff --git a/ansible/roles/ooni-backend/templates/nginx-api-test.conf b/ansible/roles/ooni-backend/templates/nginx-api-test.conf index 092d40d..a022f7c 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-test.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-test.conf @@ -99,14 +99,6 @@ server { proxy_set_header Host $host; set $external_remote_addr $remote_addr; - if ($remote_addr = "188.166.93.143") { - # If remote_addr is ams-pg-test trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } - if ($remote_addr = "142.93.237.101") { - # If remote_addr is ams-pg trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } proxy_set_header X-Real-IP $external_remote_addr; proxy_cache apicache; diff --git a/ansible/roles/ooni-backend/templates/rotation_nginx_conf b/ansible/roles/ooni-backend/templates/rotation_nginx_conf deleted file mode 100644 index 63255e5..0000000 --- a/ansible/roles/ooni-backend/templates/rotation_nginx_conf +++ /dev/null @@ -1,70 +0,0 @@ -# Managed by ansible, see roles/ooni-backend/tasks/main.yml -# and roles/ooni-backend/templates/rotation_nginx_conf -# Deployed by rotation tool to the test-helper hosts -proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M - max_size=5g inactive=24h use_temp_path=off; - -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - server_name _; - gzip on; - ssl_certificate /etc/ssl/private/th_fullchain.pem; - ssl_certificate_key /etc/ssl/private/th_privkey.pem; - ssl_session_timeout 5m; - ssl_session_cache shared:MozSSL:30m; - ssl_session_tickets off; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; - ssl_prefer_server_ciphers off; - add_header Strict-Transport-Security "max-age=63072000" always; - ssl_stapling on; - ssl_stapling_verify on; - resolver 127.0.0.1; - # local test helper - location / { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - proxy_pass http://127.0.0.1:8080; - - proxy_cache thcache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - # Cache POST without headers set by the test helper! - proxy_cache_methods POST; - proxy_cache_key "$request_uri|$request_body"; - proxy_cache_valid 200 10m; - proxy_cache_valid any 0; - add_header X-Cache-Status $upstream_cache_status; - - } -} - -# Used by Netdata to monitor Nginx -server { - listen 127.0.0.1:80; - server_name localhost; - - allow 5.9.112.244; # monitoring host - deny all; - - location = /stub_status { - stub_status; - } -} - -# Used by Prometheus to reach the TH -server { - listen 9001; - server_name localhost; - - allow 5.9.112.244; # monitoring host - deny all; - - location = /metrics { - proxy_pass http://127.0.0.1:9091; - } -} diff --git a/ansible/roles/ooni-backend/templates/rotation_setup.sh b/ansible/roles/ooni-backend/templates/rotation_setup.sh deleted file mode 100644 index 5706150..0000000 --- a/ansible/roles/ooni-backend/templates/rotation_setup.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -# Managed by ansible, see roles/ooni-backend/tasks/main.yml -# -# Configure test-helper droplet -# This script is run remotely on newly spawned VM by https://github.com/ooni/backend/blob/master/analysis/rotation.py -# It runs as root and with CWD=/ -# -set -euo pipefail -exec 1>/var/log/vm_rotation_setup.log 2>&1 -echo > /etc/motd - -echo "Configuring APT" -echo "deb [trusted=yes] https://ooni-deb.s3.eu-central-1.amazonaws.com unstable main" > /etc/apt/sources.list.d/ooni.list -cat < /etc/apt/trusted.gpg.d/ooni.gpg ------BEGIN PGP PUBLIC KEY BLOCK----- - -mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ -JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh -dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb -AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb -gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo -Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX -HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh -tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy -6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s -oN7jfdbznrWVigE= -=PtYb ------END PGP PUBLIC KEY BLOCK----- -EOF - -# Vector -cat < /etc/apt/trusted.gpg.d/vector.gpg ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v2 - -mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ -jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d -2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy -Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL -Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz -629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj -a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE -EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor -xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X -muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE -RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA -gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm -0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g -3vq0NvX0k2CLi/W9mTiSdHXFChI15A== -=k36w ------END PGP PUBLIC KEY BLOCK----- -EOF - -echo "deb https://repositories.timber.io/public/vector/deb/debian bullseye main" > /etc/apt/sources.list.d/vector.list - -echo "Installing packages" -export DEBIAN_FRONTEND=noninteractive -apt-get update -q -apt-get purge -qy unattended-upgrades rsyslog -apt-get upgrade -qy -apt-get install -qy --no-install-recommends chrony netdata oohelperd netdata-plugins-python - -systemctl daemon-reload -systemctl restart systemd-journald.service -logger start -systemctl restart systemd-journald.service - -apt-get install -qy --no-install-recommends vector - -echo "Configuring Vector" -# The certs are copied over by rotation.py -cat > /etc/vector/vector.toml < /etc/netdata/netdata.conf < /var/run/rotation_setup_completed diff --git a/ansible/roles/ooni-backend/templates/tor_targets.json b/ansible/roles/ooni-backend/templates/tor_targets.json index 933c4ed..66679bf 100644 --- a/ansible/roles/ooni-backend/templates/tor_targets.json +++ b/ansible/roles/ooni-backend/templates/tor_targets.json @@ -1,13 +1,13 @@ { - "128.31.0.39:9101": { - "address": "128.31.0.39:9101", - "fingerprint": "9695DFC35FFEB861329B9F1AB04C46397020CE31", + "128.31.0.39:9201": { + "address": "128.31.0.39:9201", + "fingerprint": "1A25C6358DB91342AA51720A5038B72742732498", "name": "moria1", "protocol": "or_port_dirauth" }, - "128.31.0.39:9131": { - "address": "128.31.0.39:9131", - "fingerprint": "9695DFC35FFEB861329B9F1AB04C46397020CE31", + "128.31.0.39:9231": { + "address": "128.31.0.39:9231", + "fingerprint": "1A25C6358DB91342AA51720A5038B72742732498", "name": "moria1", "protocol": "dir_port" }, @@ -23,15 +23,15 @@ "name": "gabelmoo", "protocol": "dir_port" }, - "154.35.175.225:443": { - "address": "154.35.175.225:443", - "fingerprint": "CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC", + "216.218.219.41:443": { + "address": "216.218.219.41:443", + "fingerprint": "E3E42D35F801C9D5AB23584E0025D56FE2B33396", "name": "Faravahar", "protocol": "or_port_dirauth" }, - "154.35.175.225:80": { - "address": "154.35.175.225:80", - "fingerprint": "CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC", + "216.218.219.41:80": { + "address": "216.218.219.41:80", + "fingerprint": "E3E42D35F801C9D5AB23584E0025D56FE2B33396", "name": "Faravahar", "protocol": "dir_port" }, @@ -109,14 +109,14 @@ }, "protocol": "obfs4" }, - "45.66.33.45:443": { - "address": "45.66.33.45:443", + "45.66.35.11:443": { + "address": "45.66.35.11:443", "fingerprint": "7EA6EAD6FD83083C538F44038BBFA077587DD755", "name": "dizum", "protocol": "or_port_dirauth" }, - "45.66.33.45:80": { - "address": "45.66.33.45:80", + "45.66.35.11:80": { + "address": "45.66.35.11:80", "fingerprint": "7EA6EAD6FD83083C538F44038BBFA077587DD755", "name": "dizum", "protocol": "dir_port" @@ -211,15 +211,15 @@ }, "protocol": "obfs4" }, - "86.59.21.38:443": { - "address": "86.59.21.38:443", - "fingerprint": "847B1F850344D7876491A54892F904934E4EB85D", + "217.196.147.77:443": { + "address": "217.196.147.77:443", + "fingerprint": "FAA4BCA4A6AC0FB4CA2F8AD5A11D9E122BA894F6", "name": "tor26", "protocol": "or_port_dirauth" }, - "86.59.21.38:80": { - "address": "86.59.21.38:80", - "fingerprint": "847B1F850344D7876491A54892F904934E4EB85D", + "217.196.147.77:80": { + "address": "217.196.147.77:80", + "fingerprint": "FAA4BCA4A6AC0FB4CA2F8AD5A11D9E122BA894F6", "name": "tor26", "protocol": "dir_port" }, @@ -301,4 +301,4 @@ }, "protocol": "obfs4" } -} \ No newline at end of file +} From 78019817d4122e979373d2b6159609d6b57f6b97 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 14:55:50 -0500 Subject: [PATCH 02/20] feat: add db configuration for backend-hel --- ansible/group_vars/clickhouse/vars.yml | 2 ++ ansible/host_vars/data3.htz-fsn.prod.ooni.nu | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index f1ac524..bd7f2d8 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -7,6 +7,8 @@ nftables_clickhouse_allow: ip: 168.119.7.188 - fqdn: notebook.ooni.org ip: 138.201.19.39 + - fqdn: backend-hel.ooni.org + ip: 65.108.192.151 - fqdn: clickhouseproxy.dev.ooni.io ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}" diff --git a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu index 4af3528..502a005 100644 --- a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu @@ -1,2 +1,4 @@ non_admin_usernames: [ ] clickhouse_base_path: /data/clickhouse +clickhouse_custom_databases: + - name: test \ No newline at end of file From 8f55d5d825d94f8bb23c8268bbd37985a22bb921 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 17:18:45 -0500 Subject: [PATCH 03/20] refactor: remove hostname based if blocks --- ansible/deploy-ooni-backend.yml | 39 ++++ ansible/roles/ooni-backend/defaults/main.yml | 11 ++ ansible/roles/ooni-backend/tasks/main.yml | 167 ++---------------- ansible/roles/ooni-backend/templates/444.nft | 2 - .../templates/deb_ooni_org.nginx.conf | 14 +- .../templates/nginx-api-test.conf | 149 ---------------- .../{nginx-api-fsn.conf => nginx-api.conf} | 36 ++-- 7 files changed, 95 insertions(+), 323 deletions(-) create mode 100644 ansible/roles/ooni-backend/defaults/main.yml delete mode 100644 ansible/roles/ooni-backend/templates/444.nft delete mode 100644 ansible/roles/ooni-backend/templates/nginx-api-test.conf rename ansible/roles/ooni-backend/templates/{nginx-api-fsn.conf => nginx-api.conf} (91%) diff --git a/ansible/deploy-ooni-backend.yml b/ansible/deploy-ooni-backend.yml index 24c70aa..0529e8d 100644 --- a/ansible/deploy-ooni-backend.yml +++ b/ansible/deploy-ooni-backend.yml @@ -19,3 +19,42 @@ - role: ooni-backend vars: ssl_domain: backend-hel.ooni.org + collector_id: 2 + clickhouse_url: "" # fetch from aws secrets + bucket_name: ooni-data-eu-fra-test + collectors: + - "backend-hel.ooni.org" + fastpath_version: 0.86~pr831-395 + analysis_version: 1.12~pr836-413 + deb_bucket_name: ooni-internal-deb + deb_server_name: deb-cli.ooni.org + +# - hosts: backend-fsn.ooni.org + # roles: + # - role: bootstrap + # - role: base-backend + # - role: nftables + # - role: nginx + # tags: nginx + # vars: + # nginx_user: "www-data" + # - role: dehydrated + # tags: dehydrated + # expand: yes + # vars: + # ssl_domains: + # # with dehydrated the first entry is the cert FQDN + # # and the other ones are alternative names + # - "backend-hel.ooni.org" + # - role: ooni-backend + # vars: + # ssl_domain: backend-fsn.ooni.org + # collector_id: 1 + # clickhouse_url: "" # fetch from aws secrets + # bucket_name: ooni-data-eu-fra + # collectors: + # - "backend-fsn.ooni.org" + # fastpath_version: 0.86~pr831-395 + # analysis_version: 1.12~pr836-413 + # deb_bucket_name: ooni.deb + # deb_server_name: deb.ooni.org diff --git a/ansible/roles/ooni-backend/defaults/main.yml b/ansible/roles/ooni-backend/defaults/main.yml new file mode 100644 index 0000000..8413d7e --- /dev/null +++ b/ansible/roles/ooni-backend/defaults/main.yml @@ -0,0 +1,11 @@ +--- +ssl_domain: backend-hel.ooni.org +collector_id: 2 +clickhouse_url: "" # fetch from aws secrets +bucket_name: ooni-data-eu-fra-test +collectors: + - "backend-hel.ooni.org" +fastpath_version: 0.86~pr831-395 +analysis_version: 1.12~pr836-413 +deb_bucket_name: ooni-internal-deb +deb_server_name: deb-ci.ooni.org \ No newline at end of file diff --git a/ansible/roles/ooni-backend/tasks/main.yml b/ansible/roles/ooni-backend/tasks/main.yml index 42da375..987eecf 100644 --- a/ansible/roles/ooni-backend/tasks/main.yml +++ b/ansible/roles/ooni-backend/tasks/main.yml @@ -16,8 +16,7 @@ path: /var/cache/nginx/ooni-api state: directory -- name: configure test api - when: inventory_hostname == 'backend-hel.ooni.org' +- name: configure api tags: api template: src: api.conf @@ -26,10 +25,10 @@ group: ooniapi mode: 0640 vars: - collectors: ['backend-hel.ooni.org'] + collectors: {{ collectors }} # bucket_name and collector_id must match the uploader - collector_id: 2 - bucket_name: ooni-data-eu-fra-test + collector_id: {{ collector_id }} + bucket_name: {{ bucket_name }} github_push_repo: "ooni-bot/test-lists" github_origin_repo: "ooni/test-lists" login_base_url: "https://test-lists.test.ooni.org/login" @@ -38,27 +37,6 @@ # mail_smtp_password: "DISABLED" # jwt_encryption_key and account_id_hashing_key are taken from the vault -- name: configure backend-fsn api - when: inventory_hostname == 'backend-fsn.ooni.org' - tags: api - template: - src: api.conf - dest: /etc/ooni/api.conf - owner: ooniapi - group: ooniapi - mode: 0640 - vars: - collectors: ['backend-fsn.ooni.org'] - # bucket_name and collector_id must match the uploader - collector_id: 1 - bucket_name: ooni-data-eu-fra - github_push_repo: "ooni/test-lists" - github_origin_repo: "citizenlab/test-lists" - login_base_url: "https://test-lists.ooni.org/login" - pg_uri: "" - clickhouse_url: clickhouse://api:api@localhost/default - base_url: "https://api.ooni.io" - - name: create Psiphon conffile tags: api copy: @@ -71,35 +49,22 @@ src: tor_targets.json dest: /etc/ooni/tor_targets.json -- name: configure api uploader using test bucket - when: inventory_hostname == 'backend-hel.ooni.org' - tags: api - template: - src: templates/api-uploader.conf - dest: /etc/ooni/api-uploader.conf - vars: - # bucket_name and collector_id must match the API - bucket_name: ooni-data-eu-fra-test - collector_id: 2 - -- name: configure FSN api uploader using PROD bucket - when: inventory_hostname == 'backend-fsn.ooni.org' +- name: configure api uploader using s3 bucket tags: api template: src: templates/api-uploader.conf dest: /etc/ooni/api-uploader.conf vars: # bucket_name and collector_id must match the API - bucket_name: ooni-data-eu-fra - collector_id: 1 + bucket_name: {{ bucket_name }} + collector_id: {{ collector_id }} -## Haproxy and nginx ## +## nginx ## -- name: Overwrite API nginx test conf - when: inventory_hostname == 'backend-hel.ooni.org' +- name: Overwrite API nginx conf tags: api, webserv template: - src: templates/nginx-api-test.conf + src: templates/nginx-api.conf dest: /etc/nginx/sites-available/ooni-api.conf mode: 0755 owner: root @@ -107,26 +72,6 @@ # Uses dehydrated certpath: /var/lib/dehydrated/certs/ -- name: install haproxy if not present - when: inventory_hostname in ('backend-hel.ooni.org') - tags: webserv - apt: - cache_valid_time: 86400 - name: haproxy - state: present - -- name: Deploy haproxy conf - when: inventory_hostname in ('backend-hel.ooni.org') - tags: api, webserv - template: - src: templates/haproxy.cfg - dest: /etc/haproxy/haproxy.cfg - mode: 0755 - owner: root - vars: - # Uses dehydrated - certpath: /var/lib/dehydrated/certs/ - - name: Delete old files when: inventory_hostname in ('backend-hel.ooni.org') tags: api, webserv @@ -156,27 +101,6 @@ mode: 0755 owner: root -- name: Deploy dehydrated haproxy hook - when: inventory_hostname in ('backend-hel.ooni.org') - tags: api, webserv - template: - src: templates/dehydrated_haproxy_hook.sh - dest: /etc/dehydrated/haproxy_hook.sh - mode: 0755 - owner: root - -- name: Overwrite API nginx FSN conf - when: inventory_hostname == 'backend-fsn.ooni.org' - tags: api, webserv - template: - src: templates/nginx-api-fsn.conf - dest: /etc/nginx/sites-available/ooni-api.conf - mode: 0755 - owner: root - vars: - # Uses dehydrated - certpath: /var/lib/dehydrated/certs/ - - name: Deploy API gunicorn conf tags: api template: @@ -193,30 +117,13 @@ dest=/etc/nginx/sites-enabled/ooni-api.conf state=link -- name: Configure deb.ooni.org forwarder on FSN host - when: inventory_hostname in ('backend-fsn.ooni.org', ) +- name: Configure deb forwarder tags: deb_ooni_org # Uses dehydrated template: src: deb_ooni_org.nginx.conf dest: /etc/nginx/sites-enabled/deb_ooni_org -- name: Configure deb-ci.ooni.org forwarder on test host - when: inventory_hostname == 'backend-hel.ooni.org' - tags: deb_ooni_org - blockinfile: - path: /etc/nginx/sites-enabled/deb_ooni_org_http - create: yes - block: | - # Managed by ansible, see roles/ooni-backend/tasks/main.yml - server { - listen 80; - server_name deb-ci.ooni.org; - location / { - proxy_pass https://ooni-internal-deb.s3.eu-central-1.amazonaws.com/; - } - } - - name: create badges dir tags: api file: @@ -224,7 +131,6 @@ state: directory - name: Safely reload Nginx - # TODO remove restart after transition to haproxy tags: api, deb_ooni_org, webserv shell: nginx -t && systemctl reload nginx @@ -232,12 +138,6 @@ tags: webserv shell: nginx -t && systemctl restart nginx -- name: Restart haproxy - # reload is not enough - when: inventory_hostname in ('backend-hel.ooni.org') - tags: api, deb_ooni_org, webserv - shell: systemctl restart haproxy - - name: allow incoming TCP connections to API tags: api blockinfile: @@ -246,12 +146,6 @@ block: | add rule inet filter input tcp dport 443 counter accept comment "incoming HTTPS" -- name: allow incoming TCP connections to haproxy metrics - tags: webserv - template: - src: 444.nft - dest: /etc/ooni/nftables/tcp/444.nft - - name: reload nftables service tags: api, webserv shell: systemctl reload nftables.service @@ -259,28 +153,13 @@ ## Fastpath ## - name: install fastpath if not present - # do not update package if present - when: inventory_hostname != 'backend-fsn.ooni.org' tags: fastpath apt: cache_valid_time: 86400 - name: fastpath + name: "fastpath={{ fastpath_version }}" state: present - name: configure fastpath on test - when: inventory_hostname == 'backend-hel.ooni.org' - tags: fastpath - template: - src: fastpath.conf - dest: /etc/ooni/fastpath.conf - owner: fastpath - group: fastpath - mode: 0640 - vars: - clickhouse_url: clickhouse://fastpath:fastpath@localhost/default - -- name: configure fastpath on FSN - when: inventory_hostname == 'backend-fsn.ooni.org' tags: fastpath template: src: fastpath.conf @@ -289,17 +168,16 @@ group: fastpath mode: 0640 vars: - clickhouse_url: clickhouse://fastpath:fastpath@localhost/default + clickhouse_url: {{ clickhouse_url }} ## Analysis daemon ## - name: install analysis # do not update package if present - when: inventory_hostname != 'backend-fsn.ooni.org' tags: analysis apt: cache_valid_time: 86400 - name: analysis=1.4~pr408-209 + name: "analysis={{ analysis_version }}" force: True state: present @@ -308,24 +186,15 @@ template: src: analysis.conf dest: /etc/ooni/analysis.conf - # Managed by ansible, see roles/ooni-backend/tasks/main.yml -- name: Run DB backup on backend-hel - when: inventory_hostname == 'backend-hel.ooni.org' - tags: dbbackup - template: - src: db-backup.conf - dest: /etc/ooni/db-backup.conf - mode: 0600 - vars: - public_bucket_name: ooni-data-eu-fra-test +## DB backup ## -- name: Run DB backup on FSN - when: inventory_hostname == 'backend-fsn.ooni.org' +- name: Run DB backup + when: inventory_hostname == 'backend-hel.ooni.org' tags: dbbackup template: src: db-backup.conf dest: /etc/ooni/db-backup.conf mode: 0600 vars: - public_bucket_name: ooni-data-eu-fra + public_bucket_name: {{ bucket_name }} diff --git a/ansible/roles/ooni-backend/templates/444.nft b/ansible/roles/ooni-backend/templates/444.nft deleted file mode 100644 index 03f5106..0000000 --- a/ansible/roles/ooni-backend/templates/444.nft +++ /dev/null @@ -1,2 +0,0 @@ -# roles/ooni-backend/templates/444.nft -add rule inet filter input tcp dport 444 counter accept comment "incoming haproxy metrics" diff --git a/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf b/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf index c069fd5..d3da8db 100644 --- a/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf +++ b/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf @@ -8,13 +8,13 @@ map $remote_addr $remote_addr_anon { } # log anonymized ipaddr -log_format deb_ooni_org_logfmt '$remote_addr_anon [$time_local] ' +log_format deb_logfmt '$remote_addr_anon [$time_local] ' '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; server { listen 80; - server_name deb.ooni.org; - access_log syslog:server=unix:/dev/log,severity=info deb_ooni_org_logfmt; + server_name {{ deb_server_name }}; + access_log syslog:server=unix:/dev/log,severity=info deb_logfmt; error_log syslog:server=unix:/dev/log,severity=info; gzip on; resolver 127.0.0.1; @@ -23,15 +23,15 @@ server { alias /var/lib/dehydrated/acme-challenges; } location / { - proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + proxy_pass https://{{ deb_bucket_name }}.s3.eu-central-1.amazonaws.com/; } } server { listen 443 ssl http2; listen [::]:443 ssl http2; - server_name deb.ooni.org; - access_log syslog:server=unix:/dev/log,severity=info deb_ooni_org_logfmt; + server_name {{ deb_server_name }}; + access_log syslog:server=unix:/dev/log,severity=info deb_logfmt; error_log syslog:server=unix:/dev/log,severity=info; gzip on; ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; @@ -59,6 +59,6 @@ server { resolver 127.0.0.1; location / { - proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + proxy_pass https://{{ deb_bucket_name }}.s3.eu-central-1.amazonaws.com/; } } diff --git a/ansible/roles/ooni-backend/templates/nginx-api-test.conf b/ansible/roles/ooni-backend/templates/nginx-api-test.conf deleted file mode 100644 index a022f7c..0000000 --- a/ansible/roles/ooni-backend/templates/nginx-api-test.conf +++ /dev/null @@ -1,149 +0,0 @@ -# Managed by ansible -# roles/ooni-backend/templates/nginx-api-test.conf - -# Use 2-level cache, 20MB of RAM + 5GB on disk, -proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M - max_size=5g inactive=24h use_temp_path=off; - -# anonymize ipaddr -map $remote_addr $remote_addr_anon { - ~(?P\d+\.\d+\.\d+)\. $ip.0; - ~(?P[^:]+:[^:]+): $ip::; - default 0.0.0.0; -} - -# anonymize forwarded ipaddr -map $http_x_forwarded_for $remote_fwd_anon { - ~(?P\d+\.\d+\.\d+)\. $ip.0; - ~(?P[^:]+:[^:]+): $ip::; - default 0.0.0.0; -} - - -# log anonymized ipaddr and caching status -log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' - '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; - -server { - # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests - # over Tor correctly land to the proper backend. We are listening on this custom port - # and we are configuring Tor such that it routes traffic to this port. - listen 127.0.0.1:17744; - server_name _; - access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; - gzip on; - gzip_types text/plain application/xml application/json; - - # TODO: we could use different client_max_body_size and SSL configurations for probe service paths - # and everyhing else - client_max_body_size 200M; # for measurement POST - - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header X-Frame-Options DENY always; - add_header X-Content-Type-Options nosniff always; - - # use systemd-resolved - resolver 127.0.0.53; - - # Selectively route test-list/urls to the API - location ~^/api/v1/test-list/urls { - proxy_pass http://127.0.0.1:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_cache apicache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - proxy_cache_methods HEAD GET; - # Cache only 200, 301, and 302 by default and for very short. - # Overridden by the API using the Expires header - proxy_cache_valid 200 301 302 10s; - proxy_cache_valid any 0; - add_header x-cache-status $upstream_cache_status; - add_header X-Cache-Status $upstream_cache_status; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header X-Frame-Options DENY always; - add_header X-Content-Type-Options nosniff always; - } - - location /whoami { - return 200 "{{ inventory_hostname }}"; - } - - # Serve ACME challenge from disk - location ^~ /.well-known/acme-challenge { - alias /var/lib/dehydrated/acme-challenges; - } - - # 2022-09-01 20:08 CEST temporarily block a bot scraping /files/download/* - location ~^/files/download/ { - return 301 https://explorer.ooni.org/; - } - - # new API - location / { - - # Protect /apidocs invoked with url= and/or urls= args - if ($uri ~ "^/apidocs") { set $block_apidocs X; } - if ($args ~ "url=" ) { set $block_apidocs "${block_apidocs}Y"; } - if ($args ~ "urls=" ) { set $block_apidocs "${block_apidocs}Y"; } - if ($block_apidocs ~ "XY") { return 403; } # nested "if" are not supported - - deny 216.244.66.0/24; # DotBot/1.2 - deny 114.119.128.0/19; # PetalBot - allow all; - proxy_pass http://127.0.0.1:8000; - proxy_set_header Host $host; - - set $external_remote_addr $remote_addr; - proxy_set_header X-Real-IP $external_remote_addr; - - proxy_cache apicache; - proxy_cache_min_uses 1; - proxy_cache_lock on; - proxy_cache_lock_timeout 30; - proxy_cache_lock_age 30; - proxy_cache_use_stale error timeout invalid_header updating; - proxy_cache_methods HEAD GET; - # Cache only 200, 301, and 302 by default and for very short. - # Overridden by the API using the Expires header - proxy_cache_valid 200 301 302 10s; - proxy_cache_valid any 0; - add_header x-cache-status $upstream_cache_status; - add_header X-Cache-Status $upstream_cache_status; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header X-Frame-Options DENY always; - add_header X-Content-Type-Options nosniff always; - } - - # Expose the measurement spool directory - location /measurement_spool/ { - alias /var/lib/ooniapi/measurements/incoming/; - autoindex off; - sendfile on; - tcp_nopush on; - if_modified_since off; - expires off; - etag off; - } -} - -server { - # Forward deb.ooni.org to S3 - listen 17744; - server_name deb.ooni.org; - access_log syslog:server=unix:/dev/log,severity=info ooni_api_fmt; - error_log syslog:server=unix:/dev/log,severity=info; - gzip on; - resolver 127.0.0.53; - # Serve ACME challenge from disk - location ^~ /.well-known/acme-challenge { - alias /var/lib/dehydrated/acme-challenges; - } - location / { - proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; - } -} diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api.conf similarity index 91% rename from ansible/roles/ooni-backend/templates/nginx-api-fsn.conf rename to ansible/roles/ooni-backend/templates/nginx-api.conf index 1b8d300..6e43374 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api.conf @@ -1,5 +1,5 @@ # Managed by ansible -# roles/ooni-backend/templates/nginx-api-fsn.conf +# roles/ooni-backend/templates/nginx-api.conf # Use 2-level cache, 20MB of RAM + 5GB on disk, proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M @@ -67,8 +67,6 @@ server { # verify chain of trust of OCSP response using Root CA and Intermediate certs #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; - resolver 127.0.0.1; - # Registry # Should match: # - /api/v1/login @@ -136,24 +134,30 @@ server { # TODO We should check if clients will respect a suffix added to by the # bouncer in the returned field, otherwise new clients should use another # form - location ~^/web-connectivity/(status) { - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - - proxy_pass https://wcth.ooni.io; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header X-Frame-Options DENY always; - add_header X-Content-Type-Options nosniff always; - } + # NOTE: we killed wcth.ooni.io so this return nothing essentially + # location ~^/web-connectivity/(status) { + # proxy_http_version 1.1; + # proxy_set_header Host $http_host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # proxy_read_timeout 900; + + # proxy_pass https://wcth.ooni.io; + # add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + # add_header X-Frame-Options DENY always; + # add_header X-Content-Type-Options nosniff always; + # } location /whoami { return 200 "{{ inventory_hostname }}"; } + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + location /metrics { return 200 ''; } From e86bb9c12085c86bdaf2ce774a2b86b6bf302073 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 17:38:09 -0500 Subject: [PATCH 04/20] set host vars from ansible vault --- ansible/host_vars/backend-fsn.ooni.org | 10 ++++++++++ ansible/host_vars/backend-hel.ooni.org | 8 ++++++++ ansible/roles/ooni-backend/templates/dehydrated.config | 1 - ansible/roles/ooni-backend/templates/nginx-api.conf | 5 ----- 4 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 ansible/host_vars/backend-fsn.ooni.org create mode 100644 ansible/host_vars/backend-hel.ooni.org diff --git a/ansible/host_vars/backend-fsn.ooni.org b/ansible/host_vars/backend-fsn.ooni.org new file mode 100644 index 0000000..2cfbcab --- /dev/null +++ b/ansible/host_vars/backend-fsn.ooni.org @@ -0,0 +1,10 @@ +s3_ooni_open_data_access_key: "{{ vault_s3_ooni_open_data_access_key }}" +amspg_ooni_org_onion_key: "{{ vault_amspg_ooni_org_onion_key }}" +jwt_encryption_key: "{{ vault_jwt_encryption_key }}" +account_id_hashing_key: "{{ vault_account_id_hashing_key }}" +github_token: "{{ vault_github_token }}" +mail_smtp_password: "{{ vault_mail_smtp_password }}" +base_url: "https://backend-fsn.ooni.org" +tor_targets: "{{ vault_tor_targets }}" +psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}" +digital_ocean_token: "{{ vault_digital_ocean_token }}" diff --git a/ansible/host_vars/backend-hel.ooni.org b/ansible/host_vars/backend-hel.ooni.org new file mode 100644 index 0000000..1442096 --- /dev/null +++ b/ansible/host_vars/backend-hel.ooni.org @@ -0,0 +1,8 @@ +s3_ooni_open_data_access_key: "{{ vault_s3_ooni_open_data_access_key }}" +jwt_encryption_key: "{{ vault_jwt_encryption_key }}" +account_id_hashing_key: "{{ vault_account_id_hashing_key }}" +github_token: "{{ vault_github_token }}" +mail_smtp_password: "{{ vault_mail_smtp_password }}" +base_url: "https://backend-hel.ooni.org" +tor_targets: "{{ vault_tor_targets }}" +psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}" diff --git a/ansible/roles/ooni-backend/templates/dehydrated.config b/ansible/roles/ooni-backend/templates/dehydrated.config index 7a0293a..ec65176 100644 --- a/ansible/roles/ooni-backend/templates/dehydrated.config +++ b/ansible/roles/ooni-backend/templates/dehydrated.config @@ -4,4 +4,3 @@ CONFIG_D=/etc/dehydrated/conf.d BASEDIR=/var/lib/dehydrated WELLKNOWN="${BASEDIR}/acme-challenges" DOMAINS_TXT="/etc/dehydrated/domains.txt" -HOOK="/etc/dehydrated/haproxy_hook.sh" diff --git a/ansible/roles/ooni-backend/templates/nginx-api.conf b/ansible/roles/ooni-backend/templates/nginx-api.conf index 6e43374..ebeaf57 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api.conf @@ -153,11 +153,6 @@ server { return 200 "{{ inventory_hostname }}"; } - # Serve ACME challenge from disk - location ^~ /.well-known/acme-challenge { - alias /var/lib/dehydrated/acme-challenges; - } - location /metrics { return 200 ''; } From 7ea57786b229aecd00e853ae1d50c8159d24c8f4 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 17:54:25 -0500 Subject: [PATCH 05/20] refactor: consolidate all conditionals to generic --- ansible/roles/ooni-backend/handlers/main.yml | 20 ++- ansible/roles/ooni-backend/tasks/main.yml | 63 +++------ .../templates/dehydrated_haproxy_hook.sh | 23 ---- .../roles/ooni-backend/templates/haproxy.cfg | 122 ------------------ 4 files changed, 35 insertions(+), 193 deletions(-) delete mode 100644 ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh delete mode 100644 ansible/roles/ooni-backend/templates/haproxy.cfg diff --git a/ansible/roles/ooni-backend/handlers/main.yml b/ansible/roles/ooni-backend/handlers/main.yml index 84d0f4f..f91f47b 100644 --- a/ansible/roles/ooni-backend/handlers/main.yml +++ b/ansible/roles/ooni-backend/handlers/main.yml @@ -1,6 +1,22 @@ --- - name: reload nftables + tags: api, webserv service: name=nftables state=reloaded -- name: restart clickhouse - service: name=clickhouse-server state=restarted +- name: test nginx config + command: nginx -t + listen: + - restart nginx + - reload nginx + +- name: restart nginx + tags: webserv + service: + name: nginx + state: restarted + +- name: reload nginx + tags: api, deb_ooni, webserv + service: + name: nginx + state: reloaded diff --git a/ansible/roles/ooni-backend/tasks/main.yml b/ansible/roles/ooni-backend/tasks/main.yml index 987eecf..63ee471 100644 --- a/ansible/roles/ooni-backend/tasks/main.yml +++ b/ansible/roles/ooni-backend/tasks/main.yml @@ -59,7 +59,7 @@ bucket_name: {{ bucket_name }} collector_id: {{ collector_id }} -## nginx ## +## Nginx ## - name: Overwrite API nginx conf tags: api, webserv @@ -72,28 +72,24 @@ # Uses dehydrated certpath: /var/lib/dehydrated/certs/ -- name: Delete old files - when: inventory_hostname in ('backend-hel.ooni.org') - tags: api, webserv - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: - - /etc/nginx/sites-enabled/00-letsencrypt-http - - /etc/nginx/sites-enabled/deb_ooni_org - - /etc/nginx/sites-enabled/deb_ooni_org_http +- name: Create symlink for API nginx conf + tags: api + file: + src=/etc/nginx/sites-available/ooni-api.conf + dest=/etc/nginx/sites-enabled/ooni-api.conf + state=link -- name: Deploy dehydrated conf - when: inventory_hostname in ('backend-hel.ooni.org') - tags: api, webserv +- name: Configure deb forwarder + tags: deb_ooni + # Uses dehydrated template: - src: templates/dehydrated.config - dest: /etc/dehydrated/config - mode: 0755 - owner: root + src: deb_ooni.nginx.conf + dest: /etc/nginx/sites-enabled/deb_ooni + notify: + - reload nginx + - restart nginx - name: Deploy dehydrated conf - when: inventory_hostname in ('backend-hel.ooni.org') tags: api, webserv template: src: templates/dehydrated.config @@ -110,34 +106,12 @@ group: ooniapi mode: 0640 -- name: Create symlink for API nginx conf - tags: api - file: - src=/etc/nginx/sites-available/ooni-api.conf - dest=/etc/nginx/sites-enabled/ooni-api.conf - state=link - -- name: Configure deb forwarder - tags: deb_ooni_org - # Uses dehydrated - template: - src: deb_ooni_org.nginx.conf - dest: /etc/nginx/sites-enabled/deb_ooni_org - - name: create badges dir tags: api file: path: /var/www/package_badges/ state: directory -- name: Safely reload Nginx - tags: api, deb_ooni_org, webserv - shell: nginx -t && systemctl reload nginx - -- name: Restart Nginx - tags: webserv - shell: nginx -t && systemctl restart nginx - - name: allow incoming TCP connections to API tags: api blockinfile: @@ -145,10 +119,8 @@ create: yes block: | add rule inet filter input tcp dport 443 counter accept comment "incoming HTTPS" - -- name: reload nftables service - tags: api, webserv - shell: systemctl reload nftables.service + notify: + - reload nftables ## Fastpath ## @@ -190,7 +162,6 @@ ## DB backup ## - name: Run DB backup - when: inventory_hostname == 'backend-hel.ooni.org' tags: dbbackup template: src: db-backup.conf diff --git a/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh b/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh deleted file mode 100644 index 0e5b41f..0000000 --- a/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Deployed by ansible -# See roles/ooni-backend/templates/dehydrated_haproxy_hook.sh -# -# Deploys chained privkey and certificates for haproxy -# Reloads haproxy as needed - -deploy_cert() { - local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" - # Called once for each certificate - # /var/lib/dehydrated/certs/backend-hel.ooni.org/privkey.pem /var/lib/dehydrated/certs/backend-hel.ooni.org/cert.pem /var/lib/dehydrated/certs/backend-hel.ooni.org/fullchain.pem > /var/lib/dehydrated/certs/backend-hel.ooni.org/haproxy.pem - # cp "${KEYFILE}" "${FULLCHAINFILE}" /etc/nginx/ssl/; chown -R nginx: /etc/nginx/ssl - logger "deploy_cert hook reading ${KEYFILE} ${CERTFILE} ${FULLCHAINFILE}" - cat "${KEYFILE}" "${CERTFILE}" "${FULLCHAINFILE}" > "${KEYFILE}.haproxy" - logger "deploy_cert reloading haproxy" - systemctl reload haproxy.service -} - -HANDLER="$1"; shift -if [[ "${HANDLER}" =~ ^(deploy_cert)$ ]]; then - "$HANDLER" "$@" -fi diff --git a/ansible/roles/ooni-backend/templates/haproxy.cfg b/ansible/roles/ooni-backend/templates/haproxy.cfg deleted file mode 100644 index 025a4fc..0000000 --- a/ansible/roles/ooni-backend/templates/haproxy.cfg +++ /dev/null @@ -1,122 +0,0 @@ -## Deployed by ansible, see roles/ooni-backend/templates/haproxy.cfg - -# Proxies to: -# - local nginx -# - remote test helpers -# See http://interactive.blockdiag.com/?compression=deflate&src=eJyFjjELwjAQhXd_xeFuEdpBEAURBwfBXSSk6ZkEr7mSZGgR_7tNXdoiuD2--7j3SmL1rKzU8FoAFEUOqz0Y2XhuuxSHICKLiCEKg9Sg3_bmSHHaujaxISRyuJ7hRrJEgh0slVTGOr28Txz2yvQvvYw44R617XGXMTubWU7HzXq26kfl8XISykgidBphVP-whLPuOtRRhIaZ_ogVlt8d7PVYDXkS3x_pgmPP - -global - log /dev/log local0 info alert - log /dev/log local1 notice alert - chroot /var/lib/haproxy - stats socket /run/haproxy/admin.sock mode 660 level admin - stats timeout 30s - user haproxy - group haproxy - daemon - - # Default SSL material locations - ca-base /etc/ssl/certs - crt-base /etc/ssl/private - - # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate - ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 - ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 - ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets - -defaults - log global - mode http - option httplog - option dontlognull - timeout connect 5000 - timeout client 50000 - timeout server 50000 - errorfile 400 /etc/haproxy/errors/400.http - errorfile 403 /etc/haproxy/errors/403.http - errorfile 408 /etc/haproxy/errors/408.http - errorfile 500 /etc/haproxy/errors/500.http - errorfile 502 /etc/haproxy/errors/502.http - errorfile 503 /etc/haproxy/errors/503.http - errorfile 504 /etc/haproxy/errors/504.http - - log-format "%[var(txn.src_ipaddr_masked)] %ft > %b > %s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" - -frontend haproxy_metrics - # Metrics exposed on TLS port 444 - # File generated by /etc/dehydrated/haproxy_hook.sh - bind :444 ssl crt /var/lib/dehydrated/certs/"{{ inventory_hostname }}"/privkey.pem.haproxy - - http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) - - # /__haproxy_stats stats page - stats enable - stats uri /__haproxy_stats - stats refresh 5s - - # /__haproxy_prom_metrics prometheus metrics - http-request use-service prometheus-exporter if { path /__haproxy_prom_metrics } - - -frontend public_tls - # TLS on port 443 - # File generated by /etc/dehydrated/haproxy_hook.sh - bind :443 ssl crt /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem.haproxy - - http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) - - # test helpers - default_backend lb_test_helpers - - # deb.ooni.org - acl ACL_deb_ooni_org hdr(host) -i deb.ooni.org - use_backend deb_ooni_org if ACL_deb_ooni_org - - # Nginx - use_backend nginx if !{ path / } || !{ method POST } - - -frontend public_80 - # Forwarded to Nginx for ACME and deb.ooni.org - bind :80 - - http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) - - # ACME - use_backend nginx if { path_beg /.well-known/acme-challenge } - - # deb.ooni.org - acl ACL_deb_ooni_org hdr(host) -i deb.ooni.org - use_backend deb_ooni_org if ACL_deb_ooni_org - - - -backend nginx - # Local Nginx is in front of the API and more. See diagram. - default-server check - option forwardfor - #option httpchk GET / - # forward to local nginx - server nginx localhost:17744 - - -backend lb_test_helpers - # Remote testn helpers - default-server check - option forwardfor - http-check send meth POST uri / hdr Content-Type application/json body "{}" - http-check send-state - http-check comment "TH POST with empty JSON" - - server th0 0.th.ooni.org:443 ssl verify none - server th1 1.th.ooni.org:443 ssl verify none - server th2 2.th.ooni.org:443 ssl verify none - server th3 3.th.ooni.org:443 ssl verify none - #option httpchk - - -backend deb_ooni_org - #default-server check - option forwardfor - server s3-ooni-deb ooni-deb.s3.eu-central-1.amazonaws.com ssl verify none - From 118a3ab20d294f8fec83ebdcb7269d6226874408 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 18:18:29 -0500 Subject: [PATCH 06/20] refactor: configure ec2 machines from ansible --- ansible/deploy-clickhouse-proxy.yml | 11 ++++ .../roles/clickhouse_proxy/handlers/main.yml | 15 +++++ ansible/roles/clickhouse_proxy/tasks/main.yml | 11 ++++ .../clickhouse_proxy/templates/99-stream.conf | 13 +++++ .../templates/cloud-init.yml | 58 ------------------- 5 files changed, 50 insertions(+), 58 deletions(-) create mode 100644 ansible/deploy-clickhouse-proxy.yml create mode 100644 ansible/roles/clickhouse_proxy/handlers/main.yml create mode 100644 ansible/roles/clickhouse_proxy/tasks/main.yml create mode 100644 ansible/roles/clickhouse_proxy/templates/99-stream.conf diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml new file mode 100644 index 0000000..6d73c82 --- /dev/null +++ b/ansible/deploy-clickhouse-proxy.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy clickhouse proxy + hosts: + - clickhouseproxy.dev.ooni.io + roles: + - role: bootstrap + - role: nginx + tags: nginx + - role: clickhouse_proxy + vars: + clickhouse_url: "" # fetch from AWS secrets \ No newline at end of file diff --git a/ansible/roles/clickhouse_proxy/handlers/main.yml b/ansible/roles/clickhouse_proxy/handlers/main.yml new file mode 100644 index 0000000..eb1d167 --- /dev/null +++ b/ansible/roles/clickhouse_proxy/handlers/main.yml @@ -0,0 +1,15 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/clickhouse_proxy/tasks/main.yml b/ansible/roles/clickhouse_proxy/tasks/main.yml new file mode 100644 index 0000000..fb6ea49 --- /dev/null +++ b/ansible/roles/clickhouse_proxy/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Add stream nginx config + tags: webserv + template: + src: templates/99-stream.conf + dest: /etc/nginx/modules-enabled/99-stream.conf + mode: 0755 + owner: root + notify: + - reload nginx + - restart nginx \ No newline at end of file diff --git a/ansible/roles/clickhouse_proxy/templates/99-stream.conf b/ansible/roles/clickhouse_proxy/templates/99-stream.conf new file mode 100644 index 0000000..1a9000f --- /dev/null +++ b/ansible/roles/clickhouse_proxy/templates/99-stream.conf @@ -0,0 +1,13 @@ +stream { + upstream clickhouse_backend { + server {{ clickhouse_url }}; + } + + server { + listen 9000; + + proxy_pass clickhouse_backend; + } + + error_log /var/log/nginx/error.log; +} \ No newline at end of file diff --git a/tf/modules/ooni_backendproxy/templates/cloud-init.yml b/tf/modules/ooni_backendproxy/templates/cloud-init.yml index 4966322..e69de29 100644 --- a/tf/modules/ooni_backendproxy/templates/cloud-init.yml +++ b/tf/modules/ooni_backendproxy/templates/cloud-init.yml @@ -1,58 +0,0 @@ -package_update: true - -packages: - - nginx - - libnginx-mod-stream - -write_files: - - path: /etc/nginx/sites-available/default - content: | - server { - listen 80; - - server_name _; - - location / { - proxy_pass ${backend_url}; - proxy_http_version 1.1; - proxy_set_header Host \$host; - } - error_log /var/log/nginx/error.log; - } - - %{ if length(wcth_addresses) > 0 } - upstream wcths { - %{ for address in wcth_addresses } - server ${ address }; - %{ endfor } - } - server { - server_name *.${ wcth_domain_suffix }; - listen 80; - - location / { - proxy_pass http://wcths; - proxy_http_version 1.1; - proxy_set_header Host \$host; - } - } - %{ endif } - - - path: /etc/nginx/modules-enabled/99-stream.conf - content: | - stream { - upstream clickhouse_backend { - server ${clickhouse_url}:${clickhouse_port}; - } - - server { - listen 9000; - - proxy_pass clickhouse_backend; - } - - error_log /var/log/nginx/error.log; - } - -runcmd: - - service nginx restart From a6d684317184ae9d8e0268f85dda6f148bec6fa5 Mon Sep 17 00:00:00 2001 From: decfox Date: Sat, 7 Dec 2024 18:47:44 -0500 Subject: [PATCH 07/20] feat: init common module for ec2 machines --- ansible/deploy-tier0.yml | 3 + tf/modules/ec2/main.tf | 124 ++++++++++++++++++++++++ tf/modules/ec2/outputs.tf | 7 ++ tf/modules/ec2/templates/cloud-init.yml | 0 tf/modules/ec2/variables.tf | 45 +++++++++ tf/modules/ooni_backendproxy/main.tf | 1 - 6 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 tf/modules/ec2/main.tf create mode 100644 tf/modules/ec2/outputs.tf create mode 100644 tf/modules/ec2/templates/cloud-init.yml create mode 100644 tf/modules/ec2/variables.tf diff --git a/ansible/deploy-tier0.yml b/ansible/deploy-tier0.yml index 7c11a8c..d24488a 100644 --- a/ansible/deploy-tier0.yml +++ b/ansible/deploy-tier0.yml @@ -5,6 +5,9 @@ - name: Include ooni-backend playbook ansible.builtin.import_playbook: deploy-ooni-backend.yml +- name: Include ooni-clickhouse-proxy playbook + ansible.builtin.import_playbook: deploy-clickhouse-proxy.yml + - name: Include clickhouse playbook ansible.builtin.import_playbook: deploy-clickhouse.yml diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf new file mode 100644 index 0000000..d56c1b9 --- /dev/null +++ b/tf/modules/ec2/main.tf @@ -0,0 +1,124 @@ +data "aws_ssm_parameter" "ubuntu_22_ami" { + name = "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id" +} + +# Important note about security groups: +# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group#recreating-a-security-group +resource "aws_security_group" "ec2_sg" { + description = "security group for ec2" + name_prefix = var.sg_prefix + + vpc_id = var.vpc_id + + ingress { + protocol = "tcp" + from_port = 9000 + to_port = 9000 + } + + ingress { + protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + + cidr_blocks = [ + "0.0.0.0/0", + ] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + ipv6_cidr_blocks = ["::/0"] + } + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +data "cloudinit_config" "ooni_ec2" { + base64_encode = true + + part { + filename = "init.cfg" + content_type = "text/cloud-config" + content = templatefile("${path.module}/templates/cloud-init.yml", {}) + } + +} + +resource "aws_launch_template" "ooni_ec2" { + name_prefix = "${var.name}-tmpl-" + image_id = data.aws_ssm_parameter.ubuntu_22_ami.value + instance_type = var.instance_type + key_name = var.key_name + + user_data = data.cloudinit_config.ooni_ec2.rendered + + lifecycle { + create_before_destroy = true + } + + network_interfaces { + delete_on_termination = true + associate_public_ip_address = true + subnet_id = var.subnet_id + security_groups = [ + aws_security_group.ec2_sg.id, + ] + } + + tag_specifications { + resource_type = "instance" + tags = var.tags + } +} + +resource "aws_instance" "ooni_ec2" { + launch_template { + id = aws_launch_template.ooni_ec2.id + version = "$Latest" + } + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_alb_target_group" "ooni_ec2" { + name_prefix = "oo${var.tg_prefix}" + port = 80 + protocol = "HTTP" + vpc_id = var.vpc_id + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_lb_target_group_attachment" "oonibackend_proxy" { + target_id = aws_instance.ooni_ec2 + target_group_arn = aws_alb_target_group.ooni_ec2.arn +} diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf new file mode 100644 index 0000000..f4765c1 --- /dev/null +++ b/tf/modules/ec2/outputs.tf @@ -0,0 +1,7 @@ +output "aws_instance_id" { + value = aws_instance.oonibackend_proxy.id +} + +output "aws_instance_public_dns" { + value = aws_instance.ooni_ec2.public_dns +} \ No newline at end of file diff --git a/tf/modules/ec2/templates/cloud-init.yml b/tf/modules/ec2/templates/cloud-init.yml new file mode 100644 index 0000000..e69de29 diff --git a/tf/modules/ec2/variables.tf b/tf/modules/ec2/variables.tf new file mode 100644 index 0000000..9006fd2 --- /dev/null +++ b/tf/modules/ec2/variables.tf @@ -0,0 +1,45 @@ +variable "vpc_id" { + description = "the id of the VPC to deploy the instance into" +} + +variable "subnet_id" { + description = "the ids of the subnet to deploy the instance into" +} + +variable "private_subnet_cidr" { + description = "the cidr block of the private subnet to allow traffic from for the clickhouse proxy" +} + + variable "tags" { + description = "tags to apply to the resources" + default = {} + type = map(string) +} + +variable "key_name" { + description = "Name of AWS key pair" +} + +variable "name" { + description = "Name of the resources" +} + +variable "instance_type" { + default = "t2.micro" +} + +variable "stage" { + default = "one of dev, stage, test, prod" +} + +variable "dns_zone_ooni_io" { + description = "id of the DNS zone for ooni_io" +} + +variable "sg_prefix" { + description = "security group prefix" +} + +variable "tg_prefix" { + description = "target group prefix. Will be prefixed with `oo`, example: bkprx -> oobkprx" +} \ No newline at end of file diff --git a/tf/modules/ooni_backendproxy/main.tf b/tf/modules/ooni_backendproxy/main.tf index 110461d..a5674a6 100644 --- a/tf/modules/ooni_backendproxy/main.tf +++ b/tf/modules/ooni_backendproxy/main.tf @@ -14,7 +14,6 @@ resource "aws_security_group" "nginx_sg" { protocol = "tcp" from_port = 9000 to_port = 9000 - cidr_blocks = var.private_subnet_cidr } ingress { From 863ae727642b100ea1f6ad837a94da010ffc558e Mon Sep 17 00:00:00 2001 From: decfox Date: Mon, 9 Dec 2024 17:40:33 -0500 Subject: [PATCH 08/20] refactor: move backendproxy to generic ec2 module --- ansible/deploy-ooni-th.yml | 12 ++++++++ tf/environments/dev/main.tf | 41 +++++++++++++++++++++----- tf/modules/ec2/main.tf | 59 +++++++++++++++---------------------- tf/modules/ec2/variables.tf | 20 +++++++++++++ 4 files changed, 88 insertions(+), 44 deletions(-) create mode 100644 ansible/deploy-ooni-th.yml diff --git a/ansible/deploy-ooni-th.yml b/ansible/deploy-ooni-th.yml new file mode 100644 index 0000000..633cfc4 --- /dev/null +++ b/ansible/deploy-ooni-th.yml @@ -0,0 +1,12 @@ +--- +- name: Deploy webconnectivity test helpers + hosts: + - 0.do.th.dev.ooni.io + - 0.do.th.prod.ooni.io + - 1.do.th.prod.ooni.io + - 2.do.th.prod.ooni.io + roles: + - role: bootstrap + - role: nginx + tags: nginx + - role: \ No newline at end of file diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 2b14235..4480a7a 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -415,7 +415,7 @@ module "ooniapi_reverseproxy" { } module "ooni_backendproxy" { - source = "../../modules/ooni_backendproxy" + source = "../../modules/ec2" stage = local.environment @@ -427,11 +427,38 @@ module "ooni_backendproxy" { key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "t3a.nano" - backend_url = "https://backend-fsn.ooni.org/" - wcth_addresses = module.ooni_th_droplet.droplet_ipv4_address - wcth_domain_suffix = "th.ooni.org" - clickhouse_url = "clickhouse1.prod.ooni.io" - clickhouse_port = "9000" + name = "oonibkprx" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 9000, + to_port = 9000, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"] + }] + + sg_prefix = "oobkprx" + tg_prefix = "bkprx" tags = merge( local.tags, @@ -439,8 +466,6 @@ module "ooni_backendproxy" { ) } - - #### OONI Run service module "ooniapi_oonirun_deployer" { diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf index d56c1b9..4c14395 100644 --- a/tf/modules/ec2/main.tf +++ b/tf/modules/ec2/main.tf @@ -10,48 +10,35 @@ resource "aws_security_group" "ec2_sg" { vpc_id = var.vpc_id - ingress { - protocol = "tcp" - from_port = 9000 - to_port = 9000 - } - - ingress { - protocol = "tcp" - from_port = 80 - to_port = 80 - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - protocol = "tcp" - from_port = 22 - to_port = 22 - cidr_blocks = ["0.0.0.0/0"] + lifecycle { + create_before_destroy = true } - egress { - from_port = 0 - to_port = 0 - protocol = "-1" + tags = var.tags +} - cidr_blocks = [ - "0.0.0.0/0", - ] - } +resource "aws_security_group_rule" "ec2_sg_ingress" { + count = length(var.ingress_rules) - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - ipv6_cidr_blocks = ["::/0"] - } + type = "ingress" + from_port = var.ingress_rules[count.index].from_port + to_port = var.ingress_rules[count.index].to_port + protocol = var.ingress_rules[count.index].protocol + cidr_blocks = [var.ingress_rules[count.index].cidr_blocks] + ipv6_cidr_blocks = [var.ingress_rules[count.index].ipv6_cidr_blocks] + security_group_id = aws_security_group.ec2_sg.id +} - lifecycle { - create_before_destroy = true - } +resource "aws_security_group_rule" "ec2_sg_egress" { + count = length(var.egress_rules) - tags = var.tags + type = "egress" + from_port = var.egress_rules[count.index].from_port + to_port = var.egress_rules[count.index].to_port + protocol = var.egress_rules[count.index].protocol + cidr_blocks = var.egress_rules[count.index].cidr_blocks + ipv6_cidr_blocks = var.egress_rules[count.index].ipv6_cidr_blocks + security_group_id = aws_security_group.ec2_sg.id } data "cloudinit_config" "ooni_ec2" { diff --git a/tf/modules/ec2/variables.tf b/tf/modules/ec2/variables.tf index 9006fd2..6999961 100644 --- a/tf/modules/ec2/variables.tf +++ b/tf/modules/ec2/variables.tf @@ -40,6 +40,26 @@ variable "sg_prefix" { description = "security group prefix" } +variable "ingress_rules" { + type = list(object({ + from_port = number + to_port = number + protocol = string + cidr_blocks = list(string) + ipv6_cidr_blocks = optional(list(string)) + })) +} + +variable "egress_rules" { + type = list(object({ + from_port = number + to_port = number + protocol = string + cidr_blocks = optional(list(string)) + ipv6_cidr_blocks = optional(list(string)) + })) +} + variable "tg_prefix" { description = "target group prefix. Will be prefixed with `oo`, example: bkprx -> oobkprx" } \ No newline at end of file From adf63e57c82a148377505f9573523f4a5bb75609 Mon Sep 17 00:00:00 2001 From: decfox Date: Tue, 10 Dec 2024 09:19:10 -0500 Subject: [PATCH 09/20] fix: cidr blocks arrays for ec2 module --- tf/modules/ec2/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf index 4c14395..327b0cf 100644 --- a/tf/modules/ec2/main.tf +++ b/tf/modules/ec2/main.tf @@ -24,8 +24,8 @@ resource "aws_security_group_rule" "ec2_sg_ingress" { from_port = var.ingress_rules[count.index].from_port to_port = var.ingress_rules[count.index].to_port protocol = var.ingress_rules[count.index].protocol - cidr_blocks = [var.ingress_rules[count.index].cidr_blocks] - ipv6_cidr_blocks = [var.ingress_rules[count.index].ipv6_cidr_blocks] + cidr_blocks = var.ingress_rules[count.index].cidr_blocks + ipv6_cidr_blocks = var.ingress_rules[count.index].ipv6_cidr_blocks security_group_id = aws_security_group.ec2_sg.id } From 3b7deeb8f05014bfb7f0550736c55f3163ffe0cb Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 11 Dec 2024 17:27:18 -0500 Subject: [PATCH 10/20] fix: eof errors --- ansible/deploy-clickhouse-proxy.yml | 20 +++++++++---------- ansible/deploy-clickhouse.yml | 6 +++--- ansible/deploy-ooni-th.yml | 12 ----------- ansible/host_vars/data3.htz-fsn.prod.ooni.nu | 3 ++- ansible/inventory | 6 +++--- ansible/roles/clickhouse_proxy/tasks/main.yml | 2 +- .../clickhouse_proxy/templates/99-stream.conf | 2 +- 7 files changed, 20 insertions(+), 31 deletions(-) delete mode 100644 ansible/deploy-ooni-th.yml diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml index 6d73c82..803b1b1 100644 --- a/ansible/deploy-clickhouse-proxy.yml +++ b/ansible/deploy-clickhouse-proxy.yml @@ -1,11 +1,11 @@ --- -- name: Deploy clickhouse proxy - hosts: - - clickhouseproxy.dev.ooni.io - roles: - - role: bootstrap - - role: nginx - tags: nginx - - role: clickhouse_proxy - vars: - clickhouse_url: "" # fetch from AWS secrets \ No newline at end of file +## - name: Deploy clickhouse proxy + ## hosts: + ## - clickhouseproxy.dev.ooni.io + ## roles: + ## - role: bootstrap + ## - role: nginx + ## tags: nginx + ## - role: clickhouse_proxy + ## vars: + ## clickhouse_url: "" # fetch from AWS secrets diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index c2d34cc..f6781fb 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,13 +1,13 @@ --- - name: Deploy oonidata clickhouse hosts hosts: - - notebook.ooni.org - - data1.htz-fsn.prod.ooni.nu + # - notebook.ooni.org + # - data1.htz-fsn.prod.ooni.nu #- data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true tags: - clickhouse roles: - - prometheus_node_exporter + # - prometheus_node_exporter - oonidata_clickhouse diff --git a/ansible/deploy-ooni-th.yml b/ansible/deploy-ooni-th.yml deleted file mode 100644 index 633cfc4..0000000 --- a/ansible/deploy-ooni-th.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Deploy webconnectivity test helpers - hosts: - - 0.do.th.dev.ooni.io - - 0.do.th.prod.ooni.io - - 1.do.th.prod.ooni.io - - 2.do.th.prod.ooni.io - roles: - - role: bootstrap - - role: nginx - tags: nginx - - role: \ No newline at end of file diff --git a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu index 502a005..661157a 100644 --- a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu @@ -1,4 +1,5 @@ non_admin_usernames: [ ] clickhouse_base_path: /data/clickhouse clickhouse_custom_databases: - - name: test \ No newline at end of file + - name: test + \ No newline at end of file diff --git a/ansible/inventory b/ansible/inventory index a44f8d4..e0b76da 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -5,9 +5,9 @@ ghs-ams ## Role tags [clickhouse] -notebook.ooni.org -data1.htz-fsn.prod.ooni.nu -data2.htz-fsn.prod.ooni.nu +# notebook.ooni.org +# data1.htz-fsn.prod.ooni.nu +# data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu ## Location tags diff --git a/ansible/roles/clickhouse_proxy/tasks/main.yml b/ansible/roles/clickhouse_proxy/tasks/main.yml index fb6ea49..1697d13 100644 --- a/ansible/roles/clickhouse_proxy/tasks/main.yml +++ b/ansible/roles/clickhouse_proxy/tasks/main.yml @@ -8,4 +8,4 @@ owner: root notify: - reload nginx - - restart nginx \ No newline at end of file + - restart nginx diff --git a/ansible/roles/clickhouse_proxy/templates/99-stream.conf b/ansible/roles/clickhouse_proxy/templates/99-stream.conf index 1a9000f..c5b8062 100644 --- a/ansible/roles/clickhouse_proxy/templates/99-stream.conf +++ b/ansible/roles/clickhouse_proxy/templates/99-stream.conf @@ -10,4 +10,4 @@ stream { } error_log /var/log/nginx/error.log; -} \ No newline at end of file +} From d2925ce18d9184e2052ac042e3305128c3df9914 Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 11 Dec 2024 17:27:37 -0500 Subject: [PATCH 11/20] fix: more eof errors --- tf/modules/ec2/outputs.tf | 2 +- tf/modules/ec2/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf index f4765c1..6d6cfca 100644 --- a/tf/modules/ec2/outputs.tf +++ b/tf/modules/ec2/outputs.tf @@ -4,4 +4,4 @@ output "aws_instance_id" { output "aws_instance_public_dns" { value = aws_instance.ooni_ec2.public_dns -} \ No newline at end of file +} diff --git a/tf/modules/ec2/variables.tf b/tf/modules/ec2/variables.tf index 6999961..78a0250 100644 --- a/tf/modules/ec2/variables.tf +++ b/tf/modules/ec2/variables.tf @@ -62,4 +62,4 @@ variable "egress_rules" { variable "tg_prefix" { description = "target group prefix. Will be prefixed with `oo`, example: bkprx -> oobkprx" -} \ No newline at end of file +} From b7ebb36243e1ba2a6fd79ecca22114a020a90ec6 Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 11 Dec 2024 17:29:21 -0500 Subject: [PATCH 12/20] revert comments --- ansible/deploy-clickhouse.yml | 6 +++--- ansible/inventory | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index f6781fb..fc09e7a 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,9 +1,9 @@ --- - name: Deploy oonidata clickhouse hosts hosts: - # - notebook.ooni.org - # - data1.htz-fsn.prod.ooni.nu - #- data2.htz-fsn.prod.ooni.nu + - notebook.ooni.org + - data1.htz-fsn.prod.ooni.nu + - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true tags: diff --git a/ansible/inventory b/ansible/inventory index e0b76da..a44f8d4 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -5,9 +5,9 @@ ghs-ams ## Role tags [clickhouse] -# notebook.ooni.org -# data1.htz-fsn.prod.ooni.nu -# data2.htz-fsn.prod.ooni.nu +notebook.ooni.org +data1.htz-fsn.prod.ooni.nu +data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu ## Location tags From cfb1049a5be22b830ecb5f2a9243e063777b88e0 Mon Sep 17 00:00:00 2001 From: decfox Date: Fri, 13 Dec 2024 13:25:37 -0500 Subject: [PATCH 13/20] remove db configs from host_vars --- ansible/deploy-clickhouse.yml | 8 ++++---- ansible/host_vars/data3.htz-fsn.prod.ooni.nu | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index fc09e7a..55cbc01 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,13 +1,13 @@ --- - name: Deploy oonidata clickhouse hosts hosts: - - notebook.ooni.org - - data1.htz-fsn.prod.ooni.nu - - data2.htz-fsn.prod.ooni.nu + # - notebook.ooni.org + # - data1.htz-fsn.prod.ooni.nu + # - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true tags: - clickhouse roles: - # - prometheus_node_exporter + - prometheus_node_exporter - oonidata_clickhouse diff --git a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu index 661157a..60cd983 100644 --- a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu @@ -1,5 +1,3 @@ -non_admin_usernames: [ ] +non_admin_usernames: [] clickhouse_base_path: /data/clickhouse -clickhouse_custom_databases: - - name: test \ No newline at end of file From 6c541be8bbaee48c28160ffd5ef6b63cda3077e3 Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 18 Dec 2024 13:01:41 +0530 Subject: [PATCH 14/20] refactor: changes to allow terraform to apply cleanly --- tf/environments/dev/main.tf | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 4480a7a..0b5b885 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -414,7 +414,7 @@ module "ooniapi_reverseproxy" { ) } -module "ooni_backendproxy" { +module "ooni_clickhouse_proxy" { source = "../../modules/ec2" stage = local.environment @@ -427,7 +427,7 @@ module "ooni_backendproxy" { key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "t3a.nano" - name = "oonibkprx" + name = "oonickprx" ingress_rules = [{ from_port = 22, to_port = 22, @@ -457,15 +457,26 @@ module "ooni_backendproxy" { ipv6_cidr_blocks = ["::/0"] }] - sg_prefix = "oobkprx" - tg_prefix = "bkprx" + sg_prefix = "oockprx" + tg_prefix = "ckpr" tags = merge( local.tags, - { Name = "ooni-tier0-backendproxy" } + { Name = "ooni-tier0-clickhouseproxy" } ) } +resource "aws_route53_record" "clickhouse_proxy_alias" { + zone_id = local.dns_zone_ooni_io + name = "clickhouseproxy.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_clickhouse_proxy.aws_instance_public_dns + ] +} + #### OONI Run service module "ooniapi_oonirun_deployer" { From 01302a4bb0fbb7a01006b71f268303ba77d05dcd Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 18 Dec 2024 13:01:53 +0530 Subject: [PATCH 15/20] refactor: changes to allow terraform to apply cleanly --- tf/modules/ec2/main.tf | 2 +- tf/modules/ec2/outputs.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf index 327b0cf..1e7691f 100644 --- a/tf/modules/ec2/main.tf +++ b/tf/modules/ec2/main.tf @@ -106,6 +106,6 @@ resource "aws_alb_target_group" "ooni_ec2" { } resource "aws_lb_target_group_attachment" "oonibackend_proxy" { - target_id = aws_instance.ooni_ec2 + target_id = aws_instance.ooni_ec2.id target_group_arn = aws_alb_target_group.ooni_ec2.arn } diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf index 6d6cfca..4a99b3c 100644 --- a/tf/modules/ec2/outputs.tf +++ b/tf/modules/ec2/outputs.tf @@ -1,5 +1,5 @@ output "aws_instance_id" { - value = aws_instance.oonibackend_proxy.id + value = aws_instance.ooni_ec2.id } output "aws_instance_public_dns" { From fe40debf8631c3dfc35c248c485c8ce995e40fd0 Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 18 Dec 2024 17:01:15 +0530 Subject: [PATCH 16/20] refactor: apply clickhouse proxy ansible configs --- ansible/deploy-clickhouse-proxy.yml | 22 ++++++++++--------- ansible/deploy-clickhouse.yml | 6 ++--- ansible/inventory | 3 +++ ansible/roles/bootstrap/tasks/main.yml | 1 + ansible/roles/clickhouse_proxy/tasks/main.yml | 9 ++++++++ .../clickhouse_proxy/templates/99-stream.conf | 2 +- 6 files changed, 29 insertions(+), 14 deletions(-) diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml index 803b1b1..233ace4 100644 --- a/ansible/deploy-clickhouse-proxy.yml +++ b/ansible/deploy-clickhouse-proxy.yml @@ -1,11 +1,13 @@ --- -## - name: Deploy clickhouse proxy - ## hosts: - ## - clickhouseproxy.dev.ooni.io - ## roles: - ## - role: bootstrap - ## - role: nginx - ## tags: nginx - ## - role: clickhouse_proxy - ## vars: - ## clickhouse_url: "" # fetch from AWS secrets +- name: Deploy clickhouse proxy + hosts: + - clickhouseproxy.dev.ooni.io + become: true + roles: + - role: bootstrap + - role: nginx + tags: nginx + - role: clickhouse_proxy + vars: + clickhouse_url: "clickhouse3.prod.ooni.io" + clickhouse_port: 9000 diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index 55cbc01..f082fa0 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,9 +1,9 @@ --- - name: Deploy oonidata clickhouse hosts hosts: - # - notebook.ooni.org - # - data1.htz-fsn.prod.ooni.nu - # - data2.htz-fsn.prod.ooni.nu + - notebook.ooni.org + - data1.htz-fsn.prod.ooni.nu + - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true tags: diff --git a/ansible/inventory b/ansible/inventory index a44f8d4..6861739 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -22,3 +22,6 @@ data3.htz-fsn.prod.ooni.nu [ghs-ams] openvpn-server1.ooni.io + +[aws-proxy] +clickhouseproxy.dev.ooni.io diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 500d58f..2b65c61 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -68,6 +68,7 @@ - name: install systemd-resolved tags: resolved + ignore_errors: yes # some ubuntu hosts do not have the systemd-resolved package ansible.builtin.apt: install_recommends: no cache_valid_time: 86400 diff --git a/ansible/roles/clickhouse_proxy/tasks/main.yml b/ansible/roles/clickhouse_proxy/tasks/main.yml index 1697d13..5f18a27 100644 --- a/ansible/roles/clickhouse_proxy/tasks/main.yml +++ b/ansible/roles/clickhouse_proxy/tasks/main.yml @@ -1,4 +1,13 @@ --- +- name: Create the modules-enabled directory if not exists + tags: webserv + ansible.builtin.file: + path: /etc/nginx/modules-enabled + state: directory + mode: 0755 + owner: root + group: root + - name: Add stream nginx config tags: webserv template: diff --git a/ansible/roles/clickhouse_proxy/templates/99-stream.conf b/ansible/roles/clickhouse_proxy/templates/99-stream.conf index c5b8062..22f5a0f 100644 --- a/ansible/roles/clickhouse_proxy/templates/99-stream.conf +++ b/ansible/roles/clickhouse_proxy/templates/99-stream.conf @@ -1,6 +1,6 @@ stream { upstream clickhouse_backend { - server {{ clickhouse_url }}; + server {{ clickhouse_url }}:{{ clickhouse_port }}; } server { From e4048a258e33a1664a1f73ff7dd2cbc30794c71d Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 18 Dec 2024 22:28:37 +0530 Subject: [PATCH 17/20] fix eof --- ansible/deploy-monitoring.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/deploy-monitoring.yml b/ansible/deploy-monitoring.yml index a1eadee..0c27527 100644 --- a/ansible/deploy-monitoring.yml +++ b/ansible/deploy-monitoring.yml @@ -8,5 +8,3 @@ - prometheus - prometheus_blackbox_exporter - prometheus_alertmanager - - From 093ce47d2185649477e677473799840f81d1a7e0 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 19 Dec 2024 17:45:42 +0530 Subject: [PATCH 18/20] fix: eof errors --- ansible/deploy-ooni-backend.yml | 2 +- ansible/host_vars/data3.htz-fsn.prod.ooni.nu | 1 - ansible/roles/ooni-backend/defaults/main.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ansible/deploy-ooni-backend.yml b/ansible/deploy-ooni-backend.yml index 0529e8d..26c0603 100644 --- a/ansible/deploy-ooni-backend.yml +++ b/ansible/deploy-ooni-backend.yml @@ -20,7 +20,7 @@ vars: ssl_domain: backend-hel.ooni.org collector_id: 2 - clickhouse_url: "" # fetch from aws secrets + clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse3.prod.ooni.io/oonitest" bucket_name: ooni-data-eu-fra-test collectors: - "backend-hel.ooni.org" diff --git a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu index 60cd983..bf1b24a 100644 --- a/ansible/host_vars/data3.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/data3.htz-fsn.prod.ooni.nu @@ -1,3 +1,2 @@ non_admin_usernames: [] clickhouse_base_path: /data/clickhouse - \ No newline at end of file diff --git a/ansible/roles/ooni-backend/defaults/main.yml b/ansible/roles/ooni-backend/defaults/main.yml index 8413d7e..33ce9e4 100644 --- a/ansible/roles/ooni-backend/defaults/main.yml +++ b/ansible/roles/ooni-backend/defaults/main.yml @@ -8,4 +8,4 @@ collectors: fastpath_version: 0.86~pr831-395 analysis_version: 1.12~pr836-413 deb_bucket_name: ooni-internal-deb -deb_server_name: deb-ci.ooni.org \ No newline at end of file +deb_server_name: deb-ci.ooni.org From ddb8a74ef8a6304e3b47ce5bd803697164459106 Mon Sep 17 00:00:00 2001 From: decfox Date: Fri, 20 Dec 2024 11:15:41 +0530 Subject: [PATCH 19/20] refactor: make backend ansible role functional --- ansible/deploy-ooni-backend.yml | 39 +- ansible/group_vars/all/vault | 464 ++++++++++++++++++ .../vars.yml} | 0 ansible/host_vars/backend-fsn.ooni.org/vault | 78 +++ .../vars.yml} | 1 - ansible/host_vars/backend-hel.ooni.org/vault | 30 ++ ansible/inventory | 3 + ansible/password-pipe | 2 + ansible/roles/base-backend/meta/main.yml | 6 - ansible/roles/base-backend/tasks/main.yml | 4 +- ansible/roles/dehydrated/handlers/main.yml | 15 + ansible/roles/dehydrated/meta/main.yml | 5 - ansible/roles/dehydrated/tasks/main.yml | 30 +- ansible/roles/ooni-backend/defaults/main.yml | 2 + ansible/roles/ooni-backend/meta/main.yml | 3 - ansible/roles/ooni-backend/tasks/main.yml | 15 +- .../ooni-backend/templates/db-backup.conf | 2 +- 17 files changed, 614 insertions(+), 85 deletions(-) create mode 100644 ansible/group_vars/all/vault rename ansible/host_vars/{backend-fsn.ooni.org => backend-fsn.ooni.org/vars.yml} (100%) create mode 100644 ansible/host_vars/backend-fsn.ooni.org/vault rename ansible/host_vars/{backend-hel.ooni.org => backend-hel.ooni.org/vars.yml} (90%) create mode 100644 ansible/host_vars/backend-hel.ooni.org/vault create mode 100755 ansible/password-pipe delete mode 100644 ansible/roles/base-backend/meta/main.yml create mode 100644 ansible/roles/dehydrated/handlers/main.yml delete mode 100644 ansible/roles/dehydrated/meta/main.yml delete mode 100644 ansible/roles/ooni-backend/meta/main.yml diff --git a/ansible/deploy-ooni-backend.yml b/ansible/deploy-ooni-backend.yml index 26c0603..9686248 100644 --- a/ansible/deploy-ooni-backend.yml +++ b/ansible/deploy-ooni-backend.yml @@ -1,7 +1,11 @@ --- -- hosts: backend-hel.ooni.org +- name: Deploy ooni backend services + hosts: backend-hel.ooni.org + become: true roles: - role: bootstrap + vars: + admin_group_name: adm - role: base-backend - role: nftables - role: nginx @@ -22,39 +26,10 @@ collector_id: 2 clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse3.prod.ooni.io/oonitest" bucket_name: ooni-data-eu-fra-test + public_bucket_name: ooni-data-eu-fra-test collectors: - "backend-hel.ooni.org" fastpath_version: 0.86~pr831-395 - analysis_version: 1.12~pr836-413 + analysis_version: 1.12~pr836-412 deb_bucket_name: ooni-internal-deb deb_server_name: deb-cli.ooni.org - -# - hosts: backend-fsn.ooni.org - # roles: - # - role: bootstrap - # - role: base-backend - # - role: nftables - # - role: nginx - # tags: nginx - # vars: - # nginx_user: "www-data" - # - role: dehydrated - # tags: dehydrated - # expand: yes - # vars: - # ssl_domains: - # # with dehydrated the first entry is the cert FQDN - # # and the other ones are alternative names - # - "backend-hel.ooni.org" - # - role: ooni-backend - # vars: - # ssl_domain: backend-fsn.ooni.org - # collector_id: 1 - # clickhouse_url: "" # fetch from aws secrets - # bucket_name: ooni-data-eu-fra - # collectors: - # - "backend-fsn.ooni.org" - # fastpath_version: 0.86~pr831-395 - # analysis_version: 1.12~pr836-413 - # deb_bucket_name: ooni.deb - # deb_server_name: deb.ooni.org diff --git a/ansible/group_vars/all/vault b/ansible/group_vars/all/vault new file mode 100644 index 0000000..6313457 --- /dev/null +++ b/ansible/group_vars/all/vault @@ -0,0 +1,464 @@ +$ANSIBLE_VAULT;1.1;AES256 +35396133653938666535306632373462613132333535633462326562663932323134336136326663 +3737396336363135653364383334653338623765343734370a386536303632326337666634323230 +37333931636561613162343765313634663562636635333062376337323130656433393866616439 +3061313766366631310a616431386463326631333036313535623264373661393261666135346563 +35323835653237363935653139623361626264366133393439653966323032313964373961353462 +64363666613863623037616337653732653736326361313831653661393932313662323635383365 +31363362373732316139666264383933376231643634303063333231616434343261653435393761 +64313662366465323437383435363137643130313735633738326365306234366633333165643338 +38353765386534306336613337353837636463326464616432346334623634306463376237623863 +39356633376532303230616236353562623666323462636462636435363138623839306363613563 +33333130623364363932363939616336366162323238633939376337633430363535396335316339 +34306137343630336662373661663465333839343366613861626233643766326462613163306532 +34366662616661323630656266356535643434666463313166303263663330633762383539613663 +36643535393266613137636565613865623564396261326231343535313732343636633938636134 +32343438336137373232356335356233353866653463633432343230313933383031356631646639 +63616132626261393365373637666237656339646438656563346466653066623263376639636365 +37343030336462346465326134383531343662626163323033656235316634386538383835326362 +66613730616534336435656331653930643266386133336562323433633865333966393161313362 +62653836323633376631616138653161653931663433303437366232343038623564303134653736 +33326230343161613863666137323464333537613561623937663932666361343336313037333832 +62353735623665303431303731313933626166363062363037646162323532346361346437353765 +39663137643531346230313138656165613935663235666337646362306262393734326462346166 +64666430636361366638646234616135313663623762343031646630643334316331643734396332 +38393564626565646161313164616663396637386439643035333932366230356432646661623339 +38653038653534666362316435393437333665393635343331633062316464363838373663313863 +30616136643237666238386162643038346662356133386335323635336466336464323834303262 +38326463393737353765386439643262366565626666363739343437376630396565373636336563 +30363332306562396630393531343363633738356237333137336336613664393262633966313430 +30353236333963313634653765636336366461646334633434306361623061353036663037616535 +66366332386361613235383032323561613737326566633939666231373430333563626539653765 +38396539613062373963363066653639346262643466663538386664303362356535623666396237 +31323237356433653133303866613937613064623266353662623839353535376533396664363761 +38653339653332666137653930343436393639363432666363663363333462376236353831336436 +33353432383163323533633432613563666533316534353761383530653563613233386137376630 +65303562616334613764393339343963626533383363363865333036393137343464393435326135 +36623038356163346365373130613031383333383138643962653137633435373639373135323166 +63326631663432643635643936346335336337656664646162353163623465366264613936363661 +34653435396131303432613365356135616464616462336533646538623630646465363730366161 +64663865613462666661326134333633623561333161646530306461393766333738346437646438 +35613435353134653737303131663235383866653266393339323361646239306365653735373764 +30323233646164393265633164633932373437373339363762626462366533393531336666383865 +30313231316633643764623961643639346239373533313034353164376432303833386661376235 +61633866323335363530383338383132373165613634306636633838393366353037343561616632 +39626230353730386465373232343864636439303963336262623839343761323331663338393739 +38323562303838313830313732343262623430393036643833363761666562353661646538646237 +66306562663836643662653463323935653738653136333136643363343661376430343534326633 +39303331373833663561303130353334643562343737626230343139343934643465303230393838 +64396631613461393533376335643331653761636164613736616165656261323335373031333362 +61653131366463643766353831346133373932613630626236323436653661343738646232376533 +32646633383466313931636263316163643865613935373939306231343335396630373232626533 +39646233363361623361663961666330646432666463353531666565666434373763336534633834 +36626161623038613734343734383266333135346535306632333331383436633464643662356665 +62383130626239396662366132653639653936333665636534346433323437313361373235383038 +36643966313637653036376636613630336135313530353133343033663135393436613635666139 +37333432643436633931313932363962366266336364383763366561366330643061623065376338 +36366432393830646630303139336333313037656633386333623363363763646163653630383362 +38303537363539616263333732303963303333303133383231653630396233323139656632616265 +36393663383864383738343631663235316639303935353766326133373166393762656464373239 +39363535343963356637396238353063643534383864373633333335636236613634323634393338 +63303564323636326265356635383334626262346534636465326435373864313231356539333232 +62633238326665636138666262613632663031653438623433326435666433663136306636646466 +64663563643631636461366462653732623839303539636364396666353834633931653638343064 +62343837376237333962386464663362613663353036383865633661306333336461313739346238 +31346237633030623932373035643832396535353335323939393936383931356530643337353137 +34613562376331636138323331363235303166323735643333613865633836383661636237663139 +30383631623963343461316462643465393637383732363664306539623533363933613863316639 +32316639323538316538663532613935316533323865633665666632653737666334623932613463 +37653137383937323632633230653662306532663936323834613234336562333939613036393330 +38363334653432643030663833373161383766646561356130613438646537633761643462346239 +63663233396363626432613130643236623266393438633733363133656462336235346161663334 +32393365376364363336366639386462656662356536393739616564373466613431376265626139 +64383733393732316230633838663037633865383139306666626533386431363462613231623563 +66633535366233353731346664393536353831326332653030343663376563306462366266643532 +31393934663937343164356636636166646166326466333661393662646634663562626135633664 +35663537633264636233346565303039353962333837306230613637646530353234666664356364 +30363830326536653935376164383534653739656435386530343437356666343437613234383836 +66636139303039643939373636646532626639303638653666393563353134663064383863663438 +32323665636538373632343162633030303563393938643362323131376537353938303932656334 +66376238323831333634343233373266303431336561383532636262346338336138643230316138 +65373931633731616135646338653765613566383063346465653536666535313835636433643335 +31626363613136646533653663646436303063623564396666623538383334353666326631316337 +62346436326431343330646433333863613835396332373764653839366263333937333437626165 +37383939363536363038353065303133333839376265636564316231363263393533646236343932 +64326139383938386365616439633662363231323664343634363131306162373434333635396436 +35653465623938393830643564656565616335396332623432613934656136323231666234356138 +36643463313937356337323235623031326264303839613336316466643365303336366461626132 +61393735323537636639636335336239326334356436366161366337343466656662386636303163 +33373839666163663061323133633832333235336238626130333334613035303632333037393266 +34303066346331383464343462373432363563613531393332363835633533313935633765653634 +33396666366236663936623963366232643835666236663831326465353637356666303339653732 +38636232313930363837393333646537636362383761643066343838613834613734366537323466 +63653734646335666365383366313265636537666338666134333138326364386133666433346431 +38626235343534323666356537633635333633393838353132333536323938316437343363333762 +37303131376137316462336339353639366436353862666561313830646434313534363764386539 +31623131643531363739373139623633626438343865383733376339623062333561623330653038 +64633466626635393136626137616234376663333564373862323366343035343236386237333638 +31323730613537633137363138616361363532663632343863663465643736306365306163373735 +32363839353530393636623838313634386634636561393761623733306637616134636135396362 +64386364666635383633363239383862623531623438663732383230656230343037316161396261 +31353361393665363232656636396234313962336431636661326239343539353135323236323434 +36313637373064393764656432613433353762323636303935303636323262306430646138636334 +37353136316531613236643866613833323264303262666232653834336661346264653066323331 +30373832643766333336306263313964646361666464663665653366623166333730363836396537 +62633339666663663339383438653666376134616535353331356135343565366138663233313238 +33336165353632626166316138333030623035663231633332316438646334343436393564343931 +36653666636534346365386638646336393265383933316362653230636131663266313936303935 +63666132373664626633303331653464613330353634363661356633373334376331383830323464 +61646163386236346132326561613035303735626263646432646564666533613261373930623066 +37373234333365346136623132626236326132653030386364613833313831623739386632663435 +66393736616635373566623062613634326162323835363632643538616663656161656238636361 +62313035396466343663626438393263653237373865663631323064393233333035363864343863 +39346566613637336566623266613233356638373636633433616633623239393563353366353637 +33323431383261333235356561303166643862643030613535623837303161373333313938613634 +31303263353839626435343932343939666364363062643432353965613136313530626132336430 +61376335643865306634333763326461373461626235333138633362386636386436383837376239 +65363466336133613834303733393932343461396336353263343938663837666433636261633639 +33633531326663633162653735373230373661633861616339373165333465323966333665633664 +32336165623237303539613837393834306133343338333166306333666461303232363935343331 +66316164356433643664366638623166393430613766323339326664653339633261666531336139 +30636266323732376430626361336664326138633366356431653931663864383339396431376237 +65613462313466633137626536646137633964346533383333333230663362306562306430386566 +36613066356637653930313466656361663762633338663135303235653462613335613265376461 +32356533663434343634386433376332313861643963636432393633613536353762393161643263 +64636230643339343461653764653164663534626132366262343638366437346136383838643063 +30653665346435396136613933653761316164316431363531656234386561646363363336623836 +62373731346639336435636463636231383736626263306538323361363133373963303063363063 +34303834396161313461323736323661383666663133396564376661323365333532336464633538 +37653666326334306639663832613036653437393339636332336336313363626633366537633739 +64343130646464633162613735376336393938323361323437363834623131646164663731383165 +36353139306135326631353334666562643836613032316531623064636263336163333933393164 +63613364656230663463333630346333313865623262376130396633643962326334313930333862 +61393630633036353031396330646262366266383164323062353366343335303962616133353134 +66323462626161626561623037646438303965623762363833353466376238396265666634383731 +38616338356166636331316539623938356266666334353866363863623733373131663435373135 +39313837353762636366666363363339326230633530303162663339643537306530643832393761 +33616561656264393735653838666263613739323063343332623531313133333561393261386163 +63666531333837373038623063333336333761633962363437353135323862626338336336333366 +39613432313633303931663466383166656462316666613135613163353535303237636162633030 +39353130333636393537336235666565643530393637626134333862623733656462663030323337 +61343230666233373433613538646336336661396139326230373834306565613864356263306462 +64633466646132613138306364383937336231376262316164393065646236636139396635353564 +39656235393463306663373733623037343263356234646332373633353433333131656639343664 +62623030316339623731623339383761316664346161623166643366303661303562303338666439 +66363636323864623565633162303565376530376636363032326666393162326265623665306263 +33303865306564303932353339323330623264323138376131663134306536376533373635343964 +31616231396234613862633935306433633231386662326331373736383461333535626461356366 +32366261316364353138323864623230383632623566373932373235653865393362303431336231 +36336330626366336665356536643631343534316666643737666564376638376130656161656338 +66393131653763306564376237336330313238393365376630366361643639613636653235313737 +32376266313666633239363636323237646335633032646639343932643163613465626339646662 +38616361353932663661663437303534303366333438366534343631633364386637663839313738 +62383662386330363265616663383238316437396662356630393263303062663361613834636234 +61383562646464356530636565393166666366373666623731646135343236376430666635353931 +33333833386465333665323934366336366433376163336430633931353634653133656464613062 +35656165663663633130346561313130393763393134356364323666663963646433656361393562 +38663130336633653739613230663839653666343966626431636364396338636561653939653239 +33343233633030346338363964613765386166316535303533323362326535643964396265646366 +34303163656566323265636139363162303037313232373837326565656536383166353336363839 +36373266633233336563383238646664613166663438373730333763353962323032303236363138 +39386531386664666437666163343432643161623766643166626465303739326264616262393038 +37613839633538353665343538386533393839333466646563363033376263333963653331343336 +31653639613632356662383466393839303261393163383565386335623131633438363436663836 +64656264356330336261333939303634343034313134623638363764376334346136386338643138 +38626538346339303139646238386431313863386633643731303930303562313065343034636538 +39633230363636376463623363666464343465366236393366323134633431303630663032396165 +39356131653136643066643335323632386662346435646438316230633438623439396362316430 +66633163396231313534313335393065366665393831383338336337363065323164343237613534 +39626561373139623666323139653232663364643431646663656663623061323238616465663132 +63343832363762313936313134386263393038306135393739623164643831623136356633663063 +30363032353736366235373065653564623534313738643663303337313531373561336433346538 +64383336346463663832376233663961623664306339663837363638643434393839646433633232 +64346238393738316632323562633266633761343765383263626266346665346530333963616361 +64326661643633316462393266393061306461343664643966626165636361313632643665643337 +37613333666366303062666435663139353933363366613864646361653437623537326132313234 +36613333623863643937386537343238396439323264633832656465333534346264323462393063 +63366434613239386331643233643265356530353863653062346264643966373638376561393438 +61653039623034393837623737356336643663666563333931376139353966333334303533353063 +31663763623935306362616632396639373163336634346239363631653862346437323963346631 +62386433386637363561386166633936356432373334333434666163633934643466633564336636 +63666332326162646161613430653766383862356165373766623435333937616139343362396437 +64323430383538623838316261623435613839623065343630313932653839373661323966643363 +34366537353932343763303065376462383731646431333137336535373630366138643465643461 +61616630616136643562383466393566313237326337623739613336356530626162633632326663 +61306635363536336633663230356130393364663965616364653366373363653464643536613538 +34373063623239313036616532663664663461643537393861313739393636613164666139373038 +39323637313036306366326563656131636365666533623632383137626530636561343764356437 +61626535323338633465316538343633326165303936343731306638383738366361373438333137 +61363164636133346132383436336463353639333131333163313662353233366562376533656562 +39373334663139363665643435643765646638373561303162323734613738623237333332333164 +39323530313532336237373765323637633064356538643932353463656336653561376639313264 +65353831613033386563626662613133623830633463653065376332363562386631663233623334 +30666333643163393638633033303737343333343133303535393634396563343862623661323938 +36656436306165313336306630643366336338633966633035666638613263393164623534396231 +35656137623636363939636138356438356462323433653335656435643766376466333133346433 +31303831326430633362636330376261626235353762303033303562643864353464633532393131 +66623233353766396634653363626333643363643962386565346665323038653361616332353239 +36336637373961373833656130333266313539386231633237363035303337346331636665323331 +35356262363238353064623836633833353735333634613763376530383439623136343533363365 +33313965386631393631666331666265633664316466383139613861353338343861323639363733 +36636132326663613930616338663330653565343338393037663763393433623233343462636437 +64306665353238303332376661613065623161316262616431326237663462323237313731346532 +37666535383430313133656538383861346330643330363562353465373634643132303230333532 +33303864326136633331656537636364613139353531363434383366663862393439616234653437 +66363533616538313162363133623037643238316665353161303964363439663539663234653030 +31373565613830383264643664306631316138323931373230306661343338656239616235633661 +63316334626431356234616635336461363265323366333131353233646137306463663662636262 +61376632346637353533353031616564656232623162613938333263323764623964616338353738 +32386261386463316364383363653639616664303963666632393965656362643630393766383530 +32346633306633383132396233383864373230643632643237663830313832363939376663633438 +31653063633138393763373635303532343034666136356634333832613233383264313163353965 +33323737346338613037356462643062396461643337323863346333306630663566623265313363 +30393539393364303635626633333531383636353137313561316338653536636535323838373239 +33333765643363316636666230656531613439656534626464636536323131663830353235623233 +65366465336535666534646331376466363463326139353533373733663262636261303966646436 +30376231633338393761383861623362646263383237376665626433636437623561303734663834 +33336534663239383562626331343263623065363333373839623932333733363862323139613231 +35393637643333336332613130633633613661353964303166336634633764633736363138383837 +34396461306139363336343333643666353735396666333064666536653132383834663438626131 +32393531313863616364303466333936653361306661656535316232313435376430666165643736 +37666162666563326162666538343532343730306136303737363733373536343265613864666431 +65306635623736613462316132633433363237393063656363613166666264653135656137363566 +38386164636233373833623830373937373664656663663436303466336361636263383064633937 +66363339613532613762336138353761333631656533323964383861623536333830386662383632 +30393739353764386234636337653764646539363230623834633530383962646131653633343930 +61623034346662383632376562366165383138396332373134356535623962373166353937633861 +66313135636530643664326237623138363732666438353662383237303636613635313762386434 +61383662346337396637336338633265613161396266613964393236633632363635356465653037 +35386237353734656265666135623232393938313565366536353332343637623835616365363937 +65646231643430313033326535626464346562316363373461356266353237346237343439653036 +39333539383734383030323438636636663731336536383063323130656665633234376232353830 +36643761303836316336366466343464623064373132383832353830613965343362336335383662 +35666331663532323934386565373961626463643731333631363630366339623236303061643762 +33376464333239623033393463386237396334333765363038666661353336313539336433363165 +61613664343433336365313438313132653638643463383238343338343639653537646135353835 +35666463386665623233623738313563383965363639613561326232633136633937343539323634 +65373663383539376438376131303437643537326435623863303135383436363139623331613438 +34383536363061336335306539373631363465373933306438346633623837623762376137313335 +64346436313734333836383439376230646136376339303661343033633765313361383361623537 +65393132343432306162316637613531333133633762316366303363363165633065336465336134 +33613666623835666631333962373338386130626462343930303938313262616165656435383736 +32326531356231613836366362643361643966633037643664623365636137363363346636363234 +62333735666236643965623031343337316539376562336230343361376336396635356437343664 +33653834313031313561376636376334656235313665323166636336613532666163636361336237 +35333434626366623333653466306236383263316339363062383931356636333666303535636536 +34643738316662306638323261666632613239333834366538366464663862343233663139386436 +30363261646366346139366566633162303564373264613833363065303061633336323865353564 +38633665376131623433623732666334616563363535636537633837636537376564616330356266 +33383262303035663532666662653731353330393964303465623436616262396635356363313466 +30633039393537653862383133666235613931336439373166316331396663656262643830653065 +39366639636163356239663338393737626639383063313337346464393161613564303931633938 +32303966366632643537373231363634636563376438653733373537653734343734636566373561 +35393632646432643564306333653163386561316535303865346362346162316333323364643332 +33326564363163313733306365313363376564303963303030313530383965623663323936383531 +36363233346665356562326662643764613531396561373637383331336434333337316135633633 +31323831636232663534366362356532613338663262613562363533306436356430333466393566 +36613763613239376663316365393031356239316662383738313739336234643132303935306666 +33653739333365393562666434356437393738613932346235323864313139633965396338323536 +65613931333965663438613738623030343633313837393666666263313137656230663634653065 +34663464393238383332386434353533393030366165656338396431336534653761626430303164 +62336435336162653433656239623235366139376663626263623531356230343166376463373133 +39383364386562333639306534346232656665353330666335386232633135313462313538303663 +34336363303731613035323833346265386132376438396539363631353336326639643064326661 +36353534623064633330626531303836353030363132366135336462363739373462346533646338 +33393430636233393764656432376466316531613130386565626638333965366664653537383838 +65373138306133303564323838353664303230376330383234373936386130366536353133303836 +62376635376332326338326234313562663766336430623562373834636533643533383763376264 +36386334623837363864613338323830626364653262643433663132653133633730306366666636 +34396237313336313862613131383565316439373337323936653730393261373766633537373632 +65663866343030623165326465623731373064343239386237663462643933353037303166316533 +38393138373864626161366364386434336662613234326332303564343231376262303634313832 +35653536653733353736323938616135336339653664353739366137626634646635666466326366 +61343765656437313465383433623139313935666565313638663831353066633133613039636566 +66653663396630303931633839643464363436663265323231633166343338383161366662346561 +61393231323738373930346135343334303861326532393532646537363233376637623037626266 +30353734666130333638656264653438613436376436326337366633303037656335616530386134 +63373665613233303166623733343361333739656265643031333533613432636639306233353239 +61336636393934663435623935386234646433393434333438613732663166613633353130363432 +63373131633537616130353064306663646131343164656165313665343730323337326639343863 +64373230353365643538626465343965383938643031393366313533396436646138646136616661 +66643433346535626135393364386161343532653463663363323462363036653533393563646166 +63313530613161313037656436303630643837643965636362343939353134323937306339323234 +66626539376338663531666634636335613133303234646164313439393038623232313234396366 +33393731653335383464356534356337323561623765666332353036363734376535646437313036 +63666434323266633339306135373535353135626233363535633133336639313262333035383063 +32313938623166346338323833386139613630656664306132626662376135343132353936656162 +39613030346235336530623930633730393237326137656230356238643634393666373063656265 +34373439326665663664336437303263326239313233363162666238623838643938643332373665 +38323362373336326539326132383230343562393366316162323834343866616161336438663732 +64303865653137633332323565656466373564616339623335306565336662656465303634356264 +66396563303461356637656666643762646137343735613732393435366635376232633939616364 +33623164666632633666626638633131616230336136323234613037616165643536323638636464 +38643432383331616338333731623034353432663534626133613537663166666562633234386634 +63616234313736393864346634643265656364666230643737626261353964346438396130346435 +33333534346362326664353037323733316364396264616537646563353439303632316533376632 +63323237316630626663663231313238336430396564333963623835666166363137396333326462 +34376165663739326462313239356561393563333531383430653161373630363737663136626465 +62633535636562303935336461623464383638626663373163623562383234363331636463333137 +66316364336261363930323039366431643838373865373430643761643937326435383032623030 +31336334356431626333323865373132396332653032363962343539633830333532396262326334 +31623839306632353137373865366135626538363039316233383136336465303634303164373763 +63616233363264383235393836636536326364396263336562363136616336363036346563616563 +31303562353936616461336563323237383230643637386332616339663666363261613539373764 +32613063333031636562633738613036646136666336653266373035363731353430383065356239 +66333633373561616236383534626135336362626239636434383865393031646264353734666534 +62336230623337396563636263623834306461343261663237656565396266393662616330633836 +33653230643234386462373763366334383330636132616664613535333836616238353435306632 +33336639313438343835633035646137616430613636323630643334616135343566343336383763 +30316664313364373064303538666337623865613361353334663166646437653137396436616334 +30313835623462383230343339353061656566376432623965343466636436343330316664343162 +65633065393434393631386261306166343761326166303636336630643230626662393864336436 +62653363356364343939396165383434626462613061393138316663326339623964393930663733 +37633137633235323264303932383764326331383533336663333762643539613732373834613034 +36633330373433306432663762393362333064656336323734643238613561666137326661653265 +63386331393163356335393132636531643339306165383235323935363135303432363165393836 +62646631383238363466653339633034313833626335363137353035386361343366303461333965 +64653237613264623033343231333130326237386266323232636436303731353834313834623738 +34316364643231653235313364346336373166646361366265646338656632323338613931336636 +37323032313039346637663939326663666633373364383634653438653136663537636162663761 +38653430643163363731646266316263346566663663623931663866386466613763353738633830 +36616335366630613532396533343730366236636431343335373662643833613563613435313564 +62343336343832376232633534373532383133626462613832653538373532313937313233616166 +33663936633661626466386536323131366138303164356566643536373930633763633263363366 +31643165303931663766386431356131636231363761356339636163393831653634323861363139 +65346134373230316434643733363333366236333135396537646533373962653834343130623239 +61383933323565616462636134313261376337353964336133336433343237626535633162316137 +37626662306239336233363338636130646330313637336236653739386435383864643637393733 +37383864613739336565636638343535303038653666383937613866346236383330356338363135 +36613939333632636632303062336531376137323566383238393937633531366437323234316238 +36366365326264393138653663636438333533316264383764353638383436383638636430653231 +65383766386564336435623532376130366439656461383937316533373839663762336466306365 +62386337303064353265346333376566386639663666623635343530393231323763376333396331 +33303030303335346239343236376239633962343664646635383363346363623832636536363432 +30383432323866383861346138636661306165393364616164353437316364393038376166663765 +31636630323765346537333361633939666161666631626538653063383132653063346637333938 +31313536363364616533363064336332313733656139646664626264333763663465386266663537 +36393561373134626365346434353830316634336265633864326233656365343233636661386363 +31363539333735646235666166653335366465373563366463363264633261333161366334343363 +65326439393236623537623236323333376366336435616130336531363563663635316263353838 +63616162623864663938663137373863656262343736383130656537663737646236343438623135 +31333030386231643461613231323238383366663263646138363965653538363333313465326361 +33393462643436633736313037393565613531663132393736386333386333393564316536633036 +36613336393236623363353263626333303039393863363465353463396564396130313739373061 +33343438383132323265386266636638376432313663666139343034313233643362366239623864 +33373963346239386135353964316262383235663331623234633239663337343636373264663032 +61323936326663366661323139353830303435336362333631333661396433396137616635666665 +62383939333761613062636533353537396465373130383461313565386335326136333761663636 +30656334653934363262313666373663383036666364303735303136653532333361346466646661 +62386433366434383334363339306265373862326133323262643630633030636666386433336638 +65666538633231343033376130396533313364663734656634346338656162353233313133393330 +36396566666132366630663632396662653733653763666639383335333235373932323766623833 +36326333653762356330383661643530643963303662303465376430633163636539396137623733 +38396566306465313362376466623736346531643962323038633538396437346664356532376562 +39663237663638386132373335316536356333376439633462373830346132306464363036343864 +35376136363063383861643461393462343964323365366266373738656638306463393931363936 +31613632306235353562323562616332343536636330363065336263653166313066346439643665 +66663864363765636462333235323365373237646262303461313061663031373932323065386435 +31663065333131383935356130303063306166663338373664303838316236376237366663613838 +62623636303866656265336138613037333562616339363737316631363436656565623163326234 +38353064356236356366366264656235343639623565623233343235303230363839386538303331 +63356131643334666162323962343632346134626438306233316562373439333963363033353865 +62643539633736386433313533386137383262313134353938666637336434333038363163653033 +31633233373765383939383336646531303030653339653764383332343636343236383636396534 +33356433643832303638303466313265313037386562366532336631393233373133383764653136 +36373336333064396439373234653836643364616535323265616664386135373833306463633565 +35393565383965373436356565663335373263386132663730633962306431363035623038376337 +39633064653138353232313631306135666265383766636339306462356463663834643038393966 +31393336636565656365366264363831323064613439633138623438616266326463633338633766 +32353734306439363534363063326361376164616534313363626138373064316339343761346339 +35376238306436633430333931316339653834363164386337343637653234343332386539336463 +39643361306331313364626238323634633435633730313339663333313533366264303261303165 +64353161636234663662663734333134343266353163323863373938356539303539363539653533 +61353665363266376561383032323937623265363962373731653432383134343532373734306563 +65326566306137303966396238653739343463343062363065623938323339653636356231623666 +30393230373432356263313163616535636131666364376264636462333561326166326538633833 +35613162393130643765393662373738353731333131303537303730386435303839343763326531 +62336662313862613232633563373637396232616237643966633666383335313734613736346232 +31353664323433383339343565643761333134383630346534323534303266633933623635636430 +36653335633964643463666661613763373562643832656139326233386635353339356636396566 +34326132643738363931386137333533343634373938383430393633386639366536343163323530 +33353238663233336332383936366463353530303330656634656439666130373065383766626362 +35313161626233386533313236663131373761316464623961323061643833653338376136316234 +30656336393563643664343436363465366564633830346630303062663237366238656136333531 +34396238663135643964303938326231636163356533636331646663623033613261323462636661 +66303762613163663061343936623337613762343766623036316436343564366439333666653036 +39636338323933303038623765653431616434623930663334623031653835373164653965393561 +38666339633963303536346534646134383064306332613233336462653961653337336162393864 +66626239646437316264343134336662613835663131663164383839323733313031643635333537 +36663936353366666235333532623532613031356336326665326632356363366366643562633736 +65623033643339646137346361383634373462646431626630333639653162313932666437353864 +63303330323339396139663138626166316234613061656439653837396638616536343034646432 +63383662336131323533393465613231663038613665393433366365373432643966643632383934 +37613566623934343731333364636437633331343266353830343532383339326165653962653435 +63666361613639343033623133353435666162613563363236386438396439353333366466626663 +39363634623436386562306433633166636366393864643461326132336438333932613762313234 +39653434333432323735306333666336383238303930353963313961613238386665656264363132 +30633562653437353234326665366261646134396330643133393265313162333731333238623262 +31626539666131363638356434633261383064313133636162373361663232643531336633323165 +30326665376365333062376562353864616136373362383335636131386237666336653966366332 +31323636626238623164326461366139613430363535363636623530306462336566376466653666 +30363637386236326262633766346439656361333234613663323963376665306631396266356436 +64306161373439386263396162303835303561326333326663383338383237323563376230393661 +32396165376335663362313063366439346361613566643264333836666332663533366636333636 +32663138396533323332653066616265623732623836303663313930376333386461346531663338 +35633766653161356639346639616433383435303132353538646638353566313461376561313435 +33613763656162323530373534376139363962373965396266616463643336373537346630366265 +39376430326166346132663333333863313730376539353833343663306364616134396334663131 +39376338313064633362653533666631633162666638316137346162663264663639333865626262 +35306262376239373963383032393735313064356564653761316530316133383266313333346432 +30313935366665623430623234623061626265643166306266666161663534313033383831376433 +65393932666464363466326166383131343236363161646233613561376166323137303031626239 +37643631633539616339656431313935323961663434343730373333373863306537313232626335 +64366365643230383265643837623637376430613837386533626331666636323837396238366438 +64363732376436393938616165313237653633386165393033656630616666316362623836376632 +39643835313937646631326239333937623833343533333462313231656536346632643836633736 +65356431663235653332363032353839653666623765653836333265373464356265336639666632 +38316563343666636430636238353936313565356537646136333036326566633965393632316339 +38656134353032313136383239653330633533373565346539663366623433633864643065333632 +61623034613836333235396138643534616365313634346532336238656535396665633831666563 +31386166663934616434643239633533323062643935663338663064333463626466373161336632 +32356564363431386638656261623264356561656264326531646461363531663338636231643633 +63616438393061303139623631336364646334303130656163343236383862616639363761666263 +33303533623532366233313062613232393739303562353235313861383935656638623733396431 +38353264633665363461663733613631613133316366343337363039383030306261336164613835 +37633662323437353635303734613934386438636331316433623364353965336561636432336566 +35623736336431396662343235353232643766303365666632663464326239666133393661336535 +37363633323934383265373239346230643534363732373032336331353736363034313636656635 +36623538303462346266333965646633323536303532393238643237376239383162316435353332 +61396263383037396438343162363863303964396261626431623137666164333233656162336530 +34326630353436323962393830623030376465643139653738356432303033306237363938616264 +31633835346439363736316665623763346266343937303331663065393738316631613335613637 +38333338343562633230316534373061376238393561393766333365373765313865333961653836 +34376334623739396333663536666230353361313665343464386439393630393835313364333062 +34613236333139343961633839616662333030613330626662636464383264633739323863326634 +37363939373435333132633738643232316166366437346162353566633437653138643037313030 +64303264356334656165623737343265333033373638653231656166653863626262396635646362 +31383236383562613032636363336664636337633038663732636139626436643565373564336439 +32623865386162663961326636363330366562353564313139633861626365616363653064353533 +31613435643463643034393961623031356633313336353139396265633263336463373235636363 +66323763363336306536653735313463333061643333366661343361353664333938336438376636 +37643635393330333130666563373232306562343030616630663361313566383137613237336534 +35663539646430663265623334666235636130363762616636343865636437356537643032313564 +63643262346438653863626333393933616164343464663630393864363766346138323834343333 +30653964613161356565313638383564656165623430313562653463633966653538376164386532 +38373763663161666438313331643237353461373665323262656230356634643264356366656436 +32313437626162626562353065333434333061636434393538333066666431336435363865356361 +65343731313465306562353833656437636130656166323736623161656638313361636664363236 +63386632353662366232643363313533653733363366393661643861333534613937326661653861 +61343436333434393766326438353830623231373761623837633230303337393564323839636532 +35633561633838303366316262363065376461643834363865643037306466656236633630633239 +63663966366432353837623763386363313735323738356435323361643533376531346636663139 +65653331633731663032393638336463656235363537646136613566623261343163623231636264 +65646464643266373138386266313032663537366636306238376230326230306165633632376537 +64373264353766663634383833363364346566626164633835313766633266333063303264666265 +64376634646432313863313566313331646531333138333832373361313263613533656138333830 +64376363636236643234303830343933376538303164613039336534666561313235373762326630 +35633132303461393534343430336663663162643265316466386435303238613066663431313535 +33663361376231343831666164623532613863663236396330613163396437323939373362616534 +62633063623261653138343563653331613962303730343662646331313935376135396235663734 +32356361633463346364363864393366346532363831636166316466663237623664623662616631 +34353436373837386331353334656664316262333436633464666336303364653237363536313035 +37386631393563346365366234306439306436333162393632663866663764363566313365363964 +39643837386434303466386334326639323233643861643633346532323336663736653838373337 +366534636438363631363637643763393138 diff --git a/ansible/host_vars/backend-fsn.ooni.org b/ansible/host_vars/backend-fsn.ooni.org/vars.yml similarity index 100% rename from ansible/host_vars/backend-fsn.ooni.org rename to ansible/host_vars/backend-fsn.ooni.org/vars.yml diff --git a/ansible/host_vars/backend-fsn.ooni.org/vault b/ansible/host_vars/backend-fsn.ooni.org/vault new file mode 100644 index 0000000..4a48e1a --- /dev/null +++ b/ansible/host_vars/backend-fsn.ooni.org/vault @@ -0,0 +1,78 @@ +$ANSIBLE_VAULT;1.1;AES256 +39653265353731373139626362626462376535306531333636396461376435323632323665623532 +6566663563616635636663653438616530326333366532640a376632393532613338386635393662 +63626637303861653236353137303637323930396132646632626163303739353931303464633137 +3661306234363035310a303739306561653531646437633036653639323337303561343339626530 +34646636666538376264366361383137653665373730373136353461626163346366356436336461 +36653266366363633538623330363062393562363864663765636665636135626563373238663239 +63613730386537663166386663343935303064323064303438383635363238623866663031316339 +36346236336265336662396130373333643235313237393539633066396538633661666630616637 +31333935633165333536356665363837393736306361643864323662666461313764363364373737 +37326137633631653332656461636534313035336132613239613037323364626234613136643232 +34326164326339303439323865303836313666613136383431343235653739323361366163643566 +30643463663162363830333264376138323339356430656265626565383236383430636632616365 +32343665633965636566333162386135323638383964396633326139623039313434333836303132 +66323461666335643363646265643862346332653536623433326364353766323337313836356534 +35336563373535653436333735313836636336353433623132653136663530646230363931323762 +33313861663965626465323665656431386134326465393033346239613862643236336230336632 +66336662643435333563643437363762333030326333383738366439653233393132306136326561 +65383239663434626639376632306166333934366235656438653261646163383164323533636430 +39636133366131323635623039396236343634613964636237633036343034363038336237323736 +37616434633231323866643464626437343461663537646563303437653830653934656162343939 +35316233316661356161303937633238633733633030396464636230353530353031313437366231 +32343730663065656439356639316261623132656365663730333634363164343836653032393536 +65313935313763346362623365646135333363303230363262313334306439333139306264313533 +32326564353563656235383034633131316238623735383438633864663661633632323863613961 +37376432646337633432623837383131363131363238336239353833336135343031616335633961 +34303737393864346132636336366331643634363532373763316631383439383437316536623137 +36643763653734666665363663643066333539653635643565306139383336383338353634613563 +31316437333733636534316338353837626638636530313865386630623665663630643630356430 +66313637383536386663666532306530343931616139316232633437376532626431383562333861 +31636161323431323864636265323337626236383632373436393236363334313031313733613532 +64346137636535373061313336333661363162663734643862663338353839326436653631663265 +36646565363934356636316433393166396638643563663637363232323461623833633565663935 +32326561323535636630396362653839343835316265623935393135633038323737663030333733 +31383264353265343132613764356234666235316537323539393366316334346261663639663564 +37343737306330613962346261373963643162353739366361663965666338373063323462646435 +32646439323738396134373030343539623264616164656136393136613434666332356533333031 +61383866616361313666323238623464363162353634643630323663386263623764623234613062 +30643565396431653736666633313461353535616335663262343038383634353362363962663131 +37313331663266353133396166323232643361343236363937306437636337653061356130313461 +30396465653461653766326236346236623361393636353731666532333332633430383732656131 +64373931306362333237383635343437323139386539356136363532343439373731333939393063 +33653238626465383134613066306337313930653036623337363863333330303163363338373138 +30623635643638383165393134613834346435333665343531376464646663623733623834333161 +35643732366636656262323931383236333134623931343038353531643234336665663261306231 +66313735306332613962613936366535646432363633656430666262346362376566333163353333 +63646630346536653230313361353131663037393965656466333561393935386132316166313064 +63613334626339396266336364656161623862313234313135626335643434346638346632353265 +32383639393230373464323038623462376362663538373933393863303132653536633665646662 +64353333303037666339356262646536613838626633343635663632653733363036636265396637 +64396461636435336235303261353636353839356265373035643934383034616161326130363363 +37306431353639363537393061326163326561313839356465393665626666653634656265306533 +62343838623866663766386131313839363636656635613639323437646335396534653535356364 +64303239366361623830353231656332653838313866366339333939373862653866633639333935 +37613663636130313538346161393830613139333937383239623437616465396536633037303534 +31303562386666383464393638666631613265653331366135316434653431356361646163323061 +64613963353733363138393661653966303633663638663037656566343333316435366337353362 +31346463343239636132653163653631353063616362346236326331366562653230633132643363 +37376236656261663436323230393634386266343761343165343965613532656238353663326237 +36336435333162616465346362393264616635316362366364653537663031656231643566373733 +61303964626166333535306530633136363461623034636661313161343961386432626231363335 +36316233653734366462323639633034353165363163626363323330343136303235326531353030 +62653131373533656466613165386139663836373062313632363137393130353965303633656636 +31376264313431656338356330653133663339653337326331633733653762663266383634666261 +34353832616265643436323237396638393966643135343539333332653137363863653232636435 +36643266623139313238336333336637323139633565393633346663666666613338353561363962 +36353663313934643034386661383730343836653262333832353263636161633232393431383465 +65323233343362363931303339386639643930313433656165666436393766666435653230393733 +38643466333365306566373631333261363461336462613134366138383764633031623633343461 +39616561306366343339383766623265643364623532653866316131363234653632653765343936 +65363131383630363163396433613537646634613135353639386462333962623333386461633863 +33663963323632303961323138666634356134373035663335663263303333343036356335396532 +35303164623234353566313237313363373630363536303766353561653935336466336263333534 +36366230303230373137663266343166626562313835386164613362386436313737633536353165 +39376164653366373161326434623639626337366434626564663433656231316334653336313630 +65326265336634623965666333303132643564663838636662333664633864336163313537313262 +65666366636430663235333766356437653436303936363366303734303661633431343366306530 +33646635343336386337323561316137666530653337663365323939613264363935 diff --git a/ansible/host_vars/backend-hel.ooni.org b/ansible/host_vars/backend-hel.ooni.org/vars.yml similarity index 90% rename from ansible/host_vars/backend-hel.ooni.org rename to ansible/host_vars/backend-hel.ooni.org/vars.yml index 1442096..798f5b2 100644 --- a/ansible/host_vars/backend-hel.ooni.org +++ b/ansible/host_vars/backend-hel.ooni.org/vars.yml @@ -4,5 +4,4 @@ account_id_hashing_key: "{{ vault_account_id_hashing_key }}" github_token: "{{ vault_github_token }}" mail_smtp_password: "{{ vault_mail_smtp_password }}" base_url: "https://backend-hel.ooni.org" -tor_targets: "{{ vault_tor_targets }}" psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}" diff --git a/ansible/host_vars/backend-hel.ooni.org/vault b/ansible/host_vars/backend-hel.ooni.org/vault new file mode 100644 index 0000000..1f97c3d --- /dev/null +++ b/ansible/host_vars/backend-hel.ooni.org/vault @@ -0,0 +1,30 @@ +$ANSIBLE_VAULT;1.1;AES256 +32343262656431363066623434666166643434303861636663643039383630316330656132643735 +3838653761363233623336373439646362643232613564620a393739633764393762306365666265 +39656633323133353231353732643433353662656666626232663234313732363739356466303432 +6564353734376231330a383236346534653166396237353430663631316265643764623233323436 +33373331626632613361633166313163393235623634633965343531653831343138343862366461 +34656233306532353038316636636538393436356264333338353636376231343531633566356665 +33616230393565353030336236303964666463623163633764626139363236663433393366653362 +35393539623434643232373636326232393738653330363364393031626238653735663134623431 +31393463633635623636663638333962343530303635653862616231663465303864326531643961 +35363032356265343734636432386265393530613438386636353664636338633266636130616264 +62663831353735663732343637306435323163303865623964633662306163643838336437333962 +33343534393135306165373534343631343864353561333133653765313032326531363237306564 +65323939626436373839306536616630393633613835633632326530366333663562383562353663 +30396135326239343263323666376632303835366139393137643263326338303365613039313630 +61346137646231353463396566376634666562323539623833323334306663623332626538616132 +32363765306234363938656364643364333061616364653932363237376337366632663533616634 +37633262653130313334643239616535613365353261353336396464316532366134376461376232 +35336435653233643634646166333137656632386361623533313133336638393361373762356637 +30373264626366306333643932313834613263616161656630646666626233393330346236333032 +35653531313534373565663938316637336333366633323863396431623966313038353162333534 +65343266346538656535353132376636336337626630653933313361623736623934376361623865 +64373365396436653762393438666236336433653334343035336265396239303938646231333738 +38653334613335363132343739343565376665636136356534636331326535326264343062346433 +64623038626536346435323031383934303138396363666633393465636232356463383231663937 +30383263396565313136316131386535316337333166303133613430626332363537326539353138 +33303565323031396261336533616662346663636132653739663965303930396466353339383836 +35376137333962333331613736623337656664663565326435643164356166336566653665623036 +30363739363064653231343165363966333238366637626166633165383535326130343135663931 +6636 diff --git a/ansible/inventory b/ansible/inventory index 6861739..cf2d7a0 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -25,3 +25,6 @@ openvpn-server1.ooni.io [aws-proxy] clickhouseproxy.dev.ooni.io + +[backend] +backend-hel.ooni.org diff --git a/ansible/password-pipe b/ansible/password-pipe new file mode 100755 index 0000000..e2d14df --- /dev/null +++ b/ansible/password-pipe @@ -0,0 +1,2 @@ +#!/bin/sh +exec gpg --quiet --decrypt --batch <~/.ssh/ooni-sysadmin.vaultpw.gpg diff --git a/ansible/roles/base-backend/meta/main.yml b/ansible/roles/base-backend/meta/main.yml deleted file mode 100644 index 5de9bc5..0000000 --- a/ansible/roles/base-backend/meta/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -dependencies: - - role: adm - become: false - remote_user: root - gather_facts: false diff --git a/ansible/roles/base-backend/tasks/main.yml b/ansible/roles/base-backend/tasks/main.yml index 00a7352..5857e92 100644 --- a/ansible/roles/base-backend/tasks/main.yml +++ b/ansible/roles/base-backend/tasks/main.yml @@ -94,8 +94,8 @@ create: yes block: | add rule inet filter input ip saddr {{ lookup('dig', 'prometheus.ooni.org/A') }} tcp dport 19999 counter accept comment "netdata.service" - notify: - - reload nftables + notify: + - reload nftables - name: configure netdata.service tags: netdata diff --git a/ansible/roles/dehydrated/handlers/main.yml b/ansible/roles/dehydrated/handlers/main.yml new file mode 100644 index 0000000..677c5f5 --- /dev/null +++ b/ansible/roles/dehydrated/handlers/main.yml @@ -0,0 +1,15 @@ +- name: reload nginx + service: + name: nginx + state: reloaded + +- name: reload nftables + service: + name: nftables + state: reloaded + +- name: restart dehydrated + service: + name: dehydrated + state: restarted + enabled: yes diff --git a/ansible/roles/dehydrated/meta/main.yml b/ansible/roles/dehydrated/meta/main.yml deleted file mode 100644 index e7e996b..0000000 --- a/ansible/roles/dehydrated/meta/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -dependencies: - - nginx-buster -... - diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml index 0bfaf7c..9e41642 100644 --- a/ansible/roles/dehydrated/tasks/main.yml +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -42,10 +42,8 @@ dest: /var/lib/dehydrated/acme-challenges/ooni-acme-canary mode: 0644 owner: root - -- name: reload nginx - tags: dehydrated - shell: systemctl reload nginx.service + notify: + - reload nginx - name: allow incoming TCP connections to Nginx on port 80 tags: dehydrated @@ -54,10 +52,8 @@ create: yes block: | add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP" - -- name: reload nftables service - tags: dehydrated - shell: systemctl reload nftables.service + notify: + - reload nftables - name: Configure domains {{ ssl_domains }} # https://github.com/dehydrated-io/dehydrated/blob/master/docs/domains_txt.md @@ -93,16 +89,8 @@ name: dehydrated.timer state: started enabled: yes - -- name: Run dehydrated service immediately - # creates: - # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem - tags: dehydrated - systemd: - name: dehydrated.service - state: started - enabled: yes - -- name: reload nginx - tags: dehydrated - shell: systemctl reload nginx.service + notify: + # creates: + # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem + - restart dehydrated + - reload nginx diff --git a/ansible/roles/ooni-backend/defaults/main.yml b/ansible/roles/ooni-backend/defaults/main.yml index 33ce9e4..d25ddf6 100644 --- a/ansible/roles/ooni-backend/defaults/main.yml +++ b/ansible/roles/ooni-backend/defaults/main.yml @@ -3,9 +3,11 @@ ssl_domain: backend-hel.ooni.org collector_id: 2 clickhouse_url: "" # fetch from aws secrets bucket_name: ooni-data-eu-fra-test +public_bucket_name: ooni-data-eu-fra-test collectors: - "backend-hel.ooni.org" fastpath_version: 0.86~pr831-395 analysis_version: 1.12~pr836-413 deb_bucket_name: ooni-internal-deb deb_server_name: deb-ci.ooni.org +apt_cache_valid_time: 28800 # 8h diff --git a/ansible/roles/ooni-backend/meta/main.yml b/ansible/roles/ooni-backend/meta/main.yml deleted file mode 100644 index c82f9e2..0000000 --- a/ansible/roles/ooni-backend/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: nftables diff --git a/ansible/roles/ooni-backend/tasks/main.yml b/ansible/roles/ooni-backend/tasks/main.yml index 63ee471..c3cbff8 100644 --- a/ansible/roles/ooni-backend/tasks/main.yml +++ b/ansible/roles/ooni-backend/tasks/main.yml @@ -25,15 +25,10 @@ group: ooniapi mode: 0640 vars: - collectors: {{ collectors }} - # bucket_name and collector_id must match the uploader - collector_id: {{ collector_id }} - bucket_name: {{ bucket_name }} github_push_repo: "ooni-bot/test-lists" github_origin_repo: "ooni/test-lists" login_base_url: "https://test-lists.test.ooni.org/login" pg_uri: "" - clickhouse_url: clickhouse://api:api@localhost/default # mail_smtp_password: "DISABLED" # jwt_encryption_key and account_id_hashing_key are taken from the vault @@ -54,10 +49,6 @@ template: src: templates/api-uploader.conf dest: /etc/ooni/api-uploader.conf - vars: - # bucket_name and collector_id must match the API - bucket_name: {{ bucket_name }} - collector_id: {{ collector_id }} ## Nginx ## @@ -83,7 +74,7 @@ tags: deb_ooni # Uses dehydrated template: - src: deb_ooni.nginx.conf + src: deb_ooni_org.nginx.conf dest: /etc/nginx/sites-enabled/deb_ooni notify: - reload nginx @@ -139,8 +130,6 @@ owner: fastpath group: fastpath mode: 0640 - vars: - clickhouse_url: {{ clickhouse_url }} ## Analysis daemon ## @@ -167,5 +156,3 @@ src: db-backup.conf dest: /etc/ooni/db-backup.conf mode: 0600 - vars: - public_bucket_name: {{ bucket_name }} diff --git a/ansible/roles/ooni-backend/templates/db-backup.conf b/ansible/roles/ooni-backend/templates/db-backup.conf index 4302f0e..0dd0693 100644 --- a/ansible/roles/ooni-backend/templates/db-backup.conf +++ b/ansible/roles/ooni-backend/templates/db-backup.conf @@ -4,7 +4,7 @@ "public_aws_access_key_id": "AKIAJURD7T4DTN5JMJ5Q", "public_aws_secret_access_key": "{{ s3_ooni_open_data_access_key }}", "public_bucket_name": "{{ public_bucket_name }}", - "clickhouse_url": "clickhouse://localhost/default", + "clickhouse_url": "{{ clickhouse_url }}", "__description": "tables can be backed up as: ignore, full, incremental, partition", "backup_tables": { "citizenlab": "ignore", From f51feb36b3bd7e7643007be4445f355e80da56cc Mon Sep 17 00:00:00 2001 From: decfox Date: Fri, 20 Dec 2024 11:45:02 +0530 Subject: [PATCH 20/20] fix: add password file --- ansible/ansible-playbook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/ansible-playbook b/ansible/ansible-playbook index ffbc990..3b02bae 100755 --- a/ansible/ansible-playbook +++ b/ansible/ansible-playbook @@ -8,5 +8,5 @@ ANSIBLE_SLACK_CMD=`printf "%q " "$0" "$@"` ANSIBLE_SLACK_CMD="${ANSIBLE_SLACK_CMD% }" # strip trailing whitespace export ANSIBLE_SLACK_CMD -ansible localhost --module-name include_role --args name=notify-slack +ansible localhost --module-name include_role --args name=notify-slack --vault-password-file password-pipe ansible-playbook "$@"