From 5d81f3ccd498537d818a19b83ea5e8ed36c16d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 16 Dec 2024 18:38:30 +0100 Subject: [PATCH 1/4] Change storage type of postgresql --- tf/environments/prod/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 48902a8..b460195 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -159,8 +159,8 @@ module "oonipg" { vpc_id = module.network.vpc_id subnet_ids = module.network.vpc_subnet_public[*].id db_instance_class = "db.t3.micro" - db_storage_type = "standard" - db_allocated_storage = "5" + db_storage_type = "gp3" + db_allocated_storage = "20" db_max_allocated_storage = null tags = merge( local.tags, From 6d9d6e3559e4f80d57560b31f2c52423c92b91a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 16 Dec 2024 18:41:58 +0100 Subject: [PATCH 2/4] Bump storage size to minimum value --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index b460195..f3a01dd 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -160,7 +160,7 @@ module "oonipg" { subnet_ids = module.network.vpc_subnet_public[*].id db_instance_class = "db.t3.micro" db_storage_type = "gp3" - db_allocated_storage = "20" + db_allocated_storage = "50" db_max_allocated_storage = null tags = merge( local.tags, From ab129d610f2070f3a9b9234682ddf8b90fc1bf95 Mon Sep 17 00:00:00 2001 From: DecFox <33030671+DecFox@users.noreply.github.com> Date: Wed, 18 Dec 2024 22:08:50 +0530 Subject: [PATCH 3/4] refactor: drop monitoring for db1 and amsmatomo (#130) We killed `db-1.proteus.ooni.io` and `amsmatomo.ooni.nu`. We can therefore, drop them from monitoring. --- ansible/roles/prometheus/vars/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index e77fe31..d8774c4 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -1,8 +1,6 @@ dom0_hosts: - ams-ps.ooni.nu - ams-slack-1.ooni.org - - amsmatomo.ooni.nu - - db-1.proteus.ooni.io - doams1-countly.ooni.nu - mia-echoth.ooni.nu - mia-httpth.ooni.nu From cd8f0c62802a33df230d599be2b8aecbc76296b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 19 Dec 2024 12:03:48 +0100 Subject: [PATCH 4/4] Add monitoring (#118) Migrate over the monitoring role from sysadmin to devops. Related to: https://github.com/ooni/devops/issues/27 --- ansible/ansible-playbook | 12 + ansible/deploy-bootstrap.yml | 7 + ansible/deploy-monitoring-config.yml | 10 + ansible/deploy-monitoring.yml | 10 +- ansible/inventory | 13 +- ansible/playbook.yml | 9 +- ansible/roles/monitoring/defaults/main.yml | 1 + .../monitoring/files/create_logs_table.sql | 56 ++++ .../monitoring/files/log-ingestion.service | 17 + .../roles/monitoring/tasks/log-ingestion.yml | 85 +++++ ansible/roles/monitoring/tasks/main.yml | 304 ++++++++++++++++++ ansible/roles/monitoring/templates/10514.nft | 2 + .../roles/monitoring/templates/clickhouse.gpg | 87 +++++ .../monitoring/templates/clickhouse.sources | 7 + .../templates/etc_default_prometheus | 4 + .../roles/monitoring/templates/grafana.gpg | 41 +++ .../roles/monitoring/templates/grafana.list | 1 + .../monitoring/templates/grafana.sources | 7 + ansible/roles/monitoring/templates/htpasswd | 5 + .../monitoring/templates/jupyter.service | 37 +++ ansible/roles/monitoring/templates/nginx.conf | 203 ++++++++++++ .../roles/monitoring/templates/vector.list | 2 + .../roles/monitoring/templates/vector.toml | 24 ++ ansible/roles/notify-slack/tasks/main.yml | 8 + .../notify-slack/templates/notify-slack.j2 | 10 + 25 files changed, 946 insertions(+), 16 deletions(-) create mode 100755 ansible/ansible-playbook create mode 100644 ansible/deploy-bootstrap.yml create mode 100644 ansible/deploy-monitoring-config.yml create mode 100644 ansible/roles/monitoring/defaults/main.yml create mode 100644 ansible/roles/monitoring/files/create_logs_table.sql create mode 100644 ansible/roles/monitoring/files/log-ingestion.service create mode 100644 ansible/roles/monitoring/tasks/log-ingestion.yml create mode 100644 ansible/roles/monitoring/tasks/main.yml create mode 100644 ansible/roles/monitoring/templates/10514.nft create mode 100644 ansible/roles/monitoring/templates/clickhouse.gpg create mode 100644 ansible/roles/monitoring/templates/clickhouse.sources create mode 100644 ansible/roles/monitoring/templates/etc_default_prometheus create mode 100644 ansible/roles/monitoring/templates/grafana.gpg create mode 100644 ansible/roles/monitoring/templates/grafana.list create mode 100644 ansible/roles/monitoring/templates/grafana.sources create mode 100644 ansible/roles/monitoring/templates/htpasswd create mode 100644 ansible/roles/monitoring/templates/jupyter.service create mode 100644 ansible/roles/monitoring/templates/nginx.conf create mode 100644 ansible/roles/monitoring/templates/vector.list create mode 100644 ansible/roles/monitoring/templates/vector.toml create mode 100644 ansible/roles/notify-slack/tasks/main.yml create mode 100644 ansible/roles/notify-slack/templates/notify-slack.j2 diff --git a/ansible/ansible-playbook b/ansible/ansible-playbook new file mode 100755 index 0000000..ffbc990 --- /dev/null +++ b/ansible/ansible-playbook @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ue + +## ansible-playbook is a wrapper script used to send a notification to slack +# whenever a new ansible deploy is triggered + +ANSIBLE_SLACK_CMD=`printf "%q " "$0" "$@"` +ANSIBLE_SLACK_CMD="${ANSIBLE_SLACK_CMD% }" # strip trailing whitespace +export ANSIBLE_SLACK_CMD + +ansible localhost --module-name include_role --args name=notify-slack +ansible-playbook "$@" diff --git a/ansible/deploy-bootstrap.yml b/ansible/deploy-bootstrap.yml new file mode 100644 index 0000000..81dccab --- /dev/null +++ b/ansible/deploy-bootstrap.yml @@ -0,0 +1,7 @@ +- name: Ensure all hosts are bootstrapped correctly + hosts: all + become: yes + roles: + - bootstrap + tags: + - bootstrap diff --git a/ansible/deploy-monitoring-config.yml b/ansible/deploy-monitoring-config.yml new file mode 100644 index 0000000..0c27527 --- /dev/null +++ b/ansible/deploy-monitoring-config.yml @@ -0,0 +1,10 @@ +--- +- name: Update monitoring config + hosts: monitoring.ooni.org + become: true + tags: + - monitoring + roles: + - prometheus + - prometheus_blackbox_exporter + - prometheus_alertmanager diff --git a/ansible/deploy-monitoring.yml b/ansible/deploy-monitoring.yml index a1eadee..2fd7746 100644 --- a/ansible/deploy-monitoring.yml +++ b/ansible/deploy-monitoring.yml @@ -1,12 +1,12 @@ --- -- name: Update monitoring config +- name: Deploy monitoring host hosts: monitoring.ooni.org become: true tags: - monitoring roles: - - prometheus - - prometheus_blackbox_exporter - - prometheus_alertmanager - + - monitoring + vars: + monitoring_htpasswd: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/monitoring_htpasswd', profile='oonidevops_user_prod') }}" +- ansible.builtin.import_playbook: deploy-monitoring-config.yml diff --git a/ansible/inventory b/ansible/inventory index a44f8d4..bda9a48 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -1,6 +1,6 @@ [all:children] -htz-fsn -ghs-ams +htz_fsn +ghs_ams ## Role tags @@ -12,7 +12,7 @@ data3.htz-fsn.prod.ooni.nu ## Location tags -[htz-fsn] +[htz_fsn] data.ooni.org monitoring.ooni.org notebook.ooni.org @@ -20,5 +20,10 @@ data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu -[ghs-ams] +[ghs_ams] openvpn-server1.ooni.io +amsmatomo.ooni.nu +db-1.proteus.ooni.io +ams-slack-1.ooni.org +#mia-echoth.ooni.nu +#mia-httpth.ooni.nu diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 17bcd40..7674acd 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -1,11 +1,6 @@ --- -- name: Ensure all hosts are bootstrapped correctly - hosts: all - become: yes - roles: - - bootstrap - tags: - - bootstrap +- name: Include bootstrap playbook + ansible.builtin.import_playbook: deploy-bootstrap.yml - name: Include tier0 playbook ansible.builtin.import_playbook: deploy-tier0.yml diff --git a/ansible/roles/monitoring/defaults/main.yml b/ansible/roles/monitoring/defaults/main.yml new file mode 100644 index 0000000..47ef408 --- /dev/null +++ b/ansible/roles/monitoring/defaults/main.yml @@ -0,0 +1 @@ +enable_log_ingestion: false diff --git a/ansible/roles/monitoring/files/create_logs_table.sql b/ansible/roles/monitoring/files/create_logs_table.sql new file mode 100644 index 0000000..fe6a4cf --- /dev/null +++ b/ansible/roles/monitoring/files/create_logs_table.sql @@ -0,0 +1,56 @@ +CREATE TABLE IF NOT EXISTS default.logs +( + `CODE_FILE` String, + `CODE_FUNC` String, + `CODE_LINE` String, + `INVOCATION_ID` String, + `LOGGER` LowCardinality(String), + `MESSAGE_ID` String, + `MESSAGE` String, + `PRIORITY` UInt8, + `PROCESS_NAME` String, + `SYSLOG_FACILITY` LowCardinality(String), + `SYSLOG_IDENTIFIER` LowCardinality(String), + `SYSLOG_PID` Nullable(UInt64), + `SYSLOG_TIMESTAMP` String, + `THREAD_NAME` String, + `TID` UInt64, + `UNIT` String, + `_AUDIT_LOGINUID` Nullable(UInt64), + `_AUDIT_SESSION` Nullable(UInt64), + `_BOOT_ID` String, + `_CAP_EFFECTIVE` String, + `_CMDLINE` String, + `_COMM` LowCardinality(String), + `_EXE` LowCardinality(String), + `_GID` LowCardinality(UInt32), + `_HOSTNAME` String, + `_KERNEL_DEVICE` String, + `_KERNEL_SUBSYSTEM` String, + `_MACHINE_ID` String, + `_PID` UInt32, + `_SELINUX_CONTEXT` String, + `_SOURCE_MONOTONIC_TIMESTAMP` Nullable(Int64), + `_SOURCE_REALTIME_TIMESTAMP` Int64, + `_STREAM_ID` String, + `_SYSTEMD_CGROUP` LowCardinality(String), + `_SYSTEMD_INVOCATION_ID` String, + `_SYSTEMD_SLICE` String, + `_SYSTEMD_UNIT` LowCardinality(String), + `_TRANSPORT` LowCardinality(String), + `_UDEV_SYSNAME` String, + `_UID` LowCardinality(UInt32), + `__CURSOR` String, + `__MONOTONIC_TIMESTAMP` Nullable(Int64), + `__REALTIME_TIMESTAMP` Int64, + `date` DateTime64(6) ALIAS fromUnixTimestamp64Micro(_SOURCE_REALTIME_TIMESTAMP), + `host` LowCardinality(String), + `inserted_at` DateTime DEFAULT now(), + `message` String, + `rtdate` DateTime64(6) ALIAS fromUnixTimestamp64Micro(__REALTIME_TIMESTAMP), + `timestamp` String, + INDEX timestamp_minmax_idx timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY __REALTIME_TIMESTAMP +SETTINGS index_granularity = 8192 diff --git a/ansible/roles/monitoring/files/log-ingestion.service b/ansible/roles/monitoring/files/log-ingestion.service new file mode 100644 index 0000000..ac1e948 --- /dev/null +++ b/ansible/roles/monitoring/files/log-ingestion.service @@ -0,0 +1,17 @@ +[Unit] +Description=log ingestion + +[Service] +ExecStart=/bin/sh -c 'journalctl -ojson -f | clickhouse-client --query="INSERT INTO logs FORMAT JSONEachRow" --input_format_skip_unknown_fields=1 --input_format_allow_errors_ratio=1' + +SystemCallFilter=~@clock @debug @cpu-emulation @keyring @module @mount @obsolete @raw-io @reboot @swap +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=full +ProtectKernelModules=yes +ProtectKernelTunables=yes + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/monitoring/tasks/log-ingestion.yml b/ansible/roles/monitoring/tasks/log-ingestion.yml new file mode 100644 index 0000000..645f086 --- /dev/null +++ b/ansible/roles/monitoring/tasks/log-ingestion.yml @@ -0,0 +1,85 @@ +# # Vector + +- name: vector - enable repo + tags: vector + shell: extrepo enable vector && extrepo update vector + +- name: vector - install pkg + tags: vector + apt: + # refresh cache + cache_valid_time: 0 + name: + - vector + +- name: vector - deploy SQL file to create logs table + tags: vector + copy: + src: create_logs_table.sql + dest: /etc/clickhouse-server/create_logs_table.sql + +- name: vector - create vector_logs table + tags: vector + command: clickhouse-client --multiline --multiquery --queries-file /etc/clickhouse-server/create_logs_table.sql + +- name: vector - Generate syslog certificates + tags: vector + # runs locally + delegate_to: 127.0.0.1 + shell: | + ./vault view files/pusher_ca.key.vault | openssl req -x509 -new -nodes -key /dev/stdin -sha256 -days 3650 -subj '/O=OONI/OU=CA/CN=ooni.org' -out oonicacert.pem + openssl req -newkey rsa:2048 -nodes -days 3650 -keyout node.key -out node-req.pem -subj '/CN=ooni.org/O=OONI temp CA/C=US' -batch + ./vault view files/pusher_ca.key.vault | openssl x509 -req -days 3650 -set_serial 01 -in node-req.pem -out node-cert.pem -CA oonicacert.pem -CAkey /dev/stdin + register: certs_ready + +- name: vector - Copy TLS certs + tags: vector + ansible.builtin.copy: + src: "{{ item }}" + dest: /etc/vector/ + mode: '0440' + owner: vector + loop: + - oonicacert.pem + - node-cert.pem + - node.key + when: certs_ready.changed + +- name: vector - Delete files + tags: vector + # runs locally + delegate_to: 127.0.0.1 + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - node-cert.pem + - node-req.pem + - node.key + - oonicacert.pem + +- name: vector - configure + tags: vector + template: + src: templates/vector.toml + dest: /etc/vector/vector.toml + +- name: vector - open port + tags: vector + ansible.builtin.copy: + src: templates/10514.nft + dest: /etc/ooni/nftables/tcp/ + register: nft_reload_needed + +- name: vector - reload nft + tags: vector + shell: systemctl reload nftables.service + when: nft_reload_needed.changed + +- name: vector - restart service + tags: vector + systemd: + daemon_reload: yes + enabled: yes + name: vector.service + state: restarted diff --git a/ansible/roles/monitoring/tasks/main.yml b/ansible/roles/monitoring/tasks/main.yml new file mode 100644 index 0000000..24a45cf --- /dev/null +++ b/ansible/roles/monitoring/tasks/main.yml @@ -0,0 +1,304 @@ +--- +# # monitoring host # # + +- name: Set grafana apt repo + tags: monitoring, grafana + template: + src: templates/grafana.list + dest: /etc/apt/sources.list.d/grafana.list + mode: 0644 + owner: root + +- name: Installs packages + tags: monitoring, prometheus + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - apt-transport-https + - nginx + - prometheus + - prometheus-blackbox-exporter + - extrepo + +- name: Give cap_net_raw to prometheus-blackbox-exporter + tags: monitoring, prometheus, exporter_cap_net_raw + community.general.capabilities: + path: /usr/bin/prometheus-blackbox-exporter + capability: cap_net_raw+ep + state: present + +- name: Create Prometheus environment override + # Disable strict cert check https://pkg.go.dev/crypto/x509#Certificate.VerifyHostname + tags: monitoring, prometheus, override_cert_check + template: + src: templates/etc_default_prometheus + dest: /etc/default/prometheus + mode: 0644 + owner: root + +- name: Create Grafana repo GPG pubkey + tags: apt + template: + src: templates/grafana.gpg + dest: /etc/apt/grafana.asc + mode: 0644 + owner: root + +- name: Create Grafana sources list + tags: apt + template: + src: templates/grafana.sources + dest: /etc/apt/sources.list.d/grafana.sources + mode: 0644 + owner: root + +- name: Installs grafana + tags: monitoring, grafana + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - grafana + +- name: Configure grafana + tags: monitoring, grafana + lineinfile: + path: /etc/grafana/grafana.ini + regexp: '^;?domain = ' + line: domain = grafana.ooni.org + +- name: Autoremove + tags: monitoring + apt: + autoremove: yes + +- name: Clean cache + tags: monitoring + apt: + autoclean: yes + +- name: allow HTTPS + tags: monitoring + blockinfile: + path: /etc/ooni/nftables/tcp/443.nft + create: yes + block: | + add rule inet filter input tcp dport 443 counter accept comment "HTTPS" + +- name: reload nft + tags: monitoring + shell: systemctl reload nftables.service + +- name: enable grafana + tags: monitoring + shell: systemctl enable grafana-server + +- name: start grafana + tags: monitoring + shell: systemctl start grafana-server + +- name: Deploy nginx conf + tags: monitoring, grafana + template: + src: templates/nginx.conf + dest: /etc/nginx/sites-enabled/10-monitoring + mode: 0644 + owner: root + +# TODO(art): this htaccess file contains the password of federico which cannot +# be removed, since removing it leads to a bunch of alerts being triggerd. We +# should figure out where his personal password was used and replace it with a +# role based password that is shared and stored in our team keychain. +- name: copy monitoring.htpasswd + tags: monitoring, grafana, htpasswd + template: + src: templates/htpasswd + dest: /etc/nginx/monitoring.htpasswd + mode: 0440 + owner: www-data + group: www-data + +- name: reload nginx + tags: monitoring, grafana + shell: systemctl reload nginx + +- name: Installs packages + tags: jupyter + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - jupyter-notebook + - jupyter-server + - python3-bottleneck + - python3-matplotlib + - python3-numpy + - python3-pandas + - python3-psycopg2 + - python3-scipy + - python3-seaborn + - python3-tables + - python3-tqdm + - python3-ujson + +- name: Install jupyter.service + tags: jupyter + template: + src: templates/jupyter.service + dest: /etc/systemd/system/jupyter.service + mode: 0755 + owner: root + +- name: create jupyter dir + tags: jupyter + file: + path: /var/lib/jupyter/conf + state: directory + owner: jupyter + group: jupyter + +- name: create jupyter conf + tags: jupyter + blockinfile: + path: /var/lib/jupyter/conf/jupyter_notebook_config.py + create: yes + owner: jupyter + group: jupyter + block: | + c.NotebookApp.allow_remote_access = True + c.NotebookApp.enable_mathjax = False + c.NotebookApp.open_browser = False + c.NotebookApp.password = u'' + c.NotebookApp.quit_button = False + c.NotebookApp.token = '' + +- name: reload systemd + tags: jupyter + shell: systemctl daemon-reload + +- name: Start jupyter + tags: jupyter + systemd: + name: jupyter.service + state: started + enabled: yes + +- name: Configure fail2ban + tags: fail2ban + lineinfile: + path: /etc/fail2ban/jail.conf + regexp: '^backend ' + line: backend = systemd + +- name: Configure fail2ban + tags: fail2ban + blockinfile: + create: yes + path: /etc/fail2ban/jail.d/ooni.conf + block: | + [nginx-http-auth] + enabled = true + filter = nginx-http-auth + port = http,https + journalmatch = _SYSTEMD_UNIT=nginx.service PRIORITY=3 + + [nginx-400] + enabled = true + port = http,https + filter = nginx-400 + maxretry = 2 + findtime = 300 + +- name: Configure fail2ban + tags: fail2ban + blockinfile: + create: yes + path: /etc/fail2ban/filter.d/nginx-400.conf + block: | + [Definition] + failregex = ^ -.*"(GET|POST|HEAD).*HTTP.*" 400 + ignoreregex = + +- name: reload fail2ban + tags: fail2ban + shell: systemctl reload fail2ban.service + +# jupycron + +- name: Install jupycron + tags: jupycron + apt: + cache_valid_time: 86400 + state: present + name: jupycron + +# # clickhouse # # + +- name: install clickhouse requirements + tags: clickhouse + apt: + cache_valid_time: 86400 + state: present + name: + - apt-transport-https + - ca-certificates + - dirmngr + +- name: install clickhouse keys + tags: clickhouse + template: + src: templates/clickhouse.gpg + dest: /etc/apt/clickhouse.asc + mode: 0644 + owner: root + +- name: set clickhouse repos + tags: clickhouse + template: + src: templates/clickhouse.sources + dest: /etc/apt/sources.list.d/clickhouse.sources + mode: 0644 + owner: root + +- name: pin clickhouse release train + tags: clickhouse + blockinfile: + path: /etc/apt/preferences.d/clickhouse-server + create: yes + block: | + Package: clickhouse-server + Pin: version 23.1.3.* + Pin-Priority: 999 + +- name: install clickhouse + tags: clickhouse + apt: + # refresh cache + cache_valid_time: 0 + name: + - clickhouse-server={{ clickhouse_pkg_ver }} + - clickhouse-client={{ clickhouse_pkg_ver }} + - clickhouse-common-static={{ clickhouse_pkg_ver }} + vars: + clickhouse_pkg_ver: 23.9.* + +#- name: install clickhouse conf override +# tags: clickhouse +# template: +# src: clickhouse_config.xml +# dest: /etc/clickhouse-server/config.d/ooni_conf.xml +# owner: clickhouse +# group: clickhouse +# mode: 0400 +# notify: restart clickhouse + +- name: Run clickhouse + tags: clickhouse + systemd: + name: clickhouse-server.service + state: started + enabled: yes + +- include_tasks: log-ingestion.yml + when: enable_log_ingestion diff --git a/ansible/roles/monitoring/templates/10514.nft b/ansible/roles/monitoring/templates/10514.nft new file mode 100644 index 0000000..1dd9dad --- /dev/null +++ b/ansible/roles/monitoring/templates/10514.nft @@ -0,0 +1,2 @@ +# Manged by ansible roles/monitoring/templates/10514.nft +add rule inet filter input tcp dport 10514 counter accept comment "incoming logs" diff --git a/ansible/roles/monitoring/templates/clickhouse.gpg b/ansible/roles/monitoring/templates/clickhouse.gpg new file mode 100644 index 0000000..ae3a0dc --- /dev/null +++ b/ansible/roles/monitoring/templates/clickhouse.gpg @@ -0,0 +1,87 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGGbfsABEAC+01IxYaykEHIBW4dom/YipkmjPGbkU/oDibqsHE8VgDLC10Xt +Glto0H9q1fsTIi1nZ9S4lWUuuEAFgcPBt82qnT3E11ZCDC5khjFvPb7d0iK5Wg5T +a1hHNLxuqvRn4sOCyMp7LZJkgLFavvGnXWsHOmB3TzQRaQouc+Y21hqANXFCZXCD +be3Wbdy86ZHOezoWEdHaU1868ZlCPMq10lpBFcLzvdeEfMDAYhOE43c5X0tSbxNo +//kingPLPQ7nxbgsGDKNIsghazmKyMaJA+qpzMmu8qo1ioPZfW8GcYC06cBBAklQ +2dQmGqNvpdJeItBeis9kfJvfkWYCbQ5+ebmm/uJDGCXosIf3/OliMykJhiJzILp5 +1AJqCBD6muL5aP7UorGtVrCQeW2DKB6vtw73qoWmIAmMdW80Nr2lkFu8sDM8VeCI +YUULVa2mkuAet8B3TEtFs6dRZZZNlw6fNny6gmN1dbh+aDebilkHn7eOm8A3qUO3 +PgYNP8swIuGBDQ37lEKZaN6glL7+h0TY9Pz0UsBlCOOh0f3jrjVR2eseQfMtm5TU +/cqY7/+eTzycRpHRHl+xiSiUT1XwyzCf0sAEuMX6d1Um0KFXWWHYdpe4/P4yQmLw +Ex3JaYQHM7EBbLsj9vK7J9qAnrtH0kdyz1RBVE1SV0APS7qNec2GBUXh5QARAQAB +tDpDbGlja0hvdXNlIEluYy4gUmVwb3NpdG9yaWVzIEtleSA8cGFja2FnZXNAY2xp +Y2tob3VzZS5jb20+iQJOBBMBCAA4FiEEOp6hGTqXtUi+FFfUiRn2vStI11QFAmGb +fsACGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQiRn2vStI11R0OA/9GHe5 +Gfco4zWrPSX1cCM1sZZeLZUXeRu4iJlJPahi1sjP2J0rcuv5Xp7paW23el0ksXf4 +P+xknzzp8SlCndQKFWsb404vsNK5VtPB1RJ0Sje0dFM2nO1/Kt8pVX+r1WFJ1gur +BSugsAs3erTCy/ZIB6nhRcFBGsJfvZV24bVxFQKUW6ARigia3bqOZlM4ksdvjrLL +HZ84WiLY++W3wTMIh34KzLClVokl4au8L9Z1g+24EqJMRV7722FVm3GZDdVhmzNy +gKXQvopFvrfQY8Uk9ApZOVSsx9zH65UDP0u13ki7UTlyCi9ucGgT4w6cek1DUcXN +W0FU8498J80fADcWG4bsPjiJ7WiKI29saVked4gfUqx0G5kUCP+l5oo+okm/BFrt +66sO0cEbG9KZCmgdTa2KoNUtsampIBXDG0+9c2SWp3pa1j5sErngR2dyV6L8Muxx +dQJbVR4RxXkM8LkAUMN+YAatknMKEPa7gXalt73fiuvFXpY7NOxeUmlrRXucnCY8 +vSnTNPcuQYAeNyg7z2gyrpqp81Kd+XKUIdKYuydPpvRHzhFOBIf68NUc3MR1UeZx +eBF+bnVkkyL5nRhU+1eLZ0YyW2/PBCmegUahCHEWLpXGzeNBwdnt3HrJEQg54Fu8 +VPcKFa15kJ8QghTlMwYSQkXUPZeBOYaHtva+aSq5Ag0EYyw+bgEQAOECkzewc6Gd +lGA65PIqEPL4JORnsYr0fHNHYGJ+WCHC9HuQEcFJ8Q0R7nSNbMmi6KjnTqM4QdkL +myEQs9TKK5brFBa/ObbL4JVyBJTi4LdlCg95YfEQKTClb7roaLwKH+TykBJ617R+ +QUvFp65YCwzNiS2aKu6AC6zsD2rsrJulq5LIDKvWwLWfFtwRwOHMgviuGlj/97om +XUhuPIScJ5rB+QzICaiKriF/qVVO7obTV0OcUYrTOkNKo8H/q/fw0FSvX3eznz6f +K46gi468P34VO5ZLZs4UQih7EzDqdGz4Lo8GnIrtThWH5rLyFhm7PdB6yLwf8bQe +qDviI73Zb0w9ookYJQpBdglnDyLdhVjoAA0Hp3Q5XL6yvZVZz06YL3k1VWHI2H/A +WeceTFlrExmF2u1KousrwD8/583T042oRcGW97YV5Q9Uw7iffKvJiLp5VBZa319F +svF9hxM8bZVgVqBksBI/ScIvzMgG5FHeGflJ2xEIWWv2uDiAs498kmm5wI6Hdcfs +YuTT+uPsKSNd5G+Ts/39+hirXhVygtx6pFeCRVTdrfWcOfVIVilZm7I+qDNrap2w +6dT23zJT1DvZFCXymU9dxPEoLOm7cn9oxkvjs6c/YOIeyU5CSdI5ZdO99G/8TbFw +LKdSTajkRTVfIa99j/8P0jPMetnk7PoHABEBAAGJBGwEGAEIACAWIQQ6nqEZOpe1 +SL4UV9SJGfa9K0jXVAUCYyw+bgIbAgJACRCJGfa9K0jXVMF0IAQZAQgAHRYhBIhe +K9z5awtFq/BYRT5K1HGd3po4BQJjLD5uAAoJED5K1HGd3po4CjIQALM1EjN2zCtQ +ip9m5IuPnHA6d/YFIuNAXjAnZcg6HQSi+ERFu1w3mL2ya+ll3EYOhFMVMEnja0h0 +e+ezt+WMJt1R2eH1PMZEeJ2uMXgIOl0QsTUXWemhweTQEmsgqksQMqsvFuogxNP/ +Co+cKJCiBpMIU9FJfLrSoGGidnUqr48QjGRVx69FPPthlNOj6/KkhksmJFTT7YhM +G8V7K8yxiYIcQQvRfINLZfmXF2cvGlZU21Il43nCmNb4iFUnrUOouQ7k/Oe/OzZq +ggmcfPGp+6GoSJB3exr3NonjuEhmYR1VPe+WcgsWSl5RWVEaasN5C7dcekqWsqlA +UHpvIqDfzaL3s4HBp19uxxdiy42wXotDhT02P7DJVjVJf9XWgQbmaanzLdPnrUxw +UYcLEQM17VJ37j9DyW9zPBB/pKLRwDr89xpmAKMtRzeXImnBuUJHctp/c9dFvbhJ +GgxFgLHhBCUdTsEs7phLAB32n+oP2Mas6yRgG98k46ic+ZNdJIzplfXT96n/5u+a +eyQOkY3rZh6Jbhzwpb+r5Tj1rAXIJta6prkd8d3+1Rg37x9HkEAHn3ZPDk1RwbVO +VCU0UjMOPU/6Y3FsQq30jSuNhr70p5H/okwpkS7YX/7lY0068XEpWoIyJQIHcBM0 +HZCV904mwdqfRDujziIoQ/xEm3+zIVtzdgYP/3kX4kUup2DEUEY5ND3oKc5CAjyF +UGU7thzILHtt76WrrwXqqfxjJKl9+acmLlrBLj5FQwFEhThEeWunm2T5hhgr4FfF +4MXneQqVRhUslm5+sRIm/K1ye5q4NLfgiwZD/nTM5defyI4cynrby4fy1R9SCBLo +lzvPLpvEraegxIQ3Hk1kT5HmnXi79hZVbZxIkt6wNPG1rTS0TaJ2M37nmhm4GcNj +DtKnhy4Aq569DO8Sd0U6bYKIaFFoi7X5KYnKWY37pYFl0ERU0ffQp7Mg9ruHRvHh +kA41iQ5kF/O1fJHRUfXcYnfIfvN1C77TdXLQryrA/bRNS2dlulIu4CYH+lsfhlIj +BQrRoEOWGu6PzKBwPIXKcVFfB4BcG5si/3vZD3Q1xfUj3MV2sQcgVNbd/t7jJda3 +uhX9DiEPyqdgl382Rq+LoPV4az1igDLDUe7yPrCpD1rwJMyKJyStiP4i1jiIejVa +vPiCpXH1oxCQeR8KqjdrdNbSVgDOapPauZEcwJL464ZlWkdxKX09sN6GaXFTHYTE +XkxrzhlYZaysWEO86+iBUcHnRtc1D3dMJg7hi/Du1IouTwnBsYIwd9Fz/wHpDkiu +0Z3gCmYEg4JQj4iwAtNOQfhJzhEH4kHLom8j8XI/TlgJe/vtNUUUDp3i8qczTjua +VywRbE3+9xjjsSiIuQINBGGbfsABEADXZ0mAMCsf7Y2yJ2aSBLCVPUEZESeB+kDA +ciksAguHeMVp7owI2mfAfw4Z8TEQE6aOb+cqbDbqneQ6nuaRxMNvlDhhtrKztt+U +xWrvgPMvJ59/O1ujY7+VZ+3lXJ01r0u5Xdk1zUZ1uYYl8+Lt3/WmJmmlosX22EwQ +dsAvuRuOmuM2jPTMVN81TaZ2R0Om5vT2L8/SQY7csxc5OylBLskuDQmU1PNlkwTl +tCMqHl7g08LPADLGi/3C9I/0P92A+jZ9RgDCI8BIOS5Ny7CStT96dqrje6TLbWRx +IWBjn1xsVUTxYQY4FXYfw1p7Rm5kzZGImzI4Z7wNcVHCty+X9La+7yKwABW2BeTq +P0rCXpKmy4nNOgwTDVRaA5CL8mg2doK8HOoa03SgziPGEeHHFyS7I2DJAEw+h9K8 +jXVy9Tgd7ZSXmO4Z99ovdHksX4fmD7iOWU2WRp3b2yhpO8b/TgU2r9AQaNMAOsud +N+0Q9NGkEYhWRdWsGapJpa3Q+v1s88lodPhjcIkqGUCH82T/+MhTDdotssj8a6ue +g3mcwVD38690j8ngoBCPmLCNtgZDqhtfy+905uO3ksonF/X2lFc/YG7sqBiOHoDK +yN347pPyK8KbukiMoOasDIQUsryBUeAM0M0Xzftc66BaPgH75KTPZqkOO8aTBCRx +l/AHTejV5wARAQABiQI2BBgBCAAgFiEEOp6hGTqXtUi+FFfUiRn2vStI11QFAmGb +fsACGwwACgkQiRn2vStI11RimQ//a0dHoSekuMLDPEYpQYEEr5JYYWFdFPA/ixeM +HZ566ankWMqKHynhbqqRXVhqKd0pveV+ALhgrUU1nBB9Ma5P51VNEGC0rThiLBVk +CTrV0UrUXxxgA5cTJBVZe7yyysqdCHcuuNNrlQbuzhkJWBySgeWrbmd4VUbDdL0i +GMFSAtbrYjSWreBDBgWBYd8uRiPHtc6ACBIhzGGOulw0k4NnMXHDJyCwEv7EAxES +Y/V8kLK3p7DXc6tpmd1vZ38X52CnVRM6aLQl6YWKXhA+vp9cPJehfEbK+ZPErbCB +4jF6AC4BFwntE0YSod6TVM+wJBuPplnWr4nUUhsk/FeulWGF+AhUG2XJiIn90ZHI +cxlct8Nt1zBgoquxWYmqt/z+s7TOBXnvotOMJggjgsPozFlHjoy47Xc14VwzxnU+ +NknXYqDAsTfjgXiw7NRQdJ8BUd4TSf8iINf2uCNzp4QDJ37VfN7/BiaXBGjEEN+k +OKpxkse6qm06GFd+bXGSHh6H7z2d973k2QIfW03opWAwS+AdH8xmUUaAyyhEql5m +lXZVkQRS8rEO4IV5HQV8FD5iV9bdNihaydlKP09/D9ZDlie0dJZXOIHQDuwDt4Ab +3reSs1T5utClQE0FS8ZZjPuitq/l+TsT83TASVynvEmwvNiEbpOsNjvB1u++w4ni +qCHQv3Y= +=AKGw +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/monitoring/templates/clickhouse.sources b/ansible/roles/monitoring/templates/clickhouse.sources new file mode 100644 index 0000000..db4eda1 --- /dev/null +++ b/ansible/roles/monitoring/templates/clickhouse.sources @@ -0,0 +1,7 @@ +Architectures: amd64 +Suites: stable +Uris: https://packages.clickhouse.com/deb +Types: deb +Components: main +Enabled: yes +Signed-By: /etc/apt/clickhouse.asc diff --git a/ansible/roles/monitoring/templates/etc_default_prometheus b/ansible/roles/monitoring/templates/etc_default_prometheus new file mode 100644 index 0000000..5ba9e6c --- /dev/null +++ b/ansible/roles/monitoring/templates/etc_default_prometheus @@ -0,0 +1,4 @@ +# Managed by ansible, see +# monitoring/templates/etc_default_prometheus +ARGS="" +GODEBUG=x509ignoreCN=0 diff --git a/ansible/roles/monitoring/templates/grafana.gpg b/ansible/roles/monitoring/templates/grafana.gpg new file mode 100644 index 0000000..35771a2 --- /dev/null +++ b/ansible/roles/monitoring/templates/grafana.gpg @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGTnhmkBDADUE+SzjRRyitIm1siGxiHlIlnn6KO4C4GfEuV+PNzqxvwYO+1r +mcKlGDU0ugo8ohXruAOC77Kwc4keVGNU89BeHvrYbIftz/yxEneuPsCbGnbDMIyC +k44UOetRtV9/59Gj5YjNqnsZCr+e5D/JfrHUJTTwKLv88A9eHKxskrlZr7Un7j3i +Ef3NChlOh2Zk9Wfk8IhAqMMTferU4iTIhQk+5fanShtXIuzBaxU3lkzFSG7VuAH4 +CBLPWitKRMn5oqXUE0FZbRYL/6Qz0Gt6YCJsZbaQ3Am7FCwWCp9+ZHbR9yU+bkK0 +Dts4PNx4Wr9CktHIvbypT4Lk2oJEPWjcCJQHqpPQZXbnclXRlK5Ea0NVpaQdGK+v +JS4HGxFFjSkvTKAZYgwOk93qlpFeDML3TuSgWxuw4NIDitvewudnaWzfl9tDIoVS +Bb16nwJ8bMDzovC/RBE14rRKYtMLmBsRzGYHWd0NnX+FitAS9uURHuFxghv9GFPh +eTaXvc4glM94HBUAEQEAAbQmR3JhZmFuYSBMYWJzIDxlbmdpbmVlcmluZ0BncmFm +YW5hLmNvbT6JAdQEEwEKAD4WIQS1Oud7rbYwpoMEYAWWP6J3EEWFRQUCZOeGaQIb +AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCWP6J3EEWFRUiADACa +i+xytv2keEFJWjXNnFAx6/obnHRcXOI3w6nH/zL8gNI7YN5jcdQT2NYvKVYTb3fW +GuMsjHWgat5Gq3AtJrOKABpZ6qeYNPk0Axn/dKtOTwXjZ4pKX3bbUYvVfs0fCEZv +B0HHIj2wI9kgMpoTrkj22LE8layZTPOoQ+3/FbLzS8hN3CYZj25mHN7bpZq8EbV3 +8FW9EU0HM0tg6CvoxkRiVqAuAC0KnVIZAdhD4dlYKuncq64nMvT1A5wxSYbnE+uf +mnWQQhhS6BOwRqN054yw1FrWNDFsvnOSHmr8dIiriv+aZYvx5JQFJ7oZP3LwdYyg +ocQcAJA8HFTIk3P6uJiIF/zdDzocgdKs+IYDoId0hxX7sGCvqdrsveq8n3m7uQiN +7FvSiV0eXIdV4F7340kc8EKiYwpuYSaZX0UWKLenzlUvD+W4pZCWtoXzPsW7PKUt +q1xdW0+NY+AGLCvSJCc5F4S5kFCObfBAYBbldjwwJFocdq/YOvvWYTPyV7kJeJS5 +AY0EZOeGaQEMALNIFUricEIwtZiX7vSDjwxobbqPKqzdek8x3ud0CyYlrbGHy0k+ +FDEXstjJQQ1s9rjJSu3sv5wyg9GDAUH3nzO976n/ZZvKPti3p2XU2UFx5gYkaaFV +D56yYxqGY0YU5ft6BG+RUz3iEPg3UBUzt0sCIYnG9+CsDqGOnRYIIa46fu2/H9Vu +8JvvSq9xbsK9CfoQDkIcoQOixPuI4P7eHtswCeYR/1LUTWEnYQWsBCf57cEpzR6t +7mlQnzQo9z4i/kp4S0ybDB77wnn+isMADOS+/VpXO+M7Zj5tpfJ6PkKch3SGXdUy +3zht8luFOYpJr2lVzp7n3NwB4zW08RptTzTgFAaW/NH2JjYI+rDvQm4jNs08Dtsp +nm4OQvBA9Df/6qwMEOZ9i10ixqk+55UpQFJ3nf4uKlSUM7bKXXVcD/odq804Y/K4 +y3csE059YVIyaPexEvYSYlHE2odJWRg2Q1VehmrOSC8Qps3xpU7dTHXD74ZpaYbr +haViRS5v/lCsiwARAQABiQG8BBgBCgAmFiEEtTrne622MKaDBGAFlj+idxBFhUUF +AmTnhmkCGwwFCQPCZwAACgkQlj+idxBFhUUNbQv8DCcfi3GbWfvp9pfY0EJuoFJX +LNgci7z7smXq7aqDp2huYQ+MulnPAydjRCVW2fkHItF2Ks6l+2/8t5Xz0eesGxST +xTyR31ARENMXaq78Lq+itZ+usOSDNuwJcEmJM6CceNMLs4uFkX2GRYhchkry7P0C +lkLxUTiB43ooi+CqILtlNxH7kM1O4Ncs6UGZMXf2IiG9s3JDCsYVPkC5QDMOPkTy +2ZriF56uPerlJveF0dC61RZ6RlM3iSJ9Fwvea0Oy4rwkCcs5SHuwoDTFyxiyz0QC +9iqi3fG3iSbLvY9UtJ6X+BtDqdXLAT9Pq527mukPP3LwpEqFVyNQKnGLdLOu2YXc +TWWWseSQkHRzBmjD18KTD74mg4aXxEabyT4snrXpi5+UGLT4KXGV5syQO6Lc0OGw +9O/0qAIU+YW7ojbKv8fr+NB31TGhGYWASjYlN1NvPotRAK6339O0/Rqr9xGgy3AY +SR+ic2Y610IM7xccKuTVAW9UofKQwJZChqae9VVZ +=J9CI +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/monitoring/templates/grafana.list b/ansible/roles/monitoring/templates/grafana.list new file mode 100644 index 0000000..adbad20 --- /dev/null +++ b/ansible/roles/monitoring/templates/grafana.list @@ -0,0 +1 @@ +deb https://packages.grafana.com/oss/deb stable main diff --git a/ansible/roles/monitoring/templates/grafana.sources b/ansible/roles/monitoring/templates/grafana.sources new file mode 100644 index 0000000..dd17e11 --- /dev/null +++ b/ansible/roles/monitoring/templates/grafana.sources @@ -0,0 +1,7 @@ +Architectures: amd64 +Suites: stable +Uris: https://apt.grafana.com +Types: deb +Components: main +Enabled: yes +Signed-By: /etc/apt/grafana.asc diff --git a/ansible/roles/monitoring/templates/htpasswd b/ansible/roles/monitoring/templates/htpasswd new file mode 100644 index 0000000..3c18c80 --- /dev/null +++ b/ansible/roles/monitoring/templates/htpasswd @@ -0,0 +1,5 @@ +# ansible-managed in ooni/sysadmin.git +# Username should be taken from @openobservatory.org domain +# Password should be generated with scripts/ngx-mkpasswd +# don't remove federico, because it will break monitoring +{{ monitoring_htpasswd }} diff --git a/ansible/roles/monitoring/templates/jupyter.service b/ansible/roles/monitoring/templates/jupyter.service new file mode 100644 index 0000000..d8c1dc9 --- /dev/null +++ b/ansible/roles/monitoring/templates/jupyter.service @@ -0,0 +1,37 @@ +[Unit] +Description=Jupyter +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/jupyter-notebook --no-browser +Environment=JUPYTER_CONFIG_DIR=/var/lib/jupyter/conf +Environment=JUPYTER_DATA_DIR=/var/lib/jupyter/data +Environment=JUPYTER_RUNTIME_DIR=/var/lib/jupyter/run +Restart=on-failure +# DynamicUser=yes implies ProtectSystem=strict ProtectHome=read-only PrivateTmp, RemoveIPC, NoNewPrivileges, RestrictSUIDSGID +DynamicUser=yes +PrivateDevices=yes +PrivateMounts=yes +PrivateUsers=yes +DevicePolicy=closed +ProtectHostname=yes +ProtectHome=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK +RestrictNamespaces=yes +RestrictRealtime=yes +MemoryDenyWriteExecute=yes +LockPersonality=yes + +CacheDirectory=jupyter +ReadWriteDirectories=/var/lib/jupyter +RuntimeDirectory=jupyter +StateDirectory=jupyter +WorkingDirectory=/var/lib/jupyter +WorkingDirectory=/var/lib/jupyter + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/monitoring/templates/nginx.conf b/ansible/roles/monitoring/templates/nginx.conf new file mode 100644 index 0000000..c383297 --- /dev/null +++ b/ansible/roles/monitoring/templates/nginx.conf @@ -0,0 +1,203 @@ +# Managed by ansible +# roles/ooni-backend/monitoring/nginx.conf + +# Grafana +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name grafana.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + + ssl_certificate /var/lib/dehydrated/certs/grafana.ooni.org/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/grafana.ooni.org/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/grafana.ooni.org/chain.pem; + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + add_header Strict-Transport-Security "max-age=63072000" always; + ssl_stapling on; + ssl_stapling_verify on; + + resolver 127.0.0.1; + + # Grafana uses its own authentication + + location / { + proxy_pass http://localhost:3000; + # do not forward the basic auth header to grafana + proxy_set_header Authorization ""; + proxy_set_header Host $http_host; + } + + # Proxy Grafana Live WebSocket connections. + location /api/live { + rewrite ^/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000/; + } +} + +# Netdata +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name netdata.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + + ssl_certificate /var/lib/dehydrated/certs/netdata.ooni.org/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/netdata.ooni.org/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/netdata.ooni.org/chain.pem; + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + + location /{ + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_pass_request_headers on; + proxy_set_header Connection "keep-alive"; + proxy_store off; + proxy_pass http://127.0.0.1:19999/; + auth_basic "OONI Monitoring"; + auth_basic_user_file /etc/nginx/monitoring.htpasswd; + gzip on; + gzip_proxied any; + gzip_types *; + } +} + +# Prometheus +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name prometheus.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + + ssl_certificate /var/lib/dehydrated/certs/prometheus.ooni.org/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/prometheus.ooni.org/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/prometheus.ooni.org/chain.pem; + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + location / { + proxy_pass http://127.0.0.1:9090; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + auth_basic "OONI Monitoring"; + auth_basic_user_file /etc/nginx/monitoring.htpasswd; + } +} + +# Jupyter +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name jupyter.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + + ssl_certificate /var/lib/dehydrated/certs/jupyter.ooni.org/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/jupyter.ooni.org/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/jupyter.ooni.org/chain.pem; + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + + location / { + proxy_pass http://127.0.0.1:8888; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + auth_basic "OONI Monitoring"; + auth_basic_user_file /etc/nginx/monitoring.htpasswd; + + # websocket headers + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Scheme $scheme; + + proxy_buffering off; + } +} diff --git a/ansible/roles/monitoring/templates/vector.list b/ansible/roles/monitoring/templates/vector.list new file mode 100644 index 0000000..1b56727 --- /dev/null +++ b/ansible/roles/monitoring/templates/vector.list @@ -0,0 +1,2 @@ +# See ansible/roles/monitoring/tasks/main.yml +deb https://repositories.timber.io/public/vector/deb/debian bullseye main diff --git a/ansible/roles/monitoring/templates/vector.toml b/ansible/roles/monitoring/templates/vector.toml new file mode 100644 index 0000000..1b23658 --- /dev/null +++ b/ansible/roles/monitoring/templates/vector.toml @@ -0,0 +1,24 @@ +# Managed by ansible + +[sources.local_journald] +type = "journald" +exclude_units = [ "clickhouse" ] + +[sources.remote_vector] +type = "vector" +address = "0.0.0.0:10514" + +tls.enabled = true +tls.verify_certificate = true +tls.verify_hostname = false +tls.ca_file = "/etc/vector/oonicacert.pem" +tls.crt_file = "/etc/vector/node-cert.pem" +tls.key_file = "/etc/vector/node.key" + +[sinks.clickhouse_sink] +type = "clickhouse" +inputs = [ "local_journald", "remote_vector" ] +database = "default" +endpoint = "http://localhost:8123" +table = "logs" +date_time_best_effort = true diff --git a/ansible/roles/notify-slack/tasks/main.yml b/ansible/roles/notify-slack/tasks/main.yml new file mode 100644 index 0000000..83290d0 --- /dev/null +++ b/ansible/roles/notify-slack/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: "notify #ooni-bots at openobservatory.slack.com" + slack: + token: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ansible_slack_token', profile='oonidevops_user_prod') }}" + channel: "#ooni-bots" + msg: "{{ lookup('template', 'notify-slack.j2') }}" + delegate_to: localhost + run_once: true diff --git a/ansible/roles/notify-slack/templates/notify-slack.j2 b/ansible/roles/notify-slack/templates/notify-slack.j2 new file mode 100644 index 0000000..6f43d7d --- /dev/null +++ b/ansible/roles/notify-slack/templates/notify-slack.j2 @@ -0,0 +1,10 @@ +{% set cleanness = 'dirty' if lookup('pipe', 'git status -s') else 'clean' %} +{% set head = lookup('pipe', 'git rev-parse HEAD') %} +{% set head7 = lookup('pipe', 'git rev-parse --short HEAD') %} +{% set branch = lookup('pipe', 'git describe --always --exact-match --all HEAD') | replace('heads/', '') %} +{% set user_slug = lookup('pipe', 'whoami') + '@' + lookup('pipe', 'hostname') %} +{% if lookup('file', '~/.ooni-sysadmin/user_slug', errors='ignore') %} +{% set user_slug = lookup('file', '~/.ooni-sysadmin/user_slug') %} +{% endif %} + +{{ user_slug }} runs `{{ lookup('env', 'ANSIBLE_SLACK_CMD') }}` on top of {{ cleanness }} `` (`{{ branch }}`)