From d4693544c92e6824fdd7d428f5475b6e9ed5e56e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 11:20:50 +0100 Subject: [PATCH 01/24] Add playbook for deploying datadog agent to hosts --- ansible/deploy-datadog-agent.yml | 7 +++++++ ansible/group_vars/have_datadog_agent/vars.yml | 1 + ansible/group_vars/have_datadog_agent/vault | 8 ++++++++ ansible/inventory.yml | 4 ++++ 4 files changed, 20 insertions(+) create mode 100644 ansible/deploy-datadog-agent.yml create mode 100644 ansible/group_vars/have_datadog_agent/vars.yml create mode 100644 ansible/group_vars/have_datadog_agent/vault diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml new file mode 100644 index 00000000..ab93aee6 --- /dev/null +++ b/ansible/deploy-datadog-agent.yml @@ -0,0 +1,7 @@ +# Runnning this role requires ansible >= 2.10 and running: +# ansible-galaxy install datadog.datadog +- hosts: have_datadog_agent + roles: + - { role: datadog.datadog, become: yes } + vars: + datadog_api_key: "{{ ooni_datadog_api_key }}" diff --git a/ansible/group_vars/have_datadog_agent/vars.yml b/ansible/group_vars/have_datadog_agent/vars.yml new file mode 100644 index 00000000..6541ebc6 --- /dev/null +++ b/ansible/group_vars/have_datadog_agent/vars.yml @@ -0,0 +1 @@ +ooni_datadog_api_key: "{{ vault_ooni_datadog_api_key }}" diff --git a/ansible/group_vars/have_datadog_agent/vault b/ansible/group_vars/have_datadog_agent/vault new file mode 100644 index 00000000..11b8d3b2 --- /dev/null +++ b/ansible/group_vars/have_datadog_agent/vault @@ -0,0 +1,8 @@ +$ANSIBLE_VAULT;1.1;AES256 +63316539646632353261333838373064663165633733333362363461336665323532386332383864 +6262363162383032323764633466646435346566333564350a326636623161383663616563396435 +35646338373037636538303937336338643032653636633362343139393539663566343833643063 +6630633238366535370a373834633763616663653334636561653261643238303730613365366164 +32386235303839346130393463373161633938343336383433646261616166326339313764623530 +36356538636138656336316438333537663663323535666630333065303732373836373662333831 +313937663936666436623235363838343561 diff --git a/ansible/inventory.yml b/ansible/inventory.yml index e4cf246b..3966ba31 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -82,6 +82,10 @@ all: have_tor: {} + have_datadog_agent: + hosts: + oonidata.ooni.org: {} + monitoring: hosts: monitoring.ooni.org: {} From 3566bb8eaacd08a4dc5152ba09a9d374942536cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 11:23:03 +0100 Subject: [PATCH 02/24] Update Readme to mention new version of ansible --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d3a4ffd8..15d32a08 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ It is recommended you use a [python virtualenv](https://virtualenv.pypa.io/en/la Once you activate the virtualenv you can install the correct version of ansible and dnspython with: ``` -pip install ansible==2.9.16 +# pip install ansible==2.9.16 +pip install ansible==2.10.7 ``` From 7d5cd27a4e06d0067c0bd3022681dbd27f1dfa4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 11:23:15 +0100 Subject: [PATCH 03/24] Remove travis vault creation in vault command We are not using travis anymore and the script is has been broken for a while now --- ansible/vault | 3 --- 1 file changed, 3 deletions(-) diff --git a/ansible/vault b/ansible/vault index f945370f..3fa37376 100755 --- a/ansible/vault +++ b/ansible/vault @@ -1,6 +1,3 @@ #!/bin/bash -ex gitroot=$(git rev-parse --show-toplevel) ansible-vault "$@" --vault-password-file "${gitroot}/ansible/password-pipe" -if [ "$1" != view ] && [ "$1" != decrypt ] && [ "$1" != rekey ]; then - "${gitroot}/scripts/mk-travis-vault" -fi From e64ce48dc666b4fbc9848631e7c24f230029c1e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 11:24:42 +0100 Subject: [PATCH 04/24] Send data to EU site --- ansible/deploy-datadog-agent.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index ab93aee6..ed7370b6 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -5,3 +5,4 @@ - { role: datadog.datadog, become: yes } vars: datadog_api_key: "{{ ooni_datadog_api_key }}" + datadog_site: "datadoghq.eu" From b19d74160b182c43921bb1da6fcf740bc2276d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 11:55:07 +0100 Subject: [PATCH 05/24] Read the hostname from the inventory hostname --- ansible/deploy-datadog-agent.yml | 7 +++++++ ansible/group_vars/all/vars.yml | 6 ++++++ ansible/inventory.yml | 28 +++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index ed7370b6..c56423e1 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -6,3 +6,10 @@ vars: datadog_api_key: "{{ ooni_datadog_api_key }}" datadog_site: "datadoghq.eu" + datadog_config: + hostname: "{{ inventory_hostname }}" + tags: + - "cloud_provider:{{ dd_tag_cloud_provider }}" + - "availability_zone:{{ dd_tag_availability_zone }}" + - "env:{{ dd_tag_env }}" + - "service:{{ dd_tag_service }}" diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index 4daf4ebf..6d292e21 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -105,4 +105,10 @@ ngxprom_exporter_ports: { 9100: '{{ node_exporter_ipv4 }}' } ntp_pool: debian have_ssd: false + +# defaults for datadog tag keys +dd_tag_cloud_provider: "undefined" +dd_tag_availability_zone: "undefined" +dd_tag_env: "undefined" +dd_tag_service: "undefined" ... diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 3966ba31..7cf36b54 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -84,7 +84,32 @@ all: have_datadog_agent: hosts: - oonidata.ooni.org: {} + #backend-fsn.ooni.org: + # dd_tag_cloud_provider: hetzner + # dd_tag_availability_zone: eu-fsn + # dd_tag_env: prod + # dd_tag_service: oonibackend + #backend-hel.ooni.org: + # dd_tag_cloud_provider: hetzner + # dd_tag_availability_zone: eu-hel + # dd_tag_env: test + # dd_tag_service: oonibackend + #monitoring.ooni.org: + # dd_tag_cloud_provider: hetzner + # dd_tag_availability_zone: eu-fsn + # dd_tag_env: prod + # dd_tag_service: monitoring + #data.ooni.org: + # dd_tag_cloud_provider: hetzner + # dd_tag_availability_zone: eu-fsn + # dd_tag_env: test + # dd_tag_service: oonidata + oonidata.ooni.org: + dd_tag_cloud_provider: hetzner + dd_tag_availability_zone: eu-fsn + dd_tag_env: prod + dd_tag_service: oonidata + monitoring: hosts: @@ -100,6 +125,7 @@ all: backend-hel.ooni.org: {} monitoring.ooni.org: {} oonidata.ooni.org: {} + data.ooni.org: {} probe_services: hosts: From 0fec43c640d950ed6b9fe9c023c5a7d5acf01008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 11:55:33 +0100 Subject: [PATCH 06/24] Add agent to data.ooni.org --- ansible/inventory.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 7cf36b54..69976f1d 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -99,11 +99,11 @@ all: # dd_tag_availability_zone: eu-fsn # dd_tag_env: prod # dd_tag_service: monitoring - #data.ooni.org: - # dd_tag_cloud_provider: hetzner - # dd_tag_availability_zone: eu-fsn - # dd_tag_env: test - # dd_tag_service: oonidata + data.ooni.org: + dd_tag_cloud_provider: hetzner + dd_tag_availability_zone: eu-fsn + dd_tag_env: test + dd_tag_service: oonidata oonidata.ooni.org: dd_tag_cloud_provider: hetzner dd_tag_availability_zone: eu-fsn From c813ade8c6dbf6bcddaca291fd57c67b5eec59f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 12:02:29 +0100 Subject: [PATCH 07/24] Fix typo in availability zone tag --- ansible/deploy-datadog-agent.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index c56423e1..6f1fbfe8 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -10,6 +10,6 @@ hostname: "{{ inventory_hostname }}" tags: - "cloud_provider:{{ dd_tag_cloud_provider }}" - - "availability_zone:{{ dd_tag_availability_zone }}" + - "availability-zone:{{ dd_tag_availability_zone }}" - "env:{{ dd_tag_env }}" - "service:{{ dd_tag_service }}" From 399b5958df5c9614c0571f53dc7a4a864fa73d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 12:03:49 +0100 Subject: [PATCH 08/24] Add region tag --- ansible/deploy-datadog-agent.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index 6f1fbfe8..90be205d 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -11,5 +11,6 @@ tags: - "cloud_provider:{{ dd_tag_cloud_provider }}" - "availability-zone:{{ dd_tag_availability_zone }}" + - "region:{{ dd_tag_availability_zone }}" - "env:{{ dd_tag_env }}" - "service:{{ dd_tag_service }}" From adb4cbeb77562390cf26022de4f37f7791f76f76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 12:17:34 +0100 Subject: [PATCH 09/24] Add support for journald log collection * Move datadog into dedicated role --- ansible/deploy-datadog-agent.yml | 13 +------------ ansible/roles/oodd/meta/main.yml | 20 ++++++++++++++++++++ ansible/roles/oodd/requirements.yml | 1 + ansible/roles/oodd/tasks/main.yml | 5 +++++ 4 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 ansible/roles/oodd/meta/main.yml create mode 100644 ansible/roles/oodd/requirements.yml create mode 100644 ansible/roles/oodd/tasks/main.yml diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index 90be205d..2a126eee 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -2,15 +2,4 @@ # ansible-galaxy install datadog.datadog - hosts: have_datadog_agent roles: - - { role: datadog.datadog, become: yes } - vars: - datadog_api_key: "{{ ooni_datadog_api_key }}" - datadog_site: "datadoghq.eu" - datadog_config: - hostname: "{{ inventory_hostname }}" - tags: - - "cloud_provider:{{ dd_tag_cloud_provider }}" - - "availability-zone:{{ dd_tag_availability_zone }}" - - "region:{{ dd_tag_availability_zone }}" - - "env:{{ dd_tag_env }}" - - "service:{{ dd_tag_service }}" + - oodd diff --git a/ansible/roles/oodd/meta/main.yml b/ansible/roles/oodd/meta/main.yml new file mode 100644 index 00000000..bfae2a15 --- /dev/null +++ b/ansible/roles/oodd/meta/main.yml @@ -0,0 +1,20 @@ +dependencies: + roles: + - { role: datadog.datadog, become: yes } + vars: + datadog_api_key: "{{ ooni_datadog_api_key }}" + datadog_site: "datadoghq.eu" + datadog_checks: + journald: + logs: + - type: journald + container_mode: true + datadog_config: + hostname: "{{ inventory_hostname }}" + logs_enabled: true + tags: + - "cloud_provider:{{ dd_tag_cloud_provider }}" + - "availability-zone:{{ dd_tag_availability_zone }}" + - "region:{{ dd_tag_availability_zone }}" + - "env:{{ dd_tag_env }}" + - "service:{{ dd_tag_service }}" diff --git a/ansible/roles/oodd/requirements.yml b/ansible/roles/oodd/requirements.yml new file mode 100644 index 00000000..35165e17 --- /dev/null +++ b/ansible/roles/oodd/requirements.yml @@ -0,0 +1 @@ +- name: datadog.datadog diff --git a/ansible/roles/oodd/tasks/main.yml b/ansible/roles/oodd/tasks/main.yml new file mode 100644 index 00000000..b9d6b369 --- /dev/null +++ b/ansible/roles/oodd/tasks/main.yml @@ -0,0 +1,5 @@ +- name: Add dd-agent user to systemd-journal group + ansible.builtin.user: + name: dd-agent + groups: systemd-journal + append: yes From 7435e0687ab846eaa6e2e9e89778c479b599ef68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 12:18:56 +0100 Subject: [PATCH 10/24] Fix yaml formatting --- ansible/roles/oodd/meta/main.yml | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ansible/roles/oodd/meta/main.yml b/ansible/roles/oodd/meta/main.yml index bfae2a15..aefa81d6 100644 --- a/ansible/roles/oodd/meta/main.yml +++ b/ansible/roles/oodd/meta/main.yml @@ -1,20 +1,20 @@ dependencies: - roles: - - { role: datadog.datadog, become: yes } - vars: - datadog_api_key: "{{ ooni_datadog_api_key }}" - datadog_site: "datadoghq.eu" - datadog_checks: - journald: - logs: - - type: journald - container_mode: true - datadog_config: - hostname: "{{ inventory_hostname }}" - logs_enabled: true - tags: - - "cloud_provider:{{ dd_tag_cloud_provider }}" - - "availability-zone:{{ dd_tag_availability_zone }}" - - "region:{{ dd_tag_availability_zone }}" - - "env:{{ dd_tag_env }}" - - "service:{{ dd_tag_service }}" + - role: datadog.datadog + become: yes + vars: + datadog_api_key: "{{ ooni_datadog_api_key }}" + datadog_site: "datadoghq.eu" + datadog_checks: + journald: + logs: + - type: journald + container_mode: true + datadog_config: + hostname: "{{ inventory_hostname }}" + logs_enabled: true + tags: + - "cloud_provider:{{ dd_tag_cloud_provider }}" + - "availability-zone:{{ dd_tag_availability_zone }}" + - "region:{{ dd_tag_availability_zone }}" + - "env:{{ dd_tag_env }}" + - "service:{{ dd_tag_service }}" From 78007d975629f4386cd6861870ae93e23bf8647a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 12:22:09 +0100 Subject: [PATCH 11/24] Add backend-hel to monitoring too --- ansible/inventory.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 69976f1d..0e4948ad 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -89,16 +89,16 @@ all: # dd_tag_availability_zone: eu-fsn # dd_tag_env: prod # dd_tag_service: oonibackend - #backend-hel.ooni.org: - # dd_tag_cloud_provider: hetzner - # dd_tag_availability_zone: eu-hel - # dd_tag_env: test - # dd_tag_service: oonibackend #monitoring.ooni.org: # dd_tag_cloud_provider: hetzner # dd_tag_availability_zone: eu-fsn # dd_tag_env: prod # dd_tag_service: monitoring + backend-hel.ooni.org: + dd_tag_cloud_provider: hetzner + dd_tag_availability_zone: eu-hel + dd_tag_env: test + dd_tag_service: oonibackend data.ooni.org: dd_tag_cloud_provider: hetzner dd_tag_availability_zone: eu-fsn From 39f0b0af9727461b860ae62350e3ac9a315dfb5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 13:15:37 +0100 Subject: [PATCH 12/24] Refactor of the tags injection --- ansible/deploy-datadog-agent.yml | 17 ++++++++++++++++- ansible/group_vars/all/vars.yml | 6 ------ ansible/roles/oodd/meta/main.yml | 9 ++------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index 2a126eee..5469f804 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -1,5 +1,20 @@ # Runnning this role requires ansible >= 2.10 and running: # ansible-galaxy install datadog.datadog - hosts: have_datadog_agent + pre_tasks: + - name: set datadog tags + set_fact: + datadog_tags: >- + [ + {% if dd_tag_cloud_provider is defined %}"cloud_provider:{{ dd_tag_cloud_provider }}",{% endif %} + {% if dd_tag_availability_zone is defined %}"availability-zone:{{ dd_tag_availability_zone }}",{% endif %} + {% if dd_tag_availability_zone is defined %}"region:{{ dd_tag_availability_zone }}",{% endif %} + {% if dd_tag_env is defined %}"env:{{ dd_tag_env }}",{% endif %} + {% if dd_tag_service is defined %}"service:{{ dd_tag_service }}",{% endif %} + "configuration_manager:ansible" + ] roles: - - oodd + - role: oodd + vars: + tags: "{{ datadog_tags }}" + hostname: "{{ inventory_hostname }}" diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index 6d292e21..4daf4ebf 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -105,10 +105,4 @@ ngxprom_exporter_ports: { 9100: '{{ node_exporter_ipv4 }}' } ntp_pool: debian have_ssd: false - -# defaults for datadog tag keys -dd_tag_cloud_provider: "undefined" -dd_tag_availability_zone: "undefined" -dd_tag_env: "undefined" -dd_tag_service: "undefined" ... diff --git a/ansible/roles/oodd/meta/main.yml b/ansible/roles/oodd/meta/main.yml index aefa81d6..4b5a7a87 100644 --- a/ansible/roles/oodd/meta/main.yml +++ b/ansible/roles/oodd/meta/main.yml @@ -10,11 +10,6 @@ dependencies: - type: journald container_mode: true datadog_config: - hostname: "{{ inventory_hostname }}" + hostname: "{{ hostname }}" logs_enabled: true - tags: - - "cloud_provider:{{ dd_tag_cloud_provider }}" - - "availability-zone:{{ dd_tag_availability_zone }}" - - "region:{{ dd_tag_availability_zone }}" - - "env:{{ dd_tag_env }}" - - "service:{{ dd_tag_service }}" + tags: "{{ tags }}" From baa62b293a28d1a3c813cff522a8c69effc487ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 14:44:56 +0100 Subject: [PATCH 13/24] Add agent to monitoring host --- ansible/inventory.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 0e4948ad..69ad4c65 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -89,11 +89,11 @@ all: # dd_tag_availability_zone: eu-fsn # dd_tag_env: prod # dd_tag_service: oonibackend - #monitoring.ooni.org: - # dd_tag_cloud_provider: hetzner - # dd_tag_availability_zone: eu-fsn - # dd_tag_env: prod - # dd_tag_service: monitoring + monitoring.ooni.org: + dd_tag_cloud_provider: hetzner + dd_tag_availability_zone: eu-fsn + dd_tag_env: prod + dd_tag_service: monitoring backend-hel.ooni.org: dd_tag_cloud_provider: hetzner dd_tag_availability_zone: eu-hel From c501e4a3f52975df7eb9141d5a4bbb70a0e51c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 15:10:51 +0100 Subject: [PATCH 14/24] Add support for scraping prometheus metrics from monitoring host --- ansible/host_vars/monitoring.ooni.org/vars.yml | 10 ++++++++++ ansible/roles/oodd/defaults/main.yml | 5 +++++ ansible/roles/oodd/meta/main.yml | 6 +----- 3 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 ansible/roles/oodd/defaults/main.yml diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 9dd0a898..cb86ec65 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -152,4 +152,14 @@ blackbox_jobs: module: icmp targets: "{{ groups['dom0'] | list }}" +ooni_datadog_checks: + journald: + logs: + - type: journald + container_mode: true + openmetrics: + openmetrics_endpoint: "http://prometheus.ooni.org:9090/metrics" + namespace: "ooniprom" + metrics: + - ".*" ... diff --git a/ansible/roles/oodd/defaults/main.yml b/ansible/roles/oodd/defaults/main.yml new file mode 100644 index 00000000..b8e2b83e --- /dev/null +++ b/ansible/roles/oodd/defaults/main.yml @@ -0,0 +1,5 @@ +ooni_datadog_checks: + journald: + logs: + - type: journald + container_mode: true diff --git a/ansible/roles/oodd/meta/main.yml b/ansible/roles/oodd/meta/main.yml index 4b5a7a87..04654fdb 100644 --- a/ansible/roles/oodd/meta/main.yml +++ b/ansible/roles/oodd/meta/main.yml @@ -4,11 +4,7 @@ dependencies: vars: datadog_api_key: "{{ ooni_datadog_api_key }}" datadog_site: "datadoghq.eu" - datadog_checks: - journald: - logs: - - type: journald - container_mode: true + datadog_checks: "{{ ooni_datadog_checks }}" datadog_config: hostname: "{{ hostname }}" logs_enabled: true From 508e6b97bc6a135e5e2b7b9e2d7b60a98bdc3739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 15:21:10 +0100 Subject: [PATCH 15/24] Remove duplicate grafana source list --- ansible/roles/monitoring/templates/grafana.list | 1 - 1 file changed, 1 deletion(-) delete mode 100644 ansible/roles/monitoring/templates/grafana.list diff --git a/ansible/roles/monitoring/templates/grafana.list b/ansible/roles/monitoring/templates/grafana.list deleted file mode 100644 index adbad20b..00000000 --- a/ansible/roles/monitoring/templates/grafana.list +++ /dev/null @@ -1 +0,0 @@ -deb https://packages.grafana.com/oss/deb stable main From f469c9425c60c4f658103721373f3d3008cd4509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 15:34:03 +0100 Subject: [PATCH 16/24] Fix datadog openmetrics config --- ansible/host_vars/monitoring.ooni.org/vars.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index cb86ec65..698f2fac 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -158,8 +158,11 @@ ooni_datadog_checks: - type: journald container_mode: true openmetrics: - openmetrics_endpoint: "http://prometheus.ooni.org:9090/metrics" - namespace: "ooniprom" - metrics: - - ".*" + instances: + - openmetrics_endpoint: "http://prometheus.ooni.org:9090/metrics" + namespace: "ooniprom" + metrics: + - "^prom.+" + - "^net_.+" + - "^go_.+" ... From 11644c103a39e6256d8f4323fc67a93cecc6f08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 16:09:20 +0100 Subject: [PATCH 17/24] Add statsd metric collection --- .../host_vars/backend-fsn.ooni.org/vars.yml | 18 ++++++++++++++++++ ansible/host_vars/monitoring.ooni.org/vars.yml | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/ansible/host_vars/backend-fsn.ooni.org/vars.yml b/ansible/host_vars/backend-fsn.ooni.org/vars.yml index f9700c35..d6262712 100644 --- a/ansible/host_vars/backend-fsn.ooni.org/vars.yml +++ b/ansible/host_vars/backend-fsn.ooni.org/vars.yml @@ -8,3 +8,21 @@ base_url: "https://ams-pg.ooni.org/" tor_targets: "{{ vault_tor_targets }}" psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}" digital_ocean_token: "{{ vault_digital_ocean_token }}" +ooni_datadog_checks: + journald: + logs: + - type: journald + container_mode: true + openmetrics: + instances: + - openmetrics_endpoint: "http://localhost:9090/metrics" + namespace: "ooniprom" + metrics: + - "^prom.+" + - "^net_.+" + - "^go_.+" + statsd: + instances: + - host: localhost + port: 8126 + service: "oonimonitoring" diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 698f2fac..643e55a8 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -165,4 +165,9 @@ ooni_datadog_checks: - "^prom.+" - "^net_.+" - "^go_.+" + statsd: + instances: + - host: localhost + port: 8126 + service: "oonimonitoring" ... From 6f3303fccaf694cd4d1fbb333a5769148aa0a46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 17:07:21 +0100 Subject: [PATCH 18/24] Scrape netdata via the netadata openmetrics endpoint --- ansible/host_vars/backend-fsn.ooni.org/vars.yml | 9 ++++----- ansible/host_vars/monitoring.ooni.org/vars.yml | 11 +++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ansible/host_vars/backend-fsn.ooni.org/vars.yml b/ansible/host_vars/backend-fsn.ooni.org/vars.yml index d6262712..e7b75ac1 100644 --- a/ansible/host_vars/backend-fsn.ooni.org/vars.yml +++ b/ansible/host_vars/backend-fsn.ooni.org/vars.yml @@ -21,8 +21,7 @@ ooni_datadog_checks: - "^prom.+" - "^net_.+" - "^go_.+" - statsd: - instances: - - host: localhost - port: 8126 - service: "oonimonitoring" + - openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus" + namespace: "ooninetdata" + metrics: + - "^netdata.+" diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 643e55a8..3ada88b8 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -159,15 +159,14 @@ ooni_datadog_checks: container_mode: true openmetrics: instances: - - openmetrics_endpoint: "http://prometheus.ooni.org:9090/metrics" + - openmetrics_endpoint: "http://localhost:9090/metrics" namespace: "ooniprom" metrics: - "^prom.+" - "^net_.+" - "^go_.+" - statsd: - instances: - - host: localhost - port: 8126 - service: "oonimonitoring" + - openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus" + namespace: "ooninetdata" + metrics: + - "^netdata.+" ... From 0afcc8a79696a5d5c3506e3cc0ca69d9ecb9e539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 30 Jan 2024 18:50:28 +0100 Subject: [PATCH 19/24] Fix netdata URL --- ansible/host_vars/backend-fsn.ooni.org/vars.yml | 8 ++++---- ansible/host_vars/monitoring.ooni.org/vars.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/host_vars/backend-fsn.ooni.org/vars.yml b/ansible/host_vars/backend-fsn.ooni.org/vars.yml index e7b75ac1..c09676bd 100644 --- a/ansible/host_vars/backend-fsn.ooni.org/vars.yml +++ b/ansible/host_vars/backend-fsn.ooni.org/vars.yml @@ -21,7 +21,7 @@ ooni_datadog_checks: - "^prom.+" - "^net_.+" - "^go_.+" - - openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus" - namespace: "ooninetdata" - metrics: - - "^netdata.+" + - metrics: + - ^netdata_statsd.* + namespace: ooninetdata + openmetrics_endpoint: http://localhost:19999/api/v1/allmetrics?format=prometheus×tamps=no&types=yes&help=yes diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 3ada88b8..62043afc 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -165,8 +165,8 @@ ooni_datadog_checks: - "^prom.+" - "^net_.+" - "^go_.+" - - openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus" - namespace: "ooninetdata" - metrics: - - "^netdata.+" + - metrics: + - ^netdata_statsd.* + namespace: ooninetdata + openmetrics_endpoint: http://localhost:19999/api/v1/allmetrics?format=prometheus×tamps=no&types=yes&help=yes ... From 20d65b9b14e1768fe5626a51764bfc63e0454d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 31 Jan 2024 08:48:41 +0100 Subject: [PATCH 20/24] Add federated metrics collection from prometheus --- ansible/host_vars/monitoring.ooni.org/vars.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 62043afc..7845b8af 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -166,7 +166,21 @@ ooni_datadog_checks: - "^net_.+" - "^go_.+" - metrics: - - ^netdata_statsd.* + - "^netdata_statsd.+" + excluded_metrics: + - ".+_datadog_.+" namespace: ooninetdata openmetrics_endpoint: http://localhost:19999/api/v1/allmetrics?format=prometheus×tamps=no&types=yes&help=yes + - openmetrics_endpoint: "http://localhost:9090/federate?match[]=\{job!=""\}" + namespace: "oonipromfdrtd" + metrics: + - "^oohelperd_.+" + - "^probe_.+" + - "^ooni_.+" + - "^go_.+" + - "^up.+" + - "^http_.+" + - "^netdata_statsd_.+" + excluded_metrics: + - ".+_datadog_.+" ... From d24f46f71fce6fe2cf920e37939d19d9578715c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 31 Jan 2024 08:50:40 +0100 Subject: [PATCH 21/24] Fix escaping --- ansible/host_vars/monitoring.ooni.org/vars.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 7845b8af..7a5a3e80 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -166,12 +166,12 @@ ooni_datadog_checks: - "^net_.+" - "^go_.+" - metrics: - - "^netdata_statsd.+" + - "^netdata_statsd.+" excluded_metrics: - - ".+_datadog_.+" + - ".+_datadog_.+" namespace: ooninetdata - openmetrics_endpoint: http://localhost:19999/api/v1/allmetrics?format=prometheus×tamps=no&types=yes&help=yes - - openmetrics_endpoint: "http://localhost:9090/federate?match[]=\{job!=""\}" + openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus×tamps=no&types=yes&help=yes" + - openmetrics_endpoint: 'http://localhost:9090/federate?match[]={job!=""}' namespace: "oonipromfdrtd" metrics: - "^oohelperd_.+" @@ -182,5 +182,5 @@ ooni_datadog_checks: - "^http_.+" - "^netdata_statsd_.+" excluded_metrics: - - ".+_datadog_.+" + - ".+_datadog_.+" ... From ecbabd60aa560bbd7f51f4d816b597fcd51cc4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 31 Jan 2024 11:12:07 +0100 Subject: [PATCH 22/24] Split prometheus metrics into separate instances to faciliate scraping --- ansible/host_vars/monitoring.ooni.org/vars.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 7a5a3e80..d35211f2 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -171,9 +171,23 @@ ooni_datadog_checks: - ".+_datadog_.+" namespace: ooninetdata openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus×tamps=no&types=yes&help=yes" - - openmetrics_endpoint: 'http://localhost:9090/federate?match[]={job!=""}' + prometheus: + instances: + - prometheus_url: 'http://localhost:9090/federate?match[]={job!=""}' namespace: "oonipromfdrtd" metrics: + - "oohelperd_*" + - "probe_*" + - "ooni_*" + - "go_*" + - "up*" + - "http_*" + - prometheus_url: 'http://localhost:9090/federate?match[]={job!=""}' + namespace: "oonipromstatsd" + metrics: + - "netdata_statsd_*" + metric_patterns: + include: - "^oohelperd_.+" - "^probe_.+" - "^ooni_.+" @@ -181,6 +195,6 @@ ooni_datadog_checks: - "^up.+" - "^http_.+" - "^netdata_statsd_.+" - excluded_metrics: + exclude: - ".+_datadog_.+" ... From c390d419ea1b82667ce0683eb4815d6b913258a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 5 Mar 2024 16:49:57 +0100 Subject: [PATCH 23/24] Remove datadog agent --- ansible/deploy-datadog-agent.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index 5469f804..0a50c609 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -13,8 +13,8 @@ {% if dd_tag_service is defined %}"service:{{ dd_tag_service }}",{% endif %} "configuration_manager:ansible" ] - roles: - - role: oodd - vars: - tags: "{{ datadog_tags }}" - hostname: "{{ inventory_hostname }}" + tasks: + - ansible.builtin.apt: + name: datadog-agent + state: absent + purge: yes From 1c31036f69335b52013bff64a35edc555273be6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 5 Mar 2024 16:51:05 +0100 Subject: [PATCH 24/24] Remove datadog agent --- ansible/deploy-datadog-agent.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ansible/deploy-datadog-agent.yml b/ansible/deploy-datadog-agent.yml index 0a50c609..e9688103 100644 --- a/ansible/deploy-datadog-agent.yml +++ b/ansible/deploy-datadog-agent.yml @@ -14,7 +14,8 @@ "configuration_manager:ansible" ] tasks: - - ansible.builtin.apt: - name: datadog-agent - state: absent - purge: yes + - name: remove datadog agent + ansible.builtin.apt: + name: datadog-agent + state: absent + purge: yes