Monitoring dd

README.md

@@ -10,7 +10,8 @@ It is recommended you use a [python virtualenv](https://virtualenv.pypa.io/en/la
 Once you activate the virtualenv you can install the correct version of ansible and dnspython with:
 
 ```
-pip install ansible==2.9.16
+# pip install ansible==2.9.16
+pip install ansible==2.10.7
 ```
 
 

ansible/deploy-datadog-agent.yml

@@ -0,0 +1,21 @@
+# Runnning this role requires ansible >= 2.10 and running:
+# ansible-galaxy install datadog.datadog
+- hosts: have_datadog_agent
+  pre_tasks:
+    - name: set datadog tags
+      set_fact:
+        datadog_tags: >-
+          [
+          {% if dd_tag_cloud_provider is defined %}"cloud_provider:{{ dd_tag_cloud_provider }}",{% endif %}
+          {% if dd_tag_availability_zone is defined %}"availability-zone:{{ dd_tag_availability_zone }}",{% endif %}
+          {% if dd_tag_availability_zone is defined %}"region:{{ dd_tag_availability_zone }}",{% endif %}
+          {% if dd_tag_env is defined %}"env:{{ dd_tag_env }}",{% endif %}
+          {% if dd_tag_service is defined %}"service:{{ dd_tag_service }}",{% endif %}
+          "configuration_manager:ansible"
+          ]
+  tasks:
+    - name: remove datadog agent
+      ansible.builtin.apt:
+        name: datadog-agent
+        state: absent
+        purge: yes

ansible/group_vars/have_datadog_agent/vars.yml

@@ -0,0 +1 @@
+ooni_datadog_api_key: "{{ vault_ooni_datadog_api_key }}"

ansible/group_vars/have_datadog_agent/vault

@@ -0,0 +1,8 @@
+$ANSIBLE_VAULT;1.1;AES256
+63316539646632353261333838373064663165633733333362363461336665323532386332383864
+6262363162383032323764633466646435346566333564350a326636623161383663616563396435
+35646338373037636538303937336338643032653636633362343139393539663566343833643063
+6630633238366535370a373834633763616663653334636561653261643238303730613365366164
+32386235303839346130393463373161633938343336383433646261616166326339313764623530
+36356538636138656336316438333537663663323535666630333065303732373836373662333831
+313937663936666436623235363838343561

ansible/host_vars/backend-fsn.ooni.org/vars.yml

@@ -8,3 +8,20 @@ base_url: "https://ams-pg.ooni.org/"
 tor_targets: "{{ vault_tor_targets }}"
 psiphon_config: "{{ vault_orchestra_psiphon_config_file_content }}"
 digital_ocean_token: "{{ vault_digital_ocean_token }}"
+ooni_datadog_checks:
+  journald:
+    logs:
+    - type: journald
+      container_mode: true
+  openmetrics:
+    instances:
+      - openmetrics_endpoint: "http://localhost:9090/metrics"
+        namespace: "ooniprom"
+        metrics:
+          - "^prom.+"
+          - "^net_.+"
+          - "^go_.+"
+      - metrics:
+        - ^netdata_statsd.*
+        namespace: ooninetdata
+        openmetrics_endpoint: http://localhost:19999/api/v1/allmetrics?format=prometheus&timestamps=no&types=yes&help=yes

ansible/host_vars/monitoring.ooni.org/vars.yml

@@ -152,4 +152,49 @@ blackbox_jobs:
     module: icmp
     targets: "{{ groups['dom0'] | list }}"
 
+ooni_datadog_checks:
+  journald:
+    logs:
+    - type: journald
+      container_mode: true
+  openmetrics:
+    instances:
+      - openmetrics_endpoint: "http://localhost:9090/metrics"
+        namespace: "ooniprom"
+        metrics:
+          - "^prom.+"
+          - "^net_.+"
+          - "^go_.+"
+      - metrics:
+          - "^netdata_statsd.+"
+        excluded_metrics:
+          - ".+_datadog_.+"
+        namespace: ooninetdata
+        openmetrics_endpoint: "http://localhost:19999/api/v1/allmetrics?format=prometheus&timestamps=no&types=yes&help=yes"
+  prometheus:
+    instances:
+      - prometheus_url: 'http://localhost:9090/federate?match[]={job!=""}'
+        namespace: "oonipromfdrtd"
+        metrics:
+          - "oohelperd_*"
+          - "probe_*"
+          - "ooni_*"
+          - "go_*"
+          - "up*"
+          - "http_*"
+      - prometheus_url: 'http://localhost:9090/federate?match[]={job!=""}'
+        namespace: "oonipromstatsd"
+        metrics:
+          - "netdata_statsd_*"
+    metric_patterns:
+        include:
+          - "^oohelperd_.+"
+          - "^probe_.+"
+          - "^ooni_.+"
+          - "^go_.+"
+          - "^up.+"
+          - "^http_.+"
+          - "^netdata_statsd_.+"
+        exclude:
+          - ".+_datadog_.+"
 ...

ansible/inventory.yml

@@ -82,6 +82,35 @@ all:
 
     have_tor: {}
 
+    have_datadog_agent:
+      hosts:
+        #backend-fsn.ooni.org:
+        #  dd_tag_cloud_provider: hetzner
+        #  dd_tag_availability_zone: eu-fsn
+        #  dd_tag_env: prod
+        #  dd_tag_service: oonibackend
+        monitoring.ooni.org:
+          dd_tag_cloud_provider: hetzner
+          dd_tag_availability_zone: eu-fsn
+          dd_tag_env: prod
+          dd_tag_service: monitoring
+        backend-hel.ooni.org:
+          dd_tag_cloud_provider: hetzner
+          dd_tag_availability_zone: eu-hel
+          dd_tag_env: test
+          dd_tag_service: oonibackend
+        data.ooni.org:
+          dd_tag_cloud_provider: hetzner
+          dd_tag_availability_zone: eu-fsn
+          dd_tag_env: test
+          dd_tag_service: oonidata
+        oonidata.ooni.org:
+          dd_tag_cloud_provider: hetzner
+          dd_tag_availability_zone: eu-fsn
+          dd_tag_env: prod
+          dd_tag_service: oonidata
+
+
     monitoring:
       hosts:
         monitoring.ooni.org: {}
@@ -96,6 +125,7 @@ all:
         backend-hel.ooni.org: {}
         monitoring.ooni.org: {}
         oonidata.ooni.org: {}
+        data.ooni.org: {}
 
     probe_services:
       hosts:

ansible/roles/oodd/defaults/main.yml

@@ -0,0 +1,5 @@
+ooni_datadog_checks:
+  journald:
+    logs:
+      - type: journald
+        container_mode: true

ansible/roles/oodd/meta/main.yml

@@ -0,0 +1,11 @@
+dependencies:
+  - role: datadog.datadog
+    become: yes
+    vars:
+      datadog_api_key: "{{ ooni_datadog_api_key }}"
+      datadog_site: "datadoghq.eu"
+      datadog_checks: "{{ ooni_datadog_checks }}"
+      datadog_config:
+        hostname: "{{ hostname }}"
+        logs_enabled: true
+        tags: "{{ tags }}"

ansible/roles/oodd/requirements.yml

@@ -0,0 +1 @@
+- name: datadog.datadog

ansible/roles/oodd/tasks/main.yml

@@ -0,0 +1,5 @@
+- name: Add dd-agent user to systemd-journal group
+  ansible.builtin.user:
+    name: dd-agent
+    groups: systemd-journal
+    append: yes

ansible/vault

@@ -1,6 +1,3 @@
 #!/bin/bash -ex
 gitroot=$(git rev-parse --show-toplevel)
 ansible-vault "$@" --vault-password-file "${gitroot}/ansible/password-pipe"
-if [ "$1" != view ] && [ "$1" != decrypt ] && [ "$1" != rekey ]; then
-  "${gitroot}/scripts/mk-travis-vault"
-fi