Add support for node_exporter prometheus monitoring

ansible/README.md

@@ -8,7 +8,12 @@ pyenv activate ooni-devops
 
 Install deps:
 ```
-pip install ansible dnspython boto3
+pip install ansible dnspython boto3 passlib
+```
+
+Install ansible galaxy modules:
+```
+ansible-galaxy install -r requirements.yml
 ```
 
 Setup AWS credentials, you should add 2 profiles called `oonidevops_user_dev` and `oonidevops_user_prod` which have access to the development and production environment respectively

ansible/group_vars/all/vars.yml

@@ -26,4 +26,4 @@ ssh_users:
 admin_usernames: [ art, majakomel, mehul, norbel ]
 root_usernames: [ art, mehul ]
 non_admin_usernames: [ agrabeli ]
-deactivated_usernames: [ sbs, federico, sarath ]
+deactivated_usernames: [ sbs, federico, sarath ]

ansible/group_vars/dev/vars.yml

@@ -0,0 +1 @@
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"

ansible/group_vars/prod/vars.yml

@@ -0,0 +1 @@
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"

ansible/inventory

@@ -1,7 +1,11 @@
 [all]
-monitoring.ooni.org
-openvpn-server1.ooni.io
 # This requires manual setup of ~/.ssh/config
 #codesign-box
+
+[prod]
 data.ooni.org
-oonidatatest.ooni.nu
+monitoring.ooni.org
+openvpn-server1.ooni.io
+
+[dev]
+oonidatatest.ooni.nu

ansible/playbook-bootstrap.yml

@@ -2,6 +2,7 @@
 # OS setup with something other than ansible (eg. cloud-init)
 - name: Ensure ssh_users are synched on all hosts
   hosts: all
-  remote_user: root
+  #remote_user: root
+  become: yes
   roles:
-    - ssh_users
+    - bootstrap

ansible/requirements.yml

@@ -1,3 +1,4 @@
 - src: willshersystems.sshd
 - src: nginxinc.nginx
 - src: geerlingguy.certbot
+- src: geerlingguy.node_exporter

ansible/roles/bootstrap/tasks/main.yml

@@ -0,0 +1,40 @@
+- name: Set the hostname to inventory_hostname
+  ansible.builtin.hostname:
+    name: "{{ inventory_hostname }}"
+
+- name: Install common packages
+  ansible.builtin.apt:
+    name:
+      - bash-completion
+      - ca-certificates
+      - curl
+      - file
+      - git
+      - htop
+      - iotop
+      - lsof
+      - lvm2
+      - man-db
+      - mtr
+      - net-tools
+      - openssl
+      - python3-passlib
+      - rsync
+      - screen
+      - strace
+      - tcpdump
+      - tmux
+      - vim
+    state: latest
+    update_cache: yes
+    install_recommends: no
+
+- ansible.builtin.include_role:
+    name: nftables
+  tags:
+    - nftables
+
+- ansible.builtin.include_role:
+    name: prometheus_node_exporter
+  tags:
+    - node_exporter

ansible/roles/nftables/README.adoc

@@ -0,0 +1,25 @@
+Install nftables based firewall
+
+Set up /etc/ooni/nftables/
+
+Rules for specific services are *not* configured by this role
+
+When creating rules to accept TCP traffic from any IPv4/6 address,
+files are named with the port number to detect collisions.
+
+Example (also see roles/nftables/tasks/main.yml):
+
+/etc/ooni/nftables/tcp/8080.nft
+
+```
+add rule inet filter input tcp dport 8080 counter accept comment "MyService"
+```
+
+
+Otherwise:
+
+/etc/ooni/nftables/tcp/5432_postgres_internal.nft
+
+```
+add rule inet filter input ip saddr { 10.0.0.0/8, 192.168.0.0/16 } tcp dport 5432 counter accept comment "Internal PostgreSQL"
+```

ansible/roles/nftables/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: Reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded

ansible/roles/nftables/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: Install nftables
+  ansible.builtin.apt:
+    cache_valid_time: 86400
+    name: nftables
+  tags:
+    - nftables
+
+- name: create config dir
+  ansible.builtin.file:
+    path: /etc/ooni/nftables/tcp
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  tags:
+    - nftables
+
+- name: allow SSH
+  ansible.builtin.blockinfile:
+    path: /etc/ooni/nftables/tcp/22.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 22 counter accept comment "Incoming SSH"
+  tags:
+    - nftables
+
+- name: Overwrite nftables.conf
+  ansible.builtin.template:
+    src: templates/nftables.conf
+    dest: /etc/nftables.conf
+    mode: 0755
+    owner: root
+  notify:
+    - Reload nftables
+  tags:
+    - nftables
+
+- name: enable nftables service
+  ansible.builtin.systemd_service:
+    name: nftables
+    enabled: yes
+    state: started
+  tags:
+    - nftables

ansible/roles/nftables/templates/nftables.conf

@@ -0,0 +1,41 @@
+#!/usr/sbin/nft -f
+#
+# Nftables configuration script
+#
+# Managed by ansible
+# roles/nftables/templates/nftables.conf
+#
+# The ruleset is applied atomically
+
+flush ruleset
+
+table inet filter {
+  chain input {
+    type filter hook input priority 0;
+    policy drop;
+    iif lo accept comment "Accept incoming traffic from localhost"
+    ct state invalid drop
+    ct state established,related accept comment "Accept traffic related to outgoing connections"
+    icmp type echo-request accept
+    icmpv6 type echo-request counter packets 0 bytes 0 accept
+    icmpv6 type { nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 1 accept
+    icmpv6 type { nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 255 counter packets 1 bytes 72 accept
+  }
+
+  chain forward {
+    type filter hook forward priority 0;
+    policy accept;
+  }
+
+  chain output {
+    type filter hook output priority 0;
+    policy accept;
+  }
+}
+
+# Configure TCP traffic rules
+include "/etc/ooni/nftables/tcp/*.nft"
+
+# Configure any other rule
+include "/etc/ooni/nftables/*.nft"
+

ansible/roles/oonidata/handlers/main.yml

@@ -3,6 +3,7 @@
     name: jupyterhub
     state: restarted
     daemon_reload: true
+
 - name: Restart oonipipeline-worker
   ansible.builtin.systemd_service:
     name: oonipipeline-worker

ansible/roles/prometheus_node_exporter/handlers/main.yml

@@ -0,0 +1,21 @@
+- name: Test nginx config
+  command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen:
+    - Restart nginx-prometheus
+    - Reload nginx-prometheus
+
+- name: Reload nginx
+  ansible.builtin.systemd_service:
+    name: nginx
+    state: reloaded
+
+- name: Restart nginx
+  ansible.builtin.systemd_service:
+    name: nginx
+    state: restarted
+
+- name: Reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded

ansible/roles/prometheus_node_exporter/tasks/main.yml

@@ -0,0 +1,56 @@
+- ansible.builtin.include_role:
+    name: nginx
+  tags:
+    - nginx
+    - node_exporter
+
+- ansible.builtin.include_role:
+    name: geerlingguy.node_exporter
+  vars:
+    node_exporter_host: "localhost"
+    node_exporter_port: 8100
+  tags:
+    - node_exporter
+
+- name: create ooni configuration directory
+  ansible.builtin.file:
+    path: "/etc/ooni/"
+    state: directory
+    owner: root
+  tags:
+    - node_exporter
+
+- name: Add a user to a password file and ensure permissions are set
+  community.general.htpasswd:
+    path: /etc/ooni/prometheus_passwd
+    name: prom
+    password: "{{ prometheus_metrics_password }}"
+    owner: root
+    group: www-data
+    mode: 0640
+  tags:
+    - node_exporter
+
+- name: Setup oonidata nginx config
+  ansible.builtin.template:
+    src: nginx-prometheus.j2
+    dest: /etc/nginx/sites-enabled/01-prometheus
+    mode: "0655"
+  notify:
+    - Restart nginx
+  tags:
+    - node_exporter
+    - config
+
+- name: Allow prometheus monitoring
+  ansible.builtin.blockinfile:
+    path: /etc/ooni/nftables/tcp/9100.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9100 counter accept comment "Incoming prometheus monitoring"
+  notify:
+    - Reload nftables
+  tags:
+    - nftables
+    - node_exporter
+    - config

ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2

@@ -0,0 +1,20 @@
+# ansible-managed in ooni/devops.git
+
+server {
+    listen 9100;
+
+    server_name _;
+    access_log  /var/log/nginx/{{ inventory_hostname }}.access.log;
+    error_log   /var/log/nginx/{{ inventory_hostname }}.log warn;
+
+    location /metrics {
+        auth_basic           "Administrator’s Area";
+        auth_basic_user_file /etc/ooni/prometheus_passwd;
+
+        proxy_pass http://127.0.0.1:8100;
+
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+}