Skip to content

Commit

Permalink
Re-organization of monitoring
Browse files Browse the repository at this point in the history
* Split playbooks into tiers
* Add support for clickhouse cluster monitoring
  • Loading branch information
hellais committed Dec 4, 2024
1 parent 118030f commit 0a8be55
Showing 20 changed files with 374 additions and 217 deletions.
13 changes: 13 additions & 0 deletions ansible/deploy-clickhouse.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
---
- name: Deploy oonidata clickhouse hosts
hosts:
- notebook.ooni.org
- data1.htz-fsn.prod.ooni.nu
#- data2.htz-fsn.prod.ooni.nu
- data3.htz-fsn.prod.ooni.nu
become: true
tags:
- clickhouse
roles:
- prometheus_node_exporter
- oonidata_clickhouse
12 changes: 12 additions & 0 deletions ansible/deploy-monitoring.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
- name: Update monitoring config
hosts: monitoring.ooni.org
become: true
tags:
- monitoring
roles:
- prometheus
- prometheus_blackbox_exporter
- prometheus_alertmanager


19 changes: 19 additions & 0 deletions ansible/deploy-tier0.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
- name: Include monitoring playbook
ansible.builtin.import_playbook: deploy-monitoring.yml

- name: Include clickhouse playbook
ansible.builtin.import_playbook: deploy-clickhouse.yml

- name: Deploy oonidata worker nodes
hosts:
- data1.htz-fsn.prod.ooni.nu
become: true
tags:
- oonidata_worker
roles:
- oonidata
vars:
enable_jupyterhub: false
enable_oonipipeline_worker: true
clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"
25 changes: 25 additions & 0 deletions ansible/deploy-tier2.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
---
- name: Setup OpenVPN server
hosts: openvpn-server1.ooni.io
become: true
remote_user: root
roles:
- ssh_users

- name: Deploy notebook host
hosts: notebook.ooni.org
become: true
tags:
- notebook
vars:
enable_oonipipeline_worker: false
roles:
- oonidata

# commented out due to the fact it requires manual config of ~/.ssh/config
#- name: Setup codesign box
# hosts: codesign-box
# become: true
# remote_user: ubuntu
# roles:
# - codesign_box
2 changes: 2 additions & 0 deletions ansible/group_vars/all/vars.yml
Original file line number Diff line number Diff line change
@@ -27,3 +27,5 @@ admin_usernames: [ art, mehul ]
root_usernames: [ art, mehul ]
non_admin_usernames: [ ]
deactivated_usernames: [ sbs, federico, sarath ]

prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"
20 changes: 19 additions & 1 deletion ansible/group_vars/clickhouse/vars.yml
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ clickhouse_config:
max_connections: 4096
keep_alive_timeout: 3
max_concurrent_queries: 100
max_server_memory_usage: 0
max_server_memory_usage: 21001001000
max_thread_pool_size: 10000
max_server_memory_usage_to_ram_ratio: 0.9
total_memory_profiler_step: 4194304
@@ -156,6 +156,10 @@ clickhouse_distributed_ddl:
clickhouse_default_profiles:
default:
readonly: 2
max_memory_usage: 11001001000
use_uncompressed_cache: 0
load_balancing: random
max_partitions_per_insert_block: 100
readonly:
readonly: 1
write:
@@ -196,3 +200,17 @@ clickhouse_default_quotas:
result_rows: 0
read_rows: 0
execution_time: 0

clickhouse_prometheus:
endpoint: "/metrics"
port: 9363
metrics: true
events: true
asynchronous_metrics: true
status_info: true

prometheus_nginx_proxy_config:
- location: /metrics/node_exporter
proxy_pass: http://127.0.0.1:8100/metrics
- location: /metrics/clickhouse
proxy_pass: http://127.0.0.1:9363/metrics
30 changes: 11 additions & 19 deletions ansible/inventory
Original file line number Diff line number Diff line change
@@ -1,32 +1,24 @@
[all]
# This requires manual setup of ~/.ssh/config
#codesign-box
[all:children]
htz-fsn
ghs-ams

[prod]
data.ooni.org
oonidata.ooni.org
monitoring.ooni.org
openvpn-server1.ooni.io
notebook.ooni.org
data1.htz-fsn.prod.ooni.nu
data2.htz-fsn.prod.ooni.nu
data3.htz-fsn.prod.ooni.nu

[dev]
oonidatatest.ooni.nu
## Role tags

[clickhouse]
notebook.ooni.org
data1.htz-fsn.prod.ooni.nu
data2.htz-fsn.prod.ooni.nu
data3.htz-fsn.prod.ooni.nu

[have_node_exporter]
## Location tags

[htz-fsn]
data.ooni.org
oonidata.ooni.org
openvpn-server1.ooni.io
monitoring.ooni.org
notebook.ooni.org
data1.htz-fsn.prod.ooni.nu
data2.htz-fsn.prod.ooni.nu
data3.htz-fsn.prod.ooni.nu
oonidatatest.ooni.nu

[ghs-ams]
openvpn-server1.ooni.io
71 changes: 4 additions & 67 deletions ansible/playbook.yml
Original file line number Diff line number Diff line change
@@ -7,71 +7,8 @@
tags:
- bootstrap

- name: Setup node_exporter on have_node_exporter hosts
hosts: all
become: yes
roles:
- prometheus_node_exporter
tags:
- prometheus_node_exporter

- name: Update monitoring config
hosts: monitoring.ooni.org
become: true
tags:
- monitoring
roles:
- prometheus
- prometheus_blackbox_exporter
- prometheus_alertmanager

- name: Setup OpenVPN server
hosts: openvpn-server1.ooni.io
become: true
remote_user: root
roles:
- ssh_users

- name: Deploy oonidata clickhouse hosts
hosts:
- data1.htz-fsn.prod.ooni.nu
#- data2.htz-fsn.prod.ooni.nu
- data3.htz-fsn.prod.ooni.nu
- notebook.ooni.org
become: true
tags:
- clickhouse
roles:
#- tailnet
- oonidata_clickhouse

- name: Deploy oonidata worker nodes
hosts:
- data1.htz-fsn.prod.ooni.nu
become: true
tags:
- oonidata_worker
roles:
- oonidata
vars:
enable_jupyterhub: false
enable_oonipipeline_worker: true
clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"

- name: Deploy notebook host
hosts: notebook.ooni.org
become: true
tags:
- notebook
vars:
enable_oonipipeline_worker: false
roles:
- oonidata
- name: Include tier0 playbook
ansible.builtin.import_playbook: deploy-tier0.yml

# commented out due to the fact it requires manual config of ~/.ssh/config
#- name: Setup codesign box
# hosts: codesign-box
# become: true
# remote_user: ubuntu
# roles:
# - codesign_box
- name: Include tier2 playbook
ansible.builtin.import_playbook: deploy-tier2.yml
1 change: 0 additions & 1 deletion ansible/requirements.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
- src: willshersystems.sshd
- src: nginxinc.nginx
- src: geerlingguy.certbot
- src: geerlingguy.node_exporter
- src: artis3n.tailscale
- src: https://github.com/idealista/clickhouse_role
scm: git
1 change: 1 addition & 0 deletions ansible/roles/nginx/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
nginx_user: nginx
Loading

0 comments on commit 0a8be55

Please sign in to comment.