Move clickhouse related functionality into specific role

ooni · Nov 11, 2024 · 8b33a52 · 8b33a52
1 parent c79c006
commit 8b33a52
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 183 deletions.
diff --git a/ansible/roles/clickhouse/tasks/main.yml b/ansible/roles/clickhouse/tasks/main.yml
diff --git a/ansible/roles/clickhouse/templates/ooni_users.xml b/ansible/roles/clickhouse/templates/ooni_users.xml
diff --git a/ansible/roles/oonidata/defaults/main.yml b/ansible/roles/oonidata/defaults/main.yml
@@ -7,7 +7,6 @@ admin_group_name: admin
 enable_oonipipeline_worker: true
 enable_oonidata_proxy: false
 enable_jupyterhub: true
-enable_clickhouse: true
 conda_forge_packages:
   - seaborn
   - dask
@@ -27,73 +26,3 @@ apt_packages:
   - curl
   - git
   - socat
-
-clickhouse_version: "24.3.13.40"
-
-clickhouse_default_profiles:
-  default:
-    readonly: 1
-  write:
-    readonly: 0
-
-clickhouse_default_users:
-  - user:
-    name: default
-    password:
-    networks:
-      - 0.0.0.0
-    profile: default
-    quota: default
-  - user:
-    name: write
-    password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}"
-    networks:
-      - 127.0.0.1
-    profile: write
-    quota: default
-
-clickhouse_keeper:
-  tcp_port: 9181
-  log_storage_path: /var/lib/clickhouse/coordination/log
-  snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots
-  coordination_settings:
-    operation_timeout_ms: 10000
-    session_timeout_ms: 30000
-    raft_logs_level: trace
-  keeper_servers:
-  - keeper_server:
-    server: data1.htz-fsn.prod.ooni.nu
-    id: 1
-    hostname: clickhouse1.prod.ooni.io
-    port: 9234
-
-  - keeper_server:
-    server: data2.htz-fsn.prod.ooni.nu
-    id: 2
-    hostname: clickhouse2.prod.ooni.io
-    port: 9234
-
-  - keeper_server:
-    server: data3.htz-fsn.prod.ooni.nu
-    id: 3
-    hostname: clickhouse3.prod.ooni.io
-    port: 9234
-
-clickhouse_remote_servers:
-  - server:
-    servername: oonidata_cluster
-    secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}"
-    shards:
-      - shard:
-        replicas:
-          - replica:
-            host: clickhouse1.prod.ooni.io
-            port: 9000
-      - shard:
-          - replica:
-            host: clickhouse2.prod.ooni.io
-            port: 9000
-      - shard:
-          - replica:
-            host: clickhouse3.prod.ooni.io
-            port: 9000
diff --git a/ansible/roles/oonidata/meta/requirements.yml b/ansible/roles/oonidata/meta/requirements.yml
@@ -1,6 +1,3 @@
 ---
 dependencies:
   - geerlingguy.certbot
-  - src: idealista.clickhouse_role
-    scm: git
-    version: 3.5.1
diff --git a/ansible/roles/oonidata/tasks/main.yml b/ansible/roles/oonidata/tasks/main.yml
@@ -62,12 +62,6 @@
     - oonidata
     - oonipipeline
 
-- ansible.builtin.import_tasks: clickhouse.yml
-  when: enable_clickhouse
-  tags:
-    - oonidata
-    - clickhouse
-
 - name: Write oonidataproxy service
   when: enable_oonidata_proxy
   ansible.builtin.template:

diff --git a/ansible/roles/oonidata_clickhouse/defaults/main.yml b/ansible/roles/oonidata_clickhouse/defaults/main.yml
@@ -0,0 +1,69 @@
+clickhouse_version: "24.3.13.40"
+
+clickhouse_default_profiles:
+  default:
+    readonly: 1
+  write:
+    readonly: 0
+
+clickhouse_default_users:
+  - user:
+    name: default
+    password:
+    networks:
+      - 0.0.0.0
+    profile: default
+    quota: default
+  - user:
+    name: write
+    password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') }}"
+    networks:
+      - 127.0.0.1
+    profile: write
+    quota: default
+
+clickhouse_keeper:
+  tcp_port: 9181
+  log_storage_path: /var/lib/clickhouse/coordination/log
+  snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots
+  coordination_settings:
+    operation_timeout_ms: 10000
+    session_timeout_ms: 30000
+    raft_logs_level: trace
+  keeper_servers:
+  - keeper_server:
+    server: data1.htz-fsn.prod.ooni.nu
+    id: 1
+    hostname: clickhouse1.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: data2.htz-fsn.prod.ooni.nu
+    id: 2
+    hostname: clickhouse2.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: data3.htz-fsn.prod.ooni.nu
+    id: 3
+    hostname: clickhouse3.prod.ooni.io
+    port: 9234
+
+clickhouse_remote_servers:
+  - server:
+    servername: oonidata_cluster
+    secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}"
+    shards:
+      - shard:
+        replicas:
+          - replica:
+            host: clickhouse1.prod.ooni.io
+            port: 9000
+      - shard:
+          - replica:
+            host: clickhouse2.prod.ooni.io
+            port: 9000
+      - shard:
+          - replica:
+            host: clickhouse3.prod.ooni.io
+            port: 9000
diff --git a/ansible/roles/oonidata_clickhouse/handlers/main.yml b/ansible/roles/oonidata_clickhouse/handlers/main.yml
@@ -0,0 +1 @@
+
diff --git a/ansible/roles/oonidata_clickhouse/meta/requirements.yml b/ansible/roles/oonidata_clickhouse/meta/requirements.yml
@@ -0,0 +1,5 @@
+---
+dependencies:
+  - src: idealista.clickhouse_role
+    scm: git
+    version: 3.5.1
diff --git a/ansible/roles/oonidata_clickhouse/tasks/main.yml b/ansible/roles/oonidata_clickhouse/tasks/main.yml
@@ -0,0 +1,5 @@
+- ansible.builtin.include_role:
+    name: idealista.clickhouse_role
+  tags:
+    - oonidata
+    - clickhouse