From 2f0137fd27ab507e626f13d14b342cf2b4ae877c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Fri, 20 Dec 2024 16:46:30 +0100
Subject: [PATCH] Deploy airflow on data1 host

---
 ansible/deploy-airflow.yml                    |  2 +-
 ansible/deploy-tier0.yml                      | 14 ++------
 ansible/group_vars/airflow/vars.yml           |  6 ++--
 ansible/inventory                             |  2 +-
 ansible/roles/oonidata_airflow/tasks/main.yml | 35 +++++++++++++++++++
 5 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/ansible/deploy-airflow.yml b/ansible/deploy-airflow.yml
index 2f07c62..15331f9 100644
--- a/ansible/deploy-airflow.yml
+++ b/ansible/deploy-airflow.yml
@@ -1,7 +1,7 @@
 ---
 - name: Deploy airflow hosts
   hosts:
-    - data2.htz-fsn.prod.ooni.nu
+    - data1.htz-fsn.prod.ooni.nu
   become: true
   roles:
     - oonidata_airflow
diff --git a/ansible/deploy-tier0.yml b/ansible/deploy-tier0.yml
index 7c11a8c..3657d54 100644
--- a/ansible/deploy-tier0.yml
+++ b/ansible/deploy-tier0.yml
@@ -8,15 +8,5 @@
 - name: Include clickhouse playbook
   ansible.builtin.import_playbook: deploy-clickhouse.yml
 
-- name: Deploy oonidata worker nodes
-  hosts:
-    - data1.htz-fsn.prod.ooni.nu
-  become: true
-  tags:
-    - oonidata_worker
-  roles:
-    - oonidata
-  vars:
-    enable_jupyterhub: false
-    enable_oonipipeline_worker: true
-    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"
+- name: Include airflow playbook
+  ansible.builtin.import_playbook: deploy-airflow.yml
diff --git a/ansible/group_vars/airflow/vars.yml b/ansible/group_vars/airflow/vars.yml
index 6d9e034..e494671 100644
--- a/ansible/group_vars/airflow/vars.yml
+++ b/ansible/group_vars/airflow/vars.yml
@@ -8,6 +8,6 @@ airflow_admin_users:
     email: admin@ooni.org
 airflow_fernet_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_fernet_key', profile='oonidevops_user_prod') }}"
 airflow_webserver_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_webserver_secret_key', profile='oonidevops_user_prod') }}"
-#airflow_executor: "LocalExecutor"
-airflow_extra_packages:
-  - virtualenv
+airflow_executor: "LocalExecutor"
+airflow_webserver_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_webserver_secret_key', profile='oonidevops_user_prod') }}"
+airflow_database_conn: "postgresql+psycopg2://airflow:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_postgresql_password', profile='oonidevops_user_prod') }}@ooni-tier0-postgres.c7mgscca82no.eu-central-1.rds.amazonaws.com/airflow"
diff --git a/ansible/inventory b/ansible/inventory
index 1e10693..1d3c41e 100644
--- a/ansible/inventory
+++ b/ansible/inventory
@@ -11,7 +11,7 @@ data2.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
 
 [airflow]
-data2.htz-fsn.prod.ooni.nu
+data1.htz-fsn.prod.ooni.nu
 
 ## Location tags
 
diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml
index ba4f07a..f4195a7 100644
--- a/ansible/roles/oonidata_airflow/tasks/main.yml
+++ b/ansible/roles/oonidata_airflow/tasks/main.yml
@@ -1,12 +1,47 @@
+- name: Checkout oonidata repo
+  ansible.builtin.git:
+    repo: 'https://github.com/ooni/data.git'
+    dest: /opt/oonidata
+    version: airflow
+
 - ansible.builtin.include_role:
     name: ooni.airflow_role
   tags:
     - oonidata
     - airflow
   vars:
+    airflow_app_home: /opt/airflow
+    airflow_dags_folder: /opt/oonidata/dags/
     airflow_webserver_host: "127.0.0.1"
     airflow_webserver_port: 8080
     airflow_webserver_base_url: "https://{{ airflow_public_fqdn }}/airflow"
+    airflow_environment_extra_vars:
+      - name: AIRFLOW_VAR_DATA_DIR
+        value: "{{ airflow_app_home }}/data_dir"
+    airflow_extra_packages:
+      - postgres
+      - virtualenv
+    airflow_services:
+      airflow_webserver:
+        service_name: airflow-webserver
+        enabled: true
+        running: true
+        state: started
+        path: airflow-webserver.service.j2
+      airflow_scheduler:
+        service_name: airflow-scheduler
+        enabled: true
+        running: true
+        state: started
+        path: airflow-scheduler.service.j2
+
+- name: Set correct permissions on oonidata repo dir
+  ansible.builtin.file:
+    path: /opt/oonidata
+    state: directory
+    mode: '0755'
+    owner: airflow
+    recurse: yes
 
 - ansible.builtin.include_role:
     name: nginx