Deploy airflow on data1 host

ansible/deploy-airflow.yml

@@ -1,7 +1,7 @@
 ---
 - name: Deploy airflow hosts
   hosts:
-    - data2.htz-fsn.prod.ooni.nu
+    - data1.htz-fsn.prod.ooni.nu
   become: true
   roles:
     - oonidata_airflow

ansible/deploy-tier0.yml

@@ -8,15 +8,5 @@
 - name: Include clickhouse playbook
   ansible.builtin.import_playbook: deploy-clickhouse.yml
 
-- name: Deploy oonidata worker nodes
-  hosts:
-    - data1.htz-fsn.prod.ooni.nu
-  become: true
-  tags:
-    - oonidata_worker
-  roles:
-    - oonidata
-  vars:
-    enable_jupyterhub: false
-    enable_oonipipeline_worker: true
-    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"
+- name: Include airflow playbook
+  ansible.builtin.import_playbook: deploy-airflow.yml

ansible/group_vars/airflow/vars.yml

@@ -8,6 +8,6 @@ airflow_admin_users:
     email: [email protected]
 airflow_fernet_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_fernet_key', profile='oonidevops_user_prod') }}"
 airflow_webserver_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_webserver_secret_key', profile='oonidevops_user_prod') }}"
-#airflow_executor: "LocalExecutor"
-airflow_extra_packages:
-  - virtualenv
+airflow_executor: "LocalExecutor"
+airflow_webserver_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_webserver_secret_key', profile='oonidevops_user_prod') }}"
+airflow_database_conn: "postgresql+psycopg2://airflow:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_postgresql_password', profile='oonidevops_user_prod') }}@ooni-tier0-postgres.c7mgscca82no.eu-central-1.rds.amazonaws.com/airflow"

ansible/inventory

@@ -11,7 +11,7 @@ data2.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
 
 [airflow]
-data2.htz-fsn.prod.ooni.nu
+data1.htz-fsn.prod.ooni.nu
 
 ## Location tags
 

ansible/roles/oonidata_airflow/tasks/main.yml

@@ -1,12 +1,47 @@
+- name: Checkout oonidata repo
+  ansible.builtin.git:
+    repo: 'https://github.com/ooni/data.git'
+    dest: /opt/oonidata
+    version: airflow
+
 - ansible.builtin.include_role:
     name: ooni.airflow_role
   tags:
     - oonidata
     - airflow
   vars:
+    airflow_app_home: /opt/airflow
+    airflow_dags_folder: /opt/oonidata/dags/
     airflow_webserver_host: "127.0.0.1"
     airflow_webserver_port: 8080
     airflow_webserver_base_url: "https://{{ airflow_public_fqdn }}/airflow"
+    airflow_environment_extra_vars:
+      - name: AIRFLOW_VAR_DATA_DIR
+        value: "{{ airflow_app_home }}/data_dir"
+    airflow_extra_packages:
+      - postgres
+      - virtualenv
+    airflow_services:
+      airflow_webserver:
+        service_name: airflow-webserver
+        enabled: true
+        running: true
+        state: started
+        path: airflow-webserver.service.j2
+      airflow_scheduler:
+        service_name: airflow-scheduler
+        enabled: true
+        running: true
+        state: started
+        path: airflow-scheduler.service.j2
+
+- name: Set correct permissions on oonidata repo dir
+  ansible.builtin.file:
+    path: /opt/oonidata
+    state: directory
+    mode: '0755'
+    owner: airflow
+    recurse: yes
 
 - ansible.builtin.include_role:
     name: nginx