diff --git a/airflow/dags/ingest_plan_communal.py b/airflow/dags/ingest_plan_communal.py new file mode 100644 index 000000000..665dde744 --- /dev/null +++ b/airflow/dags/ingest_plan_communal.py @@ -0,0 +1,55 @@ +from airflow.decorators import dag, task +from include.domain.container import Container +from include.pools import DBT_POOL +from include.utils import get_dbt_command_from_directory +from pendulum import datetime + +URL = "https://api-aln.datahub.din.developpement-durable.gouv.fr/sudocuh/enquetes/ref/plan/communal/CSV?annee_cog=2024" + + +@dag( + start_date=datetime(2024, 1, 1), + schedule="@once", + catchup=False, + doc_md=__doc__, + max_active_runs=1, + default_args={"owner": "Alexis Athlani", "retries": 3}, + tags=["SUDOCUH"], +) +def ingest_plan_communal(): + bucket_name = "airflow-staging" + plan_communal_filename = "plan_communal.csv" + table_name = "sudocuh_plan_communal" + + @task.python + def download_plan_communal() -> str: + return ( + Container() + .remote_to_s3_file_handler() + .download_http_file_and_upload_to_s3( + url=URL, + s3_key=plan_communal_filename, + s3_bucket=bucket_name, + ) + ) + + @task.python + def ingest_plan_communal() -> int | None: + return ( + Container() + .s3_csv_file_to_db_table_handler() + .ingest_s3_csv_file_to_db_table( + s3_bucket=bucket_name, + s3_key=plan_communal_filename, + table_name=table_name, + ) + ) + + @task.bash(retries=0, trigger_rule="all_success", pool=DBT_POOL) + def dbt_build(): + return get_dbt_command_from_directory(cmd="dbt build -s sudocuh+") + + (download_plan_communal() >> ingest_plan_communal() >> dbt_build()) + + +ingest_plan_communal() diff --git a/airflow/include/sql/sparte/models/for_app/for_app_commune.sql b/airflow/include/sql/sparte/models/for_app/for_app_commune.sql index 778c3ba5e..3f4025462 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_commune.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_commune.sql @@ -53,7 +53,8 @@ select end as surface_artif, commune.surface / 10000 as area, ST_Transform(commune.geom, 4326) as mpoly, - consommation.correction_status as consommation_correction_status + consommation.correction_status as consommation_correction_status, + competence.competence_planification from {{ ref('commune') }} as commune left join @@ -72,3 +73,7 @@ left join {{ ref("consommation_cog_2024") }} as consommation on commune.code = consommation.commune_code +left join + {{ ref('competence_plan_commune')}} as competence + on + commune.code = competence.commune_code diff --git a/airflow/include/sql/sparte/models/for_app/schema.yml b/airflow/include/sql/sparte/models/for_app/schema.yml index be273816c..1be24d318 100644 --- a/airflow/include/sql/sparte/models/for_app/schema.yml +++ b/airflow/include/sql/sparte/models/for_app/schema.yml @@ -182,6 +182,9 @@ models: - not_null - name: for_app_commune columns: + - name: competence_planification + data_tests: + - not_null - name: insee data_tests: - not_null diff --git a/airflow/include/sql/sparte/models/sudocuh/competence_plan_commune.sql b/airflow/include/sql/sparte/models/sudocuh/competence_plan_commune.sql new file mode 100644 index 000000000..865a179b2 --- /dev/null +++ b/airflow/include/sql/sparte/models/sudocuh/competence_plan_commune.sql @@ -0,0 +1,14 @@ +{{ config(materialized='table') }} + +SELECT + commune.code as commune_code, + CASE + WHEN code_etat_libelle_bcsi LIKE '%Compétence commune' THEN TRUE + ELSE FALSE + END AS competence_planification +FROM + {{ ref('commune') }} as commune +LEFT JOIN + {{ source('public', 'sudocuh_plan_communal') }} as plan_communal +ON + commune.code = plan_communal.code_insee diff --git a/airflow/include/sql/sparte/models/sudocuh/models.yml b/airflow/include/sql/sparte/models/sudocuh/models.yml index 29bb33286..953f0acd4 100644 --- a/airflow/include/sql/sparte/models/sudocuh/models.yml +++ b/airflow/include/sql/sparte/models/sudocuh/models.yml @@ -46,3 +46,4 @@ sources: tables: - name: sudocuh_scot - name: sudocuh_scot_communes + - name: sudocuh_plan_communal diff --git a/public_data/models/administration/Commune.py b/public_data/models/administration/Commune.py index 1ec6f0cb5..696350e82 100644 --- a/public_data/models/administration/Commune.py +++ b/public_data/models/administration/Commune.py @@ -74,6 +74,8 @@ class Meta: choices=ConsommationCorrectionStatus.choices, ) + competence_planification = models.BooleanField("Compétence planification", default=False) + # DataColorationMixin properties that need to be set when heritating default_property = "insee" # need to be set correctly to work default_color = "Yellow"