Merge pull request #601 from MTES-MCT/feat-admin-express-epci

Ajout du chargement des EPCI au dag ingest_admin_express
MTES-MCT · Oct 9, 2024 · 2070334 · 2070334
2 parents 4f513c7 + f52081c
commit 2070334
Show file tree

Hide file tree

Showing 9 changed files with 108 additions and 12 deletions.
diff --git a/airflow/include/admin_express/sources.json b/airflow/include/admin_express/sources.json
@@ -5,45 +5,50 @@
         "srid": 2154,
         "shapefile_to_table": {
             "COMMUNE.shp": "commune_metropole",
-            "DEPARTEMENT.shp": "departement_metropole"
+            "DEPARTEMENT.shp": "departement_metropole",
+            "EPCI.shp": "epci_metropole"
         },
-        "dbt_selector": "commune_metropole.sql+ departement_metropole.sql+"
+        "dbt_selector": "commune_metropole.sql+ departement_metropole.sql+ epci_metropole.sql+"
     }, {
         "name": "Guadeloupe",
         "url": "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG/ADMIN-EXPRESS-COG_3-2__SHP_RGAF09UTM20_GLP_2024-02-22/ADMIN-EXPRESS-COG_3-2__SHP_RGAF09UTM20_GLP_2024-02-22.7z",
         "srid": 32620,
         "shapefile_to_table": {
             "COMMUNE.shp": "commune_guadeloupe",
-            "DEPARTEMENT.shp": "departement_guadeloupe"
+            "DEPARTEMENT.shp": "departement_guadeloupe",
+            "EPCI.shp": "epci_guadeloupe"
         },
-        "dbt_selector": "commune_guadeloupe.sql+ departement_guadeloupe.sql+"
+        "dbt_selector": "commune_guadeloupe.sql+ departement_guadeloupe.sql+ epci_guadeloupe.sql+"
 
     }, {
         "name": "Martinique",
         "url": "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG/ADMIN-EXPRESS-COG_3-2__SHP_RGAF09UTM20_MTQ_2024-02-22/ADMIN-EXPRESS-COG_3-2__SHP_RGAF09UTM20_MTQ_2024-02-22.7z",
         "srid": 32620,
         "shapefile_to_table": {
             "COMMUNE.shp": "commune_martinique",
-            "DEPARTEMENT.shp": "departement_martinique"
+            "DEPARTEMENT.shp": "departement_martinique",
+            "EPCI.shp": "epci_martinique"
         },
-        "dbt_selector": "commune_martinique.sql+ departement_martinique.sql+"
+        "dbt_selector": "commune_martinique.sql+ departement_martinique.sql+ epci_martinique.sql+"
     }, {
         "name": "Guyanne",
         "url": "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG/ADMIN-EXPRESS-COG_3-2__SHP_UTM22RGFG95_GUF_2024-02-22/ADMIN-EXPRESS-COG_3-2__SHP_UTM22RGFG95_GUF_2024-02-22.7z",
         "srid": 2972,
         "shapefile_to_table": {
             "COMMUNE.shp": "commune_guyane",
-            "DEPARTEMENT.shp": "departement_guyane"
+            "DEPARTEMENT.shp": "departement_guyane",
+            "EPCI.shp": "epci_guyane"
         },
-        "dbt_selector": "commune_guyane.sql+ departement_guyane.sql+"
+        "dbt_selector": "commune_guyane.sql+ departement_guyane.sql+ epci_guyane.sql+"
     }, {
         "name": "La Réunion",
         "url": "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG/ADMIN-EXPRESS-COG_3-2__SHP_RGR92UTM40S_REU_2024-02-22/ADMIN-EXPRESS-COG_3-2__SHP_RGR92UTM40S_REU_2024-02-22.7z",
         "srid": 2975,
         "shapefile_to_table": {
             "COMMUNE.shp": "commune_reunion",
-            "DEPARTEMENT.shp": "departement_reunion"
+            "DEPARTEMENT.shp": "departement_reunion",
+            "EPCI.shp": "epci_reunion"
         },
-        "dbt_selector": "commune_reunion.sql+ departement_reunion.sql+"
+        "dbt_selector": "commune_reunion.sql+ departement_reunion.sql+ epci_reunion.sql+"
     }
 ]
diff --git a/airflow/include/sql/sparte/macros/admin_express/epci.sql b/airflow/include/sql/sparte/macros/admin_express/epci.sql
@@ -0,0 +1,13 @@
+{% macro epci(source_table_name) %}
+    {{ config(materialized='table') }}
+
+    SELECT
+        id,
+        nom as name,
+        code_siren as code,
+        nature,
+        ST_Area(geom) as surface,
+        geom
+    FROM
+        {{ source('public', source_table_name) }}
+{% endmacro %}
diff --git a/airflow/include/sql/sparte/models/admin_express/epci.sql b/airflow/include/sql/sparte/models/admin_express/epci.sql
@@ -0,0 +1,35 @@
+{{
+    config(
+        materialized='table',
+        indexes=[
+            {'columns': ['id'], 'type': 'btree'},
+            {'columns': ['code'], 'type': 'btree'},
+            {'columns': ['name'], 'type': 'btree'},
+            {'columns': ['geom'], 'type': 'gist'}
+        ])
+}}
+
+SELECT
+    *,
+    32620 AS srid_source
+FROM {{ ref('epci_guadeloupe') }}
+UNION ALL
+SELECT
+    *,
+    32620 AS srid_source
+FROM {{ ref('epci_martinique') }}
+UNION ALL
+SELECT
+    *,
+    2972 AS srid_source
+FROM {{ ref('epci_guyane') }}
+UNION ALL
+SELECT
+    *,
+    2975 AS srid_source
+FROM {{ ref('epci_reunion') }}
+UNION ALL
+SELECT
+    *,
+    2154 AS srid_source
+FROM {{ ref('epci_metropole') }}
diff --git a/airflow/include/sql/sparte/models/admin_express/epci_guadeloupe.sql b/airflow/include/sql/sparte/models/admin_express/epci_guadeloupe.sql
@@ -0,0 +1 @@
+{{ epci('epci_guadeloupe') }}
diff --git a/airflow/include/sql/sparte/models/admin_express/epci_guyane.sql b/airflow/include/sql/sparte/models/admin_express/epci_guyane.sql
@@ -0,0 +1 @@
+{{ epci('epci_guyane') }}
diff --git a/airflow/include/sql/sparte/models/admin_express/epci_martinique.sql b/airflow/include/sql/sparte/models/admin_express/epci_martinique.sql
@@ -0,0 +1 @@
+{{ epci('epci_martinique') }}
diff --git a/airflow/include/sql/sparte/models/admin_express/epci_metropole.sql b/airflow/include/sql/sparte/models/admin_express/epci_metropole.sql
@@ -0,0 +1 @@
+{{ epci('epci_metropole') }}
diff --git a/airflow/include/sql/sparte/models/admin_express/epci_reunion.sql b/airflow/include/sql/sparte/models/admin_express/epci_reunion.sql
@@ -0,0 +1 @@
+{{ epci('epci_reunion') }}
diff --git a/airflow/include/sql/sparte/models/admin_express/schema.yml b/airflow/include/sql/sparte/models/admin_express/schema.yml
@@ -1,17 +1,52 @@
 
 version: 2
 
+
+code_is_unique_and_not_null: &code_is_unique_and_not_null
+  - name: code
+    data_tests:
+    - not_null
+    - unique
+
+
 models:
   - name: commune
+    columns: *code_is_unique_and_not_null
   - name: commune_guadeloupe
+    columns: *code_is_unique_and_not_null
   - name: commune_guyane
+    columns: *code_is_unique_and_not_null
   - name: commune_martinique
+    columns: *code_is_unique_and_not_null
   - name: commune_reunion
+    columns: *code_is_unique_and_not_null
+  - name: commune_metropole
+    columns: *code_is_unique_and_not_null
   - name: departement
+    columns: *code_is_unique_and_not_null
   - name: departement_guadeloupe
+    columns: *code_is_unique_and_not_null
   - name: departement_guyane
+    columns: *code_is_unique_and_not_null
   - name: departement_martinique
+    columns: *code_is_unique_and_not_null
   - name: departement_reunion
+    columns: *code_is_unique_and_not_null
+  - name: departement_metropole
+    columns: *code_is_unique_and_not_null
+  - name: epci
+    columns: *code_is_unique_and_not_null
+  - name: epci_guadeloupe
+    columns: *code_is_unique_and_not_null
+  - name: epci_guyane
+    columns: *code_is_unique_and_not_null
+  - name: epci_martinique
+    columns: *code_is_unique_and_not_null
+  - name: epci_reunion
+    columns: *code_is_unique_and_not_null
+  - name: epci_metropole
+    columns: *code_is_unique_and_not_null
+
 
 sources:
   - name: public
@@ -26,5 +61,8 @@ sources:
     - name: departement_guyane
     - name: departement_martinique
     - name: departement_reunion
-    - name: epci
-    - name: region
+    - name: epci_metropole
+    - name: epci_guadeloupe
+    - name: epci_guyane
+    - name: epci_martinique
+    - name: epci_reunion