update establishment queries (#1051)

Signed-off-by: Raphaël Courivaud <[email protected]>

analytics/dbt/macros/production/events/get_last_establishment_event_status.sql

@@ -0,0 +1,25 @@
+{% macro get_last_establishment_event_status(user_source, event_name, all_users=false) %}
+(
+    SELECT
+        u.establishment_id,
+        events.new_status AS status,
+        events.event_status_label AS status_label,
+        events.new_occupancy AS occupancy,
+        events.new_sub_status,
+        events.created_at,
+        ROW_NUMBER() OVER (PARTITION BY u.establishment_id ORDER BY events.created_at DESC) AS row_num
+    FROM {{ ref('int_production_events') }} AS events
+    LEFT JOIN  {{ ref('int_production_users') }} AS u ON events.created_by = u.id
+    WHERE
+        1=1 
+        {% if event_name == 'suivi' %}
+            AND events.status_changed = TRUE
+        {% elif event_name == 'occupation' %}
+            AND events.occupancy_changed = TRUE
+        {% else %}
+        {% endif %}
+        {% if not all_users %}
+            AND events.user_source = '{{ user_source }}'
+        {% endif %}
+)
+{% endmacro %}

analytics/dbt/macros/production/events/select_last_establishment_event.sql

@@ -0,0 +1,14 @@
+{% macro select_last_establishment_event(establishment_table, event_table, column_suffix) %}
+(
+    SELECT 
+        e.id AS establishment_id,
+        evt.status AS last_event_status_{{ column_suffix }},
+        evt.status_label AS last_event_status_label_{{ column_suffix }},
+        evt.occupancy AS last_event_occupancy_{{ column_suffix }},
+        evt.new_sub_status AS last_event_sub_status_label_{{ column_suffix }},
+        evt.created_at AS last_event_date_{{ column_suffix }}
+    FROM {{ establishment_table }} AS e
+    LEFT JOIN {{ event_table }} AS evt
+        ON evt.establishment_id = e.id AND evt.row_num = 1
+)
+{% endmacro %}

analytics/dbt/models/intermediate/ff/int_ff_history_housing.sql

@@ -0,0 +1,20 @@
+WITH ff_history as (
+                       SELECT ff_idlocal, 'ff-' || ff_millesime as file_year, 
+                       FROM {{ ref('int_ff_ext_2023') }}
+                       UNION ALL
+                       SELECT ff_idlocal, 'ff-' || ff_millesime as file_year, 
+                       FROM {{ ref('int_ff_ext_2022') }}
+                       UNION ALL
+                       SELECT ff_idlocal, 'ff-' || ff_millesime as file_year,
+                       FROM {{ ref('int_ff_ext_2021') }}
+                       UNION ALL
+                       SELECT ff_idlocal, 'ff-' || ff_millesime as file_year,
+                       FROM {{ ref('int_ff_ext_2020') }}
+                       UNION ALL
+                       SELECT ff_idlocal, 'ff-' || ff_millesime as file_year,
+                       FROM {{ ref('int_ff_ext_2019') }})
+SELECT 
+    ff_idlocal as local_id, 
+    listagg(file_year, ',') as file_years, 
+FROM ff_history lh
+GROUP BY local_id

analytics/dbt/models/intermediate/ff/int_ff_owners_dedup.sql

@@ -9,8 +9,12 @@ SELECT ff_owner_idpersonne,
         max(ff_owner_birth_date) as ff_owner_birth_date,
         max(ff_owner_lastname) as ff_owner_lastname,
         max(ff_owner_firstname) as ff_owner_firstname,
+        max(ff_owner_firstnames) as ff_owner_firstnames,
+        max(ff_owner_birth_lastname) as ff_owner_birth_lastname,
         max(ff_owner_city) as ff_owner_city,
         max(ff_owner_fullname) as ff_owner_fullname,
+        max(ff_locprop) as ff_locprop,
+        max(ff_locproptxt) as ff_locproptxt,
         max(ff_owner_category) as ff_owner_category,
         max(ff_owner_category_text) as ff_owner_category_text
 FROM {{ ref('stg_ff_owners') }}

analytics/dbt/models/intermediate/production/int_production_establishment_events_last_status.sql

@@ -0,0 +1,17 @@
+WITH 
+    establishment_status_user_followup AS {{ get_last_establishment_event_status('user', 'suivi') }},
+    establishment_status_user_occupancy AS {{ get_last_establishment_event_status('user', "occupation") }},
+    last_establishment_status_user_followup AS {{ select_last_establishment_event(ref('int_production_establishments'), 'establishment_status_user_followup', 'user_followup') }},
+    last_establishment_status_user_occupancy AS {{ select_last_establishment_event(ref('int_production_establishments'), 'establishment_status_user_occupancy', 'user_occupancy') }}
+
+SELECT
+    e.id AS establishment_id,
+    hsuf.last_event_status_user_followup,
+    hsuf.last_event_status_label_user_followup,
+    hsuf.last_event_date_user_followup,
+    hsuf.last_event_sub_status_label_user_followup,
+    hszo.last_event_occupancy_user_occupancy,
+    hszo.last_event_date_user_occupancy
+FROM {{ ref('int_production_establishments') }} AS e
+LEFT JOIN last_establishment_status_user_followup AS hsuf ON e.id = hsuf.establishment_id
+LEFT JOIN last_establishment_status_user_occupancy AS hszo ON e.id = hszo.establishment_id

analytics/dbt/models/intermediate/production/int_production_establishments.sql

@@ -30,5 +30,4 @@ SELECT est.*,
         WHEN est.kind IN ('SDED', 'SDER') THEN TRUE
         ELSE FALSE
     END AS covered_by_state_service
-FROM {{ ref('stg_production_establishments') }} est
-
+FROM {{ ref('stg_production_establishments') }} est

analytics/dbt/models/intermediate/production/int_production_housing_establishments.sql

@@ -0,0 +1,13 @@
+
+SELECT
+    ph.id as housing_id,
+    STRING_AGG(DISTINCT pel.establishment_id, ' | ') as establishment_ids,
+    ARRAY_AGG(DISTINCT pel.establishment_id) as establishment_ids_array, 
+    CASE SUM(peu.user_number)
+        WHEN 0 THEN FALSE
+        ELSE TRUE
+    END as has_users
+FROM {{ ref('int_production_housing') }} as ph
+LEFT JOIN {{ ref('int_production_establishments_localities') }} as pel ON pel.geo_code = ph.geo_code
+LEFT JOIN {{ ref('int_production_establishments_users') }} as peu ON peu.establishment_id = pel.establishment_id
+GROUP BY housing_id

analytics/dbt/models/marts/common/marts_common_cities.sql

@@ -8,34 +8,6 @@ SELECT
     MIN(department_number) AS department_number,
     MIN(region_name) AS region_name,
     MIN(region_geojson_name) AS region_geojson_name,
-    SUM(vac_2024_moins_1an) AS total_vac_2024_moins_1an,
-    SUM(vac_2024_1an) AS total_vac_2024_1an,
-    SUM(vac_2024_2ans) AS total_vac_2024_2ans,
-    SUM(vac_2024_3ans) AS total_vac_2024_3ans,
-    SUM(vac_2024_4ans_plus) AS total_vac_2024_4ans_plus,
-    SUM(vac_2023_moins_1an) AS total_vac_2023_moins_1an,
-    SUM(vac_2023_1an) AS total_vac_2023_1an,
-    SUM(vac_2023_2ans) AS total_vac_2023_2ans,
-    SUM(vac_2023_3ans) AS total_vac_2023_3ans,
-    SUM(vac_2023_4ans_plus) AS total_vac_2023_4ans_plus,
-    SUM(vac_2023_total) AS total_vac_2023_total,
-    SUM(vac_2024_total) AS total_vac_2024_total,
-    SUM(vac_2024_plus_2ans) AS total_vac_2024_plus_2ans,
-    SUM(vac_2024_moins_2ans) AS total_vac_2024_moins_2ans,
-
-    SUM(vac_2020_moins_1an) AS total_vac_2020_moins_1an,
-    SUM(vac_2020_1an) AS total_vac_2020_1an,
-    SUM(vac_2020_2ans) AS total_vac_2020_2ans,
-    SUM(vac_2020_3ans) AS total_vac_2020_3ans,
-    SUM(vac_2020_4ans_plus) AS total_vac_2020_4ans_plus,
-    SUM(vac_2020_total) AS total_vac_2020_total,
-    SUM(ext_2023_gc.nombre_logements) AS nombre_logements_prives_2023,
-    SUM(ext_2020_gc.nombre_logements) AS nombre_logements_prives_2020,
-
-    100 * (SUM(vac_2024_plus_2ans)  / SUM(ext_2023_gc.nombre_logements)) AS part_logement_vacants_plus_2ans_2024_parc_prive_2023, --Part des logements vacants >2 ans en 2024
-    100 * (SUM(vac_2024_moins_2ans)  / SUM(ext_2023_gc.nombre_logements)) AS part_logement_vacants_moins_2ans_2024_parc_prive_2023, -- Part des logements ≤ 2 ans en 2023
-    100 * (SUM(vac_2023_total)  / SUM(ext_2023_gc.nombre_logements)) AS part_logement_vacants_2023_parc_prive_2023,-- Part des logements vacants en 2024
-    100 * (SUM(vac_2024_total)  / SUM(ext_2023_gc.nombre_logements)) AS part_logement_vacants_2024_parc_prive_2023,-- Part des logements vacants en 2023
     MAX(ca1.is_in) AS tlv1,  -- Prend 1 s'il y a au moins un arrondissement où la valeur est 1
     MAX(ca2.is_in) AS tlv2,  -- Idem
     MAX(action_coeur_de_ville) AS action_coeur_de_ville,
@@ -47,16 +19,6 @@ SELECT
     MAX(pig) AS pig,
     MAX(cort.signed) as ort_signed,
     MAX(cort.signed_at) as ort_signed_at,
-    MAX(cf.has_fusion) AS has_fusion,
-    MAX(cf.libelle_source) AS fusion_libelle_source,
-    MAX(cf.libelle_destination) AS fusion_libelle_destination,
-    MAX(cf.geo_code_source) AS fusion_geo_code_source,
-    MAX(cf.year) as fusion_date,
-    MAX(cs.has_scission) AS has_scission,
-    MAX(cs.libelle_source) AS scission_libelle_source,
-    MAX(cs.libelle_destination) AS scission_libelle_destination,
-    MAX(cs.geo_code_source) AS scission_geo_code_source,
-    MAX(cs.year) as scission_date,
     MAX(UU_name) as uu_name,
     MAX(UU_code) as uu_code,
     MAX(EPCI_code) as epci_code,
@@ -75,12 +37,7 @@ LEFT JOIN {{ ref('int_common_article_232_1') }} ca1 ON ca1.geo_code = ccm.city_c
 LEFT JOIN {{ ref('int_common_article_232_2') }} ca2 ON ca2.geo_code = ccm.city_code -- les données des articles 1 et 2 sont basé sur les codes insee des villes et non des arrondissements
 LEFT JOIN {{ ref('int_common_ngeo_anct_cog_2023') }} ncac ON ncac.geo_code = cc.geo_code
 LEFT JOIN {{ ref('int_common_table_appartenance') }} cta ON cta.geo_code = cc.geo_code
-LEFT JOIN {{ ref('int_common_fusions') }} cf ON cf.geo_code_destination = cc.geo_code
-LEFT JOIN {{ ref('int_common_scissions') }} cs ON cs.geo_code_destination = cc.geo_code
 LEFT JOIN {{ ref('int_common_opah') }} co ON co.geo_code = cc.geo_code
-LEFT JOIN {{ ref('int_lovac_geo_code') }} lgc ON lgc.geo_code = cc.geo_code
-LEFT JOIN {{ ref('int_ff_ext_2023_geo_code') }} ext_2023_gc ON ext_2023_gc.geo_code = cc.geo_code
-LEFT JOIN {{ ref('int_ff_ext_2020_geo_code') }} ext_2020_gc ON ext_2020_gc.geo_code = cc.geo_code
 LEFT JOIN {{ ref('int_common_ort') }} cort ON ccm.city_code = cort.geo_code
 LEFT JOIN {{ ref('int_production_geo_code_establishments') }} gce ON gce.geo_code = cc.geo_code
 GROUP BY

analytics/dbt/models/marts/import/import_gold_housing.sql

@@ -0,0 +1,54 @@
+WITH ff_data AS (
+    SELECT
+        ff_idlocal,
+        ff_jdatat AS last_mutation_date,
+        ff_dcapec2 AS cadastral_classification,
+        ff_dcntpa AS plot_area,
+        ff_geomloc AS geometry,
+        ff_ccthp AS occupancy,
+        ST_X(ST_Transform(ST_GeomFromWKB(FROM_HEX(ff_geomloc)), 'EPSG:2154', 'EPSG:4326')) AS latitude,
+        ST_Y(ST_Transform(ST_GeomFromWKB(FROM_HEX(ff_geomloc)), 'EPSG:2154', 'EPSG:4326')) AS longitude
+    FROM {{ ref('int_ff_ext_2023') }}
+),
+lovac_2024 AS (
+    SELECT 
+        ff_idlocal,
+        ff_dcntpa AS plot_area,
+        ff_jdatat AS last_mutation_date,
+        dvf_datemut AS last_transaction_date,
+        dvf_valeurfonc AS last_transaction_value,
+        ffh_ccthp AS occupancy_history,
+        ban_latitude AS latitude,
+        ban_longitude AS longitude,
+        'lovac-2024' AS data_file_years
+    FROM {{ ref('int_lovac_fil_2024') }}
+),
+lovac_2023 AS (
+    SELECT 
+        ff_idlocal, 
+        ff_dcntpa AS plot_area,
+        ff_jdatat AS last_mutation_date,
+        dvf_datemut AS last_transaction_date,
+        dvf_valeurfonc AS last_transaction_value,
+        ffh_ccthp AS occupancy_history,
+        ban_latitude AS latitude,
+        ban_longitude AS longitude,
+        'lovac-2023' AS data_file_years
+    FROM {{ ref('int_lovac_fil_2023') }}
+), 
+lovac_history AS (
+    SELECT * FROM {{ ref('int_lovac_history_housing') }}
+)
+SELECT 
+    COALESCE(l24.ff_idlocal, l23.ff_idlocal, fd.ff_idlocal) AS local_id,
+    COALESCE(l24.plot_area, l23.plot_area, fd.plot_area) AS plot_area,
+    fd.cadastral_classification AS cadastral_classification,
+    COALESCE(l24.latitude, l23.latitude, fd.latitude) AS latitude,
+    COALESCE(l24.longitude, l23.longitude, fd.longitude) AS longitude,
+    COALESCE(l24.last_mutation_date, l23.last_mutation_date, fd.last_mutation_date) AS last_mutation_date,
+    COALESCE(l24.last_transaction_date, l23.last_transaction_date) AS last_transaction_date,
+    COALESCE(l24.last_transaction_value, l23.last_transaction_value) AS last_transaction_value,
+    COALESCE(l24.occupancy_history, l23.occupancy_history) AS occupancy_history
+FROM ff_data fd
+FULL OUTER JOIN lovac_2024 l24 ON fd.ff_idlocal = l24.ff_idlocal
+FULL OUTER JOIN lovac_2023 l23 ON COALESCE(fd.ff_idlocal, l24.ff_idlocal) = l23.ff_idlocal

analytics/dbt/models/marts/import/import_gold_housing_file_years.sql

@@ -0,0 +1,42 @@
+WITH ff_data AS (
+    SELECT
+        ff_idlocal, 
+        ff_ccthp AS occupancy
+    FROM {{ ref('int_ff_ext_2023') }}
+),
+lovac_2024 AS (
+    SELECT 
+        ff_idlocal
+    FROM {{ ref('int_lovac_fil_2024') }}
+),
+lovac_2023 AS (
+    SELECT 
+        ff_idlocal
+    FROM {{ ref('int_lovac_fil_2023') }}
+), 
+lovac_history AS (
+    SELECT * FROM {{ ref('int_lovac_history_housing') }}
+)
+SELECT 
+    COALESCE(l24.ff_idlocal, l23.ff_idlocal, fd.ff_idlocal) AS local_id,
+    CASE 
+        -- Si présent dans LOVAC 2024, absent de LOVAC 2023 et loué dans FF 2023
+        WHEN l24.ff_idlocal IS NOT NULL 
+             AND l23.ff_idlocal IS NULL 
+             AND fd.occupancy = 'L' THEN 'lovac-2024,ff-23'
+
+        -- Si présent uniquement dans LOVAC 2024
+        WHEN l24.ff_idlocal IS NOT NULL THEN 'lovac-2024'
+
+        -- Si présent uniquement dans LOVAC 2023
+        WHEN l23.ff_idlocal IS NOT NULL THEN 'lovac-2023'
+
+        -- Si loué dans FF 2023
+        WHEN fd.occupancy = 'L' THEN 'ff-2023-locatif'
+
+        ELSE NULL
+    END AS data_file_years
+FROM ff_data fd
+FULL OUTER JOIN lovac_2024 l24 ON fd.ff_idlocal = l24.ff_idlocal
+FULL OUTER JOIN lovac_2023 l23 ON COALESCE(fd.ff_idlocal, l24.ff_idlocal) = l23.ff_idlocal
+WHERE fd.occupancy = 'L'

analytics/dbt/models/marts/production/marts_production_campaigns.sql

@@ -1,3 +1,9 @@
+{{
+        config(
+            materialized='table',
+            unique_key='campaign_id',
+        )
+}}
 
 SELECT 
     CAST(pc.id as VARCHAR) as campaign_id,

analytics/dbt/models/marts/production/marts_production_establishments.sql

@@ -1,10 +1,18 @@
 
+{{
+        config(
+            materialized='table',
+            unique_key='establishment_id',
+        )
+}}
+
 SELECT 
     CAST(pe.id as VARCHAR) AS establishment_id,
-    pe.*, peu.*, pec.*, peg.*, pep.*, pecc.*
+    pe.*, peu.*, pec.*, peg.*, pep.*, pecc.*, pes.*
 FROM {{ ref('int_production_establishments') }} pe 
 LEFT JOIN {{ ref('int_production_establishments_users')}} peu ON pe.id = peu.establishment_id
 LEFT JOIN {{ ref('int_production_establishments_campaigns')}} pec ON pe.id = pec.establishment_id
 LEFT JOIN {{ ref('int_production_establishments_groups')}} peg ON pe.id = peg.establishment_id
 LEFT JOIN {{ ref('int_production_establishments_perimeters')}} pep ON pe.id = pep.establishment_id
-LEFT JOIN {{ ref('int_production_establishments_campaigns_contacts') }} pecc ON pe.id = pecc.establishment_id
+LEFT JOIN {{ ref('int_production_establishments_campaigns_contacts') }} pecc ON pe.id = pecc.establishment_id
+LEFT JOIN {{ ref('int_production_establishment_events_last_status') }} pes ON pe.id = pes.establishment_id

analytics/dbt/models/marts/production/marts_production_groups.sql

@@ -1,4 +1,3 @@
-
 SELECT
     pg.*
 FROM {{ ref('int_production_groups') }}  pg

analytics/dbt/models/marts/production/marts_production_housing.sql

@@ -1,3 +1,9 @@
+{{
+        config(
+            materialized='table',
+            unique_key='housing_id',
+        )
+}}
 SELECT 
         CAST(h.id as VARCHAR) as housing_id,
         h.* EXCLUDE (vacancy_reasons, precisions), 
@@ -21,7 +27,6 @@ SELECT
         hs.last_event_date_occupancy,
         CASE WHEN energy_consumption_bdnb IN ('F', 'G') THEN TRUE ELSE FALSE END as energy_sieve, 
         CASE WHEN vacancy_start_year < DATE_PART('year', CURRENT_DATE) - 3 THEN TRUE ELSE FALSE END as vacant_two_years,
-        CASE WHEN peugc.user_number IS NOT NULL THEN TRUE ELSE FALSE END as is_on_user_teritory, 
         CASE WHEN c.opah > 2 THEN TRUE ELSE FALSE END as is_in_opah_teritory,
         c.tlv1 as is_in_tlv1_teritory,
         c.tlv2 as is_in_tlv2_teritory,
@@ -38,10 +43,13 @@ SELECT
         c.region_name,
         c.region_geojson_name,
         phc.*, 
-        phg.*
+        phg.*, 
+        phe.establishment_ids,
+        phe.establishment_ids_array,
+        phe.has_users as is_on_user_teritory
 FROM {{ ref('int_production_housing') }} h
 LEFT JOIN {{ ref('int_production_housing_last_status') }} hs ON h.id = hs.housing_id
 LEFT JOIN {{ ref('marts_common_cities') }} c ON h.city_code = c.city_code
-LEFT JOIN {{ ref('int_production_establishment_geo_code') }} peugc ON peugc.geo_code = h.geo_code AND user_number > 0
 LEFT JOIN {{ ref('int_production_housing_campaigns') }} phc ON phc.housing_id = h.id
-LEFT JOIN {{ ref('int_production_housing_groups') }} phg ON phg.housing_id = h.id
+LEFT JOIN {{ ref('int_production_housing_groups') }} phg ON phg.housing_id = h.id
+LEFT JOIN {{ ref('int_production_housing_establishments') }} phe ON phe.housing_id = h.id

analytics/dbt/models/marts/production/marts_production_owners.sql

@@ -1,3 +1,9 @@
+{{
+        config(
+            materialized='table',
+            unique_key='owner_id',
+        )
+}}
 
 SELECT 
     CAST(po.id AS VARCHAR) as owner_id,

analytics/dbt/models/marts/production/marts_production_users.sql

@@ -1,3 +1,9 @@
+{{
+        config(
+            materialized='table',
+            unique_key='user_id',
+        )
+}}
 
 SELECT
     CAST(pu.id AS VARCHAR) user_id,

analytics/dbt/models/staging/ff/stg_ff_owners.sql

@@ -9,6 +9,10 @@ SELECT
     try_strptime(CAST(jdatnss AS VARCHAR), '{{ var("dateFormat") }}') AS ff_owner_birth_date,
     dnomus as ff_owner_lastname,
     dprnus as ff_owner_firstname,
+    dprnlp as ff_owner_firstnames,
+    dnomlp as ff_owner_birth_lastname,
+    locprop as ff_locprop,
+    locproptxt as ff_locproptxt,
     dlign6 as ff_owner_city,
     REGEXP_EXTRACT(dlign6, '\d{5}') AS ff_owner_postal_code,
     ddenom as ff_owner_fullname,

analytics/dbt/profiles.yml

@@ -8,6 +8,7 @@ duckdb_profile:
         - httpfs
         - parquet
         - s3
+        - spatial
       settings:
         s3_endpoint: "{{ env_var('S3_ENDPOINT_URL') }}" 
         s3_access_key_id: "{{ env_var('S3_ACCESS_KEY_ID') }}"