From ab40c2f3a071af261787950709607e6754137d5c Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Thu, 14 Nov 2024 14:37:00 +0100 Subject: [PATCH 01/14] feat(conso): add aggregation by all combinaison --- .../majic/merge_majic_by_admin_level.sql | 41 ++++++++++++ .../sql/sparte/models/admin_express/epci.sql | 53 +++++++-------- .../sql/sparte/models/admin_express/ept.sql | 38 +++++++++++ .../aggregated/consommation_departement.sql | 8 +++ .../majic/aggregated/consommation_epci.sql | 8 +++ .../majic/aggregated/consommation_region.sql | 8 +++ .../majic/aggregated/consommation_scot.sql | 8 +++ .../models/majic/consommation_cog_2024.sql | 64 ++++++++----------- .../sql/sparte/models/majic/schema.yml | 21 +++++- 9 files changed, 186 insertions(+), 63 deletions(-) create mode 100644 airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql create mode 100644 airflow/include/sql/sparte/models/admin_express/ept.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql diff --git a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql new file mode 100644 index 000000000..50d042a34 --- /dev/null +++ b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql @@ -0,0 +1,41 @@ +{% macro merge_majic_by_admin_level( + group_by_column, + code_name +) %} +SELECT + {{ group_by_column }} as {{ code_name }}, + {% set last_available_year = 2022 %} + {% set first_available_year = 2009 %} + {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} + {% set ns = namespace(continued=false) %} + {% for type_conso_suffix in type_conso_suffixes %} + {% for start_year in range(2009, last_available_year + 1) %} + {% for end_year in range(2009, last_available_year + 1) -%} + {% if start_year > end_year -%} + {% set ns.continued = true %} + {% continue %} + {% else %} + {% set ns.continued = false %} + {% endif %} + sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) + as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} + {% if not loop.last and not ns.continued -%}, {% endif %} + {% endfor %} {% if not loop.last and not ns.continued -%}, {% endif %} + {% endfor %} + {% if not loop.last -%}, {% endif %} + {% endfor %} +FROM + {{ ref('consommation_cog_2024') }} as consommation +LEFT JOIN + {{ ref('commune') }} + ON commune.code = consommation.commune_code +LEFT JOIN + {{ ref('scot_communes') }} as scot_communes + ON commune.code = scot_communes.commune_code +-- la condition suivante est nécessaire car un grand nombre de communes ne fait pas partie d'un SCOT, +-- et plus rarement certaines communes ne font pas partie d'un EPCI +WHERE + {{ group_by_column }} IS NOT NULL +GROUP BY + {{ group_by_column }} +{% endmacro %} diff --git a/airflow/include/sql/sparte/models/admin_express/epci.sql b/airflow/include/sql/sparte/models/admin_express/epci.sql index 4e74c4033..c46b09e2d 100644 --- a/airflow/include/sql/sparte/models/admin_express/epci.sql +++ b/airflow/include/sql/sparte/models/admin_express/epci.sql @@ -8,28 +8,31 @@ {'columns': ['geom'], 'type': 'gist'} ]) }} - -SELECT - *, - 32620 AS srid_source -FROM {{ ref('epci_guadeloupe') }} -UNION ALL -SELECT - *, - 32620 AS srid_source -FROM {{ ref('epci_martinique') }} -UNION ALL -SELECT - *, - 2972 AS srid_source -FROM {{ ref('epci_guyane') }} -UNION ALL -SELECT - *, - 2975 AS srid_source -FROM {{ ref('epci_reunion') }} -UNION ALL -SELECT - *, - 2154 AS srid_source -FROM {{ ref('epci_metropole') }} +with together as ( + SELECT + *, + 32620 AS srid_source + FROM {{ ref('epci_guadeloupe') }} + UNION ALL + SELECT + *, + 32620 AS srid_source + FROM {{ ref('epci_martinique') }} + UNION ALL + SELECT + *, + 2972 AS srid_source + FROM {{ ref('epci_guyane') }} + UNION ALL + SELECT + *, + 2975 AS srid_source + FROM {{ ref('epci_reunion') }} + UNION ALL + SELECT + *, + 2154 AS srid_source + FROM {{ ref('epci_metropole') }} +) +SELECT * FROM together +WHERE is_ept = false diff --git a/airflow/include/sql/sparte/models/admin_express/ept.sql b/airflow/include/sql/sparte/models/admin_express/ept.sql new file mode 100644 index 000000000..2c8e37f93 --- /dev/null +++ b/airflow/include/sql/sparte/models/admin_express/ept.sql @@ -0,0 +1,38 @@ +{{ + config( + materialized='table', + indexes=[ + {'columns': ['id'], 'type': 'btree'}, + {'columns': ['code'], 'type': 'btree'}, + {'columns': ['name'], 'type': 'btree'}, + {'columns': ['geom'], 'type': 'gist'} + ]) +}} +with together as ( + SELECT + *, + 32620 AS srid_source + FROM {{ ref('epci_guadeloupe') }} + UNION ALL + SELECT + *, + 32620 AS srid_source + FROM {{ ref('epci_martinique') }} + UNION ALL + SELECT + *, + 2972 AS srid_source + FROM {{ ref('epci_guyane') }} + UNION ALL + SELECT + *, + 2975 AS srid_source + FROM {{ ref('epci_reunion') }} + UNION ALL + SELECT + *, + 2154 AS srid_source + FROM {{ ref('epci_metropole') }} +) +SELECT * FROM together +WHERE is_ept = true diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql new file mode 100644 index 000000000..44b261acd --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} + +{{ + merge_majic_by_admin_level( + 'departement', + 'code_departement' + ) +}} diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql new file mode 100644 index 000000000..882294f12 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} + +{{ + merge_majic_by_admin_level( + 'epci', + 'code_epci' + ) +}} diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql new file mode 100644 index 000000000..24dea9597 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} + +{{ + merge_majic_by_admin_level( + 'region', + 'code_region' + ) +}} diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql new file mode 100644 index 000000000..f2d2808bf --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} + +{{ + merge_majic_by_admin_level( + 'id_scot', + 'code_scot' + ) +}} diff --git a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql index 78a3822cf..2f0524f38 100644 --- a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql +++ b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql @@ -129,42 +129,32 @@ together as ( select *, 'MISSING_FROM_SOURCE' as correction_status from missing_from_source ) select - *, - ( - conso_2011_2012 + - conso_2012_2013 + - conso_2013_2014 + - conso_2014_2015 + - conso_2015_2016 + - conso_2016_2017 + - conso_2017_2018 + - conso_2018_2019 + - conso_2019_2020 + - conso_2020_2021 - ) as conso_2011_2021, - ( - conso_2011_2012_activite + - conso_2012_2013_activite + - conso_2013_2014_activite + - conso_2014_2015_activite + - conso_2015_2016_activite + - conso_2016_2017_activite + - conso_2017_2018_activite + - conso_2018_2019_activite + - conso_2019_2020_activite + - conso_2020_2021_activite - ) as conso_2011_2021_activite, - ( - conso_2011_2012_habitat + - conso_2012_2013_habitat + - conso_2013_2014_habitat + - conso_2014_2015_habitat + - conso_2015_2016_habitat + - conso_2016_2017_habitat + - conso_2017_2018_habitat + - conso_2018_2019_habitat + - conso_2019_2020_habitat + - conso_2020_2021_habitat - ) as conso_2011_2021_habitat + commune_code, + correction_status, + {% set last_available_year = 2022 %} + {% set first_available_year = 2009 %} + {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} + {% set ns = namespace(continued=false) %} + {% for type_conso_suffix in type_conso_suffixes %} + {% for start_year in range(2009, last_available_year + 1) %} + {% for end_year in range(2009, last_available_year + 1) -%} + {% if start_year > end_year -%} + {% set ns.continued = true %} + {% continue %} + {% else %} + {% set ns.continued = false %} + {% endif %} + ( + {% for first_year in range(start_year, end_year + 1) -%} + {% set next_year = first_year + 1 -%} + conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} + {% if not loop.last -%} + {% endif %} + {% endfor %} + ) as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} + {% if not loop.last and not ns.continued -%}, {% endif %} + {% endfor %} {% if not loop.last and not ns.continued -%}, {% endif %} + {% endfor %} + {% if not loop.last -%}, {% endif %} + {% endfor %} from together diff --git a/airflow/include/sql/sparte/models/majic/schema.yml b/airflow/include/sql/sparte/models/majic/schema.yml index 12b8789bb..f244c906e 100644 --- a/airflow/include/sql/sparte/models/majic/schema.yml +++ b/airflow/include/sql/sparte/models/majic/schema.yml @@ -29,7 +29,26 @@ models: - relationships: to: ref('commune') field: code - + - name: consommation_departement + columns: + - name: code_departement + data_tests: + - not_null + - unique + - has_all_departements + - relationships: + to: ref('departement') + field: code + - name: consommation_epci + columns: + - name: code_epci + data_tests: + - not_null + - unique + - has_all_epcis + - relationships: + to: ref('epci') + field: code sources: - name: public tables: From 281a49b0f02e1b21b770d99c5e4c1052204f64f5 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Thu, 14 Nov 2024 17:35:47 +0100 Subject: [PATCH 02/14] feat(consommation): rewrite aggregation with cumulative_flux macro --- .../majic/merge_majic_by_admin_level.sql | 36 ++++++++----------- .../aggregated/consommation_departement.sql | 7 +--- .../majic/aggregated/consommation_epci.sql | 7 +--- .../majic/aggregated/consommation_region.sql | 7 +--- .../majic/aggregated/consommation_scot.sql | 7 +--- .../models/majic/consommation_cog_2024.sql | 33 +++++++---------- .../sql/sparte/models/majic/schema.yml | 24 +++++++++++-- 7 files changed, 53 insertions(+), 68 deletions(-) diff --git a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql index 50d042a34..ff02df648 100644 --- a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql +++ b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql @@ -2,28 +2,22 @@ group_by_column, code_name ) %} +{% if not code_name %} + {% set code_name = group_by_column %} +{% endif %} SELECT - {{ group_by_column }} as {{ code_name }}, - {% set last_available_year = 2022 %} - {% set first_available_year = 2009 %} - {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} - {% set ns = namespace(continued=false) %} - {% for type_conso_suffix in type_conso_suffixes %} - {% for start_year in range(2009, last_available_year + 1) %} - {% for end_year in range(2009, last_available_year + 1) -%} - {% if start_year > end_year -%} - {% set ns.continued = true %} - {% continue %} - {% else %} - {% set ns.continued = false %} - {% endif %} - sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) - as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} - {% if not loop.last and not ns.continued -%}, {% endif %} - {% endfor %} {% if not loop.last and not ns.continued -%}, {% endif %} - {% endfor %} - {% if not loop.last -%}, {% endif %} - {% endfor %} + {{ group_by_column}} as {{ code_name }}, + {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} + {% for type_conso_suffix in type_conso_suffixes %} + {% call(start_year, end_year) cumulative_flux( + first_available_year=2009, + last_available_year=2022 + ) %} + sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) + as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} + {% endcall %} + {% if not loop.last -%}, {% endif %} + {% endfor %} FROM {{ ref('consommation_cog_2024') }} as consommation LEFT JOIN diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql index 44b261acd..cbdfb269e 100644 --- a/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql @@ -1,8 +1,3 @@ {{ config(materialized='table') }} -{{ - merge_majic_by_admin_level( - 'departement', - 'code_departement' - ) -}} +{{ merge_majic_by_admin_level('departement') }} diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql index 882294f12..f4e62aacb 100644 --- a/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql @@ -1,8 +1,3 @@ {{ config(materialized='table') }} -{{ - merge_majic_by_admin_level( - 'epci', - 'code_epci' - ) -}} +{{ merge_majic_by_admin_level('epci') }} diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql index 24dea9597..49331c059 100644 --- a/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql @@ -1,8 +1,3 @@ {{ config(materialized='table') }} -{{ - merge_majic_by_admin_level( - 'region', - 'code_region' - ) -}} +{{ merge_majic_by_admin_level('region') }} diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql index f2d2808bf..5dfcab402 100644 --- a/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql @@ -1,8 +1,3 @@ {{ config(materialized='table') }} -{{ - merge_majic_by_admin_level( - 'id_scot', - 'code_scot' - ) -}} +{{ merge_majic_by_admin_level('id_scot', 'scot') }} diff --git a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql index 2f0524f38..f46fbfbc9 100644 --- a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql +++ b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql @@ -131,29 +131,20 @@ together as ( select commune_code, correction_status, - {% set last_available_year = 2022 %} - {% set first_available_year = 2009 %} {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} - {% set ns = namespace(continued=false) %} {% for type_conso_suffix in type_conso_suffixes %} - {% for start_year in range(2009, last_available_year + 1) %} - {% for end_year in range(2009, last_available_year + 1) -%} - {% if start_year > end_year -%} - {% set ns.continued = true %} - {% continue %} - {% else %} - {% set ns.continued = false %} - {% endif %} - ( - {% for first_year in range(start_year, end_year + 1) -%} - {% set next_year = first_year + 1 -%} - conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} - {% if not loop.last -%} + {% endif %} - {% endfor %} - ) as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} - {% if not loop.last and not ns.continued -%}, {% endif %} - {% endfor %} {% if not loop.last and not ns.continued -%}, {% endif %} - {% endfor %} + {% call(start_year, end_year) cumulative_flux( + first_available_year=2009, + last_available_year=2022 + ) %} + ( + {% for first_year in range(start_year, end_year + 1) -%} + {% set next_year = first_year + 1 -%} + conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} + {% if not loop.last -%} + {% endif %} + {% endfor %} + ) as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} + {% endcall %} {% if not loop.last -%}, {% endif %} {% endfor %} from diff --git a/airflow/include/sql/sparte/models/majic/schema.yml b/airflow/include/sql/sparte/models/majic/schema.yml index f244c906e..1492841c7 100644 --- a/airflow/include/sql/sparte/models/majic/schema.yml +++ b/airflow/include/sql/sparte/models/majic/schema.yml @@ -31,7 +31,7 @@ models: field: code - name: consommation_departement columns: - - name: code_departement + - name: departement data_tests: - not_null - unique @@ -41,7 +41,7 @@ models: field: code - name: consommation_epci columns: - - name: code_epci + - name: epci data_tests: - not_null - unique @@ -49,6 +49,26 @@ models: - relationships: to: ref('epci') field: code + - name: commation_region + columns: + - name: region + data_tests: + - not_null + - unique + - has_all_regions + - relationships: + to: ref('region') + field: code + - name: consommation_scot + columns: + - name: scot + data_tests: + - not_null + - unique + - has_all_scots + - relationships: + to: ref('scot') + field: id_scot sources: - name: public tables: From 1739a28a96e94ea11786f0233f21c44f96eed291 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Thu, 14 Nov 2024 21:04:33 +0100 Subject: [PATCH 03/14] feat(conso): add percent to aggregates --- .../macros/majic/merge_majic_by_admin_level.sql | 6 ++++-- .../sparte/models/majic/consommation_cog_2024.sql | 12 +++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql index ff02df648..4fe17576f 100644 --- a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql +++ b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql @@ -14,14 +14,16 @@ SELECT last_available_year=2022 ) %} sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) - as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} + as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}, + sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) * 100 / sum(commune.surface) + as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}_percent {% endcall %} {% if not loop.last -%}, {% endif %} {% endfor %} FROM {{ ref('consommation_cog_2024') }} as consommation LEFT JOIN - {{ ref('commune') }} + {{ ref('commune') }} as commune ON commune.code = consommation.commune_code LEFT JOIN {{ ref('scot_communes') }} as scot_communes diff --git a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql index f46fbfbc9..c9fea1e9f 100644 --- a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql +++ b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql @@ -143,9 +143,19 @@ select conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} {% if not loop.last -%} + {% endif %} {% endfor %} - ) as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }} + ) as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}, + ( + {% for first_year in range(start_year, end_year + 1) -%} + {% set next_year = first_year + 1 -%} + conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} + {% if not loop.last -%} + {% endif %} + {% endfor %} + ) * 100 / commune.surface as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}_percent {% endcall %} {% if not loop.last -%}, {% endif %} {% endfor %} from together +LEFT JOIN + {{ ref('commune') }} as commune + ON commune.code = together.commune_code From 60f56020b636a86e432f3f1cd2d16bc8641abf15 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Thu, 14 Nov 2024 21:22:35 +0100 Subject: [PATCH 04/14] feat(epci_ept): remove duplication --- .../sql/sparte/models/admin_express/epci.sql | 29 +------------- .../models/admin_express/epci_tout_type.sql | 38 +++++++++++++++++++ .../sql/sparte/models/admin_express/ept.sql | 29 +------------- 3 files changed, 40 insertions(+), 56 deletions(-) create mode 100644 airflow/include/sql/sparte/models/admin_express/epci_tout_type.sql diff --git a/airflow/include/sql/sparte/models/admin_express/epci.sql b/airflow/include/sql/sparte/models/admin_express/epci.sql index c46b09e2d..a7a363f17 100644 --- a/airflow/include/sql/sparte/models/admin_express/epci.sql +++ b/airflow/include/sql/sparte/models/admin_express/epci.sql @@ -8,31 +8,4 @@ {'columns': ['geom'], 'type': 'gist'} ]) }} -with together as ( - SELECT - *, - 32620 AS srid_source - FROM {{ ref('epci_guadeloupe') }} - UNION ALL - SELECT - *, - 32620 AS srid_source - FROM {{ ref('epci_martinique') }} - UNION ALL - SELECT - *, - 2972 AS srid_source - FROM {{ ref('epci_guyane') }} - UNION ALL - SELECT - *, - 2975 AS srid_source - FROM {{ ref('epci_reunion') }} - UNION ALL - SELECT - *, - 2154 AS srid_source - FROM {{ ref('epci_metropole') }} -) -SELECT * FROM together -WHERE is_ept = false +SELECT * FROM {{ ref('epci_tout_type') }} WHERE is_ept = false diff --git a/airflow/include/sql/sparte/models/admin_express/epci_tout_type.sql b/airflow/include/sql/sparte/models/admin_express/epci_tout_type.sql new file mode 100644 index 000000000..c46b09e2d --- /dev/null +++ b/airflow/include/sql/sparte/models/admin_express/epci_tout_type.sql @@ -0,0 +1,38 @@ +{{ + config( + materialized='table', + indexes=[ + {'columns': ['id'], 'type': 'btree'}, + {'columns': ['code'], 'type': 'btree'}, + {'columns': ['name'], 'type': 'btree'}, + {'columns': ['geom'], 'type': 'gist'} + ]) +}} +with together as ( + SELECT + *, + 32620 AS srid_source + FROM {{ ref('epci_guadeloupe') }} + UNION ALL + SELECT + *, + 32620 AS srid_source + FROM {{ ref('epci_martinique') }} + UNION ALL + SELECT + *, + 2972 AS srid_source + FROM {{ ref('epci_guyane') }} + UNION ALL + SELECT + *, + 2975 AS srid_source + FROM {{ ref('epci_reunion') }} + UNION ALL + SELECT + *, + 2154 AS srid_source + FROM {{ ref('epci_metropole') }} +) +SELECT * FROM together +WHERE is_ept = false diff --git a/airflow/include/sql/sparte/models/admin_express/ept.sql b/airflow/include/sql/sparte/models/admin_express/ept.sql index 2c8e37f93..b5e41c20a 100644 --- a/airflow/include/sql/sparte/models/admin_express/ept.sql +++ b/airflow/include/sql/sparte/models/admin_express/ept.sql @@ -8,31 +8,4 @@ {'columns': ['geom'], 'type': 'gist'} ]) }} -with together as ( - SELECT - *, - 32620 AS srid_source - FROM {{ ref('epci_guadeloupe') }} - UNION ALL - SELECT - *, - 32620 AS srid_source - FROM {{ ref('epci_martinique') }} - UNION ALL - SELECT - *, - 2972 AS srid_source - FROM {{ ref('epci_guyane') }} - UNION ALL - SELECT - *, - 2975 AS srid_source - FROM {{ ref('epci_reunion') }} - UNION ALL - SELECT - *, - 2154 AS srid_source - FROM {{ ref('epci_metropole') }} -) -SELECT * FROM together -WHERE is_ept = true +SELECT * FROM {{ ref('epci_tout_type') }} WHERE is_ept = true From 11876b46df316bae1facb19a59dc9394542264ed Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 15:35:19 +0100 Subject: [PATCH 05/14] feat(conso/pop): add all tables in dbt --- .../merge_flux_population_by_admin_level.sql | 25 +-- .../sql/sparte/macros/majic/divide_majic.sql | 196 ++++++++-------- .../include/sql/sparte/macros/majic/majic.sql | 210 +++++++++--------- .../sql/sparte/macros/majic/merge_majic.sql | 196 ++++++++-------- .../majic/merge_majic_by_admin_level.sql | 28 ++- .../sql/sparte/macros/utils/sum_percent.sql | 10 + .../macros/utils/sum_percent_median_avg.sql | 16 ++ .../sparte/models/for_app/for_app_cerema.sql | 202 ++++++++--------- .../models/for_app/for_app_landconso.sql | 77 +++++++ .../for_app/for_app_landconsostatistics.sql | 80 +++++++ .../sparte/models/for_app/for_app_landpop.sql | 48 ++++ .../for_app/for_app_landpopstatistics.sql | 56 +++++ .../flux/flux_population_commune.sql | 37 +++ .../flux/flux_population_commune.yml | 52 +++++ ...ts.sql => flux_population_departement.sql} | 0 .../flux/flux_population_departement.yml | 11 + ...ion_epcis.sql => flux_population_epci.sql} | 0 .../aggregated/flux/flux_population_epci.yml | 11 + ...regions.sql => flux_population_region.sql} | 0 .../flux/flux_population_region.yml | 12 + ...ion_scots.sql => flux_population_scot.sql} | 0 .../aggregated/flux/flux_population_scot.yml | 12 + ...s.sql => stock_population_departement.sql} | 0 .../stock/stock_population_departement.yml | 13 ++ ...on_epcis.sql => stock_population_epci.sql} | 0 .../stock/stock_population_epci.yml | 13 ++ ...egions.sql => stock_population_region.sql} | 0 .../stock/stock_population_region.yml | 13 ++ ...on_scots.sql => stock_population_scot.sql} | 0 .../stock/stock_population_scot.yml | 13 ++ .../calculated/pop_epci_stats_commune.sql | 16 ++ .../calculated/pop_epci_stats_commune.yml | 39 ++++ .../calculated/pop_national_stats_epci.sql | 11 + .../calculated/pop_national_stats_region.sql | 11 + .../calculated/pop_national_stats_scot.sql | 11 + .../pop_regional_stats_departement.sql | 16 ++ .../sparte/models/insee/cog_changes_2024.yml | 40 ++++ .../sparte/models/insee/flux_population.sql | 92 ++++---- .../sparte/models/insee/flux_population.yml | 11 + .../sql/sparte/models/insee/population.yml | 27 +++ .../models/insee/population_cog_2024.yml | 13 ++ .../sql/sparte/models/insee/schema.yml | 146 ------------ .../sql/sparte/models/insee/sources.yml | 6 + .../majic/aggregated/consommation_commune.sql | 37 +++ .../calculated/conso_epci_stats_commune.sql | 22 ++ .../calculated/conso_national_stats_epci.sql | 21 ++ .../conso_national_stats_region.sql | 20 ++ .../calculated/conso_national_stats_scot.sql | 20 ++ .../conso_regional_stats_departement.sql | 22 ++ .../models/majic/consommation_cog_2024.sql | 50 ++--- .../models/majic/consommation_en_lignes.sql | 22 ++ .../sql/sparte/models/sudocuh/scot.sql | 3 +- .../sparte/tests/generic/has_all_communes.sql | 2 +- .../tests/generic/has_all_departements.sql | 2 +- .../sparte/tests/generic/has_all_epcis.sql | 2 +- .../sparte/tests/generic/has_all_regions.sql | 2 +- .../sparte/tests/generic/has_all_scots.sql | 2 +- 57 files changed, 1344 insertions(+), 653 deletions(-) create mode 100644 airflow/include/sql/sparte/macros/utils/sum_percent.sql create mode 100644 airflow/include/sql/sparte/macros/utils/sum_percent_median_avg.sql create mode 100644 airflow/include/sql/sparte/models/for_app/for_app_landconso.sql create mode 100644 airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql create mode 100644 airflow/include/sql/sparte/models/for_app/for_app_landpop.sql create mode 100644 airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_departements.sql => flux_population_departement.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_epcis.sql => flux_population_epci.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_regions.sql => flux_population_region.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_scots.sql => flux_population_scot.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_departements.sql => stock_population_departement.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_epcis.sql => stock_population_epci.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_regions.sql => stock_population_region.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_scots.sql => stock_population_scot.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml create mode 100644 airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql create mode 100644 airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.yml create mode 100644 airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql create mode 100644 airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql create mode 100644 airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql create mode 100644 airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql create mode 100644 airflow/include/sql/sparte/models/insee/cog_changes_2024.yml create mode 100644 airflow/include/sql/sparte/models/insee/flux_population.yml create mode 100644 airflow/include/sql/sparte/models/insee/population.yml create mode 100644 airflow/include/sql/sparte/models/insee/population_cog_2024.yml delete mode 100644 airflow/include/sql/sparte/models/insee/schema.yml create mode 100644 airflow/include/sql/sparte/models/insee/sources.yml create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql create mode 100644 airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql create mode 100644 airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql create mode 100644 airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql create mode 100644 airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql create mode 100644 airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql create mode 100644 airflow/include/sql/sparte/models/majic/consommation_en_lignes.sql diff --git a/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql b/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql index 95001b3a6..519a277b5 100644 --- a/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql +++ b/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql @@ -6,21 +6,16 @@ {% set code_name = group_by_column %} {% endif %} SELECT - {{ group_by_column }} as {{ code_name }}, - {% call(start_year, end_year) cumulative_flux( - first_available_year=2009, - last_available_year=2020 - ) %} - sum(population_{{ start_year }}_{{ end_year + 1 }}) - as population_{{ start_year }}_{{ end_year + 1 }}, - sum(population_{{ start_year }}_{{ end_year + 1 }}) * 100 / sum(population_{{ start_year }}) - as population_{{ start_year }}_{{ end_year + 1 }}_percent - {% endcall %} + {{ group_by_column }} as {{ code_name }}, + from_year, + to_year, + {{ sum_percent_median_avg('evolution', 'start_population') }}, + sum(start_population) as start_population FROM - {{ ref('flux_population') }} as flux_population + {{ ref('flux_population_commune') }} as population LEFT JOIN - {{ ref('commune') }} - ON commune.code = flux_population.code_commune + {{ ref('commune') }} as commune + ON commune.code = population.code_commune LEFT JOIN {{ ref('scot_communes') }} as scot_communes ON commune.code = scot_communes.commune_code @@ -29,5 +24,7 @@ LEFT JOIN WHERE {{ group_by_column }} IS NOT NULL GROUP BY - {{ group_by_column }} + {{ group_by_column }}, + from_year, + to_year {% endmacro %} diff --git a/airflow/include/sql/sparte/macros/majic/divide_majic.sql b/airflow/include/sql/sparte/macros/majic/divide_majic.sql index 5076b90c0..9ac61e9f1 100644 --- a/airflow/include/sql/sparte/macros/majic/divide_majic.sql +++ b/airflow/include/sql/sparte/macros/majic/divide_majic.sql @@ -1,104 +1,104 @@ {% macro divide_majic(initial_commune_code, final_commune_code, percent) %} SELECT '{{ final_commune_code }}' as commune_code, - conso_2009_2010 * {{ percent }} / 100 as conso_2009_2010, - conso_2009_2010_activite * {{ percent }} / 100 as conso_2009_2010_activite, - conso_2009_2010_habitat * {{ percent }} / 100 as conso_2009_2010_habitat, - conso_2009_2010_mixte * {{ percent }} / 100 as conso_2009_2010_mixte, - conso_2009_2010_route * {{ percent }} / 100 as conso_2009_2010_route, - conso_2009_2010_ferroviaire * {{ percent }} / 100 as conso_2009_2010_ferroviaire, - conso_2009_2010_inconnu * {{ percent }} / 100 as conso_2009_2010_inconnu, - conso_2010_2011 * {{ percent }} / 100 as conso_2010_2011, - conso_2010_2011_activite * {{ percent }} / 100 as conso_2010_2011_activite, - conso_2010_2011_habitat * {{ percent }} / 100 as conso_2010_2011_habitat, - conso_2010_2011_mixte * {{ percent }} / 100 as conso_2010_2011_mixte, - conso_2010_2011_route * {{ percent }} / 100 as conso_2010_2011_route, - conso_2010_2011_ferroviaire * {{ percent }} / 100 as conso_2010_2011_ferroviaire, - conso_2010_2011_inconnu * {{ percent }} / 100 as conso_2010_2011_inconnu, - conso_2011_2012 * {{ percent }} / 100 as conso_2011_2012, - conso_2011_2012_activite * {{ percent }} / 100 as conso_2011_2012_activite, - conso_2011_2012_habitat * {{ percent }} / 100 as conso_2011_2012_habitat, - conso_2011_2012_mixte * {{ percent }} / 100 as conso_2011_2012_mixte, - conso_2011_2012_route * {{ percent }} / 100 as conso_2011_2012_route, - conso_2011_2012_ferroviaire * {{ percent }} / 100 as conso_2011_2012_ferroviaire, - conso_2011_2012_inconnu * {{ percent }} / 100 as conso_2011_2012_inconnu, - conso_2012_2013 * {{ percent }} / 100 as conso_2012_2013, - conso_2012_2013_activite * {{ percent }} / 100 as conso_2012_2013_activite, - conso_2012_2013_habitat * {{ percent }} / 100 as conso_2012_2013_habitat, - conso_2012_2013_mixte * {{ percent }} / 100 as conso_2012_2013_mixte, - conso_2012_2013_route * {{ percent }} / 100 as conso_2012_2013_route, - conso_2012_2013_ferroviaire * {{ percent }} / 100 as conso_2012_2013_ferroviaire, - conso_2012_2013_inconnu * {{ percent }} / 100 as conso_2012_2013_inconnu, - conso_2013_2014 * {{ percent }} / 100 as conso_2013_2014, - conso_2013_2014_activite * {{ percent }} / 100 as conso_2013_2014_activite, - conso_2013_2014_habitat * {{ percent }} / 100 as conso_2013_2014_habitat, - conso_2013_2014_mixte * {{ percent }} / 100 as conso_2013_2014_mixte, - conso_2013_2014_route * {{ percent }} / 100 as conso_2013_2014_route, - conso_2013_2014_ferroviaire * {{ percent }} / 100 as conso_2013_2014_ferroviaire, - conso_2013_2014_inconnu * {{ percent }} / 100 as conso_2013_2014_inconnu, - conso_2014_2015 * {{ percent }} / 100 as conso_2014_2015, - conso_2014_2015_activite * {{ percent }} / 100 as conso_2014_2015_activite, - conso_2014_2015_habitat * {{ percent }} / 100 as conso_2014_2015_habitat, - conso_2014_2015_mixte * {{ percent }} / 100 as conso_2014_2015_mixte, - conso_2014_2015_route * {{ percent }} / 100 as conso_2014_2015_route, - conso_2014_2015_ferroviaire * {{ percent }} / 100 as conso_2014_2015_ferroviaire, - conso_2014_2015_inconnu * {{ percent }} / 100 as conso_2014_2015_inconnu, - conso_2015_2016 * {{ percent }} / 100 as conso_2015_2016, - conso_2015_2016_activite * {{ percent }} / 100 as conso_2015_2016_activite, - conso_2015_2016_habitat * {{ percent }} / 100 as conso_2015_2016_habitat, - conso_2015_2016_mixte * {{ percent }} / 100 as conso_2015_2016_mixte, - conso_2015_2016_route * {{ percent }} / 100 as conso_2015_2016_route, - conso_2015_2016_ferroviaire * {{ percent }} / 100 as conso_2015_2016_ferroviaire, - conso_2015_2016_inconnu * {{ percent }} / 100 as conso_2015_2016_inconnu, - conso_2016_2017 * {{ percent }} / 100 as conso_2016_2017, - conso_2016_2017_activite * {{ percent }} / 100 as conso_2016_2017_activite, - conso_2016_2017_habitat * {{ percent }} / 100 as conso_2016_2017_habitat, - conso_2016_2017_mixte * {{ percent }} / 100 as conso_2016_2017_mixte, - conso_2016_2017_route * {{ percent }} / 100 as conso_2016_2017_route, - conso_2016_2017_ferroviaire * {{ percent }} / 100 as conso_2016_2017_ferroviaire, - conso_2016_2017_inconnu * {{ percent }} / 100 as conso_2016_2017_inconnu, - conso_2017_2018 * {{ percent }} / 100 as conso_2017_2018, - conso_2017_2018_activite * {{ percent }} / 100 as conso_2017_2018_activite, - conso_2017_2018_habitat * {{ percent }} / 100 as conso_2017_2018_habitat, - conso_2017_2018_mixte * {{ percent }} / 100 as conso_2017_2018_mixte, - conso_2017_2018_route * {{ percent }} / 100 as conso_2017_2018_route, - conso_2017_2018_ferroviaire * {{ percent }} / 100 as conso_2017_2018_ferroviaire, - conso_2017_2018_inconnu * {{ percent }} / 100 as conso_2017_2018_inconnu, - conso_2018_2019 * {{ percent }} / 100 as conso_2018_2019, - conso_2018_2019_activite * {{ percent }} / 100 as conso_2018_2019_activite, - conso_2018_2019_habitat * {{ percent }} / 100 as conso_2018_2019_habitat, - conso_2018_2019_mixte * {{ percent }} / 100 as conso_2018_2019_mixte, - conso_2018_2019_route * {{ percent }} / 100 as conso_2018_2019_route, - conso_2018_2019_ferroviaire * {{ percent }} / 100 as conso_2018_2019_ferroviaire, - conso_2018_2019_inconnu * {{ percent }} / 100 as conso_2018_2019_inconnu, - conso_2019_2020 * {{ percent }} / 100 as conso_2019_2020, - conso_2019_2020_activite * {{ percent }} / 100 as conso_2019_2020_activite, - conso_2019_2020_habitat * {{ percent }} / 100 as conso_2019_2020_habitat, - conso_2019_2020_mixte * {{ percent }} / 100 as conso_2019_2020_mixte, - conso_2019_2020_route * {{ percent }} / 100 as conso_2019_2020_route, - conso_2019_2020_ferroviaire * {{ percent }} / 100 as conso_2019_2020_ferroviaire, - conso_2019_2020_inconnu * {{ percent }} / 100 as conso_2019_2020_inconnu, - conso_2020_2021 * {{ percent }} / 100 as conso_2020_2021, - conso_2020_2021_activite * {{ percent }} / 100 as conso_2020_2021_activite, - conso_2020_2021_habitat * {{ percent }} / 100 as conso_2020_2021_habitat, - conso_2020_2021_mixte * {{ percent }} / 100 as conso_2020_2021_mixte, - conso_2020_2021_route * {{ percent }} / 100 as conso_2020_2021_route, - conso_2020_2021_ferroviaire * {{ percent }} / 100 as conso_2020_2021_ferroviaire, - conso_2020_2021_inconnu * {{ percent }} / 100 as conso_2020_2021_inconnu, - conso_2021_2022 * {{ percent }} / 100 as conso_2021_2022, - conso_2021_2022_activite * {{ percent }} / 100 as conso_2021_2022_activite, - conso_2021_2022_habitat * {{ percent }} / 100 as conso_2021_2022_habitat, - conso_2021_2022_mixte * {{ percent }} / 100 as conso_2021_2022_mixte, - conso_2021_2022_route * {{ percent }} / 100 as conso_2021_2022_route, - conso_2021_2022_ferroviaire * {{ percent }} / 100 as conso_2021_2022_ferroviaire, - conso_2021_2022_inconnu * {{ percent }} / 100 as conso_2021_2022_inconnu, - conso_2022_2023 * {{ percent }} / 100 as conso_2022_2023, - conso_2022_2023_activite * {{ percent }} / 100 as conso_2022_2023_activite, - conso_2022_2023_habitat * {{ percent }} / 100 as conso_2022_2023_habitat, - conso_2022_2023_mixte * {{ percent }} / 100 as conso_2022_2023_mixte, - conso_2022_2023_route * {{ percent }} / 100 as conso_2022_2023_route, - conso_2022_2023_ferroviaire * {{ percent }} / 100 as conso_2022_2023_ferroviaire, - conso_2022_2023_inconnu * {{ percent }} / 100 as conso_2022_2023_inconnu, + conso_2009 * {{ percent }} / 100 as conso_2009, + conso_2009_activite * {{ percent }} / 100 as conso_2009_activite, + conso_2009_habitat * {{ percent }} / 100 as conso_2009_habitat, + conso_2009_mixte * {{ percent }} / 100 as conso_2009_mixte, + conso_2009_route * {{ percent }} / 100 as conso_2009_route, + conso_2009_ferroviaire * {{ percent }} / 100 as conso_2009_ferroviaire, + conso_2009_inconnu * {{ percent }} / 100 as conso_2009_inconnu, + conso_2010 * {{ percent }} / 100 as conso_2010, + conso_2010_activite * {{ percent }} / 100 as conso_2010_activite, + conso_2010_habitat * {{ percent }} / 100 as conso_2010_habitat, + conso_2010_mixte * {{ percent }} / 100 as conso_2010_mixte, + conso_2010_route * {{ percent }} / 100 as conso_2010_route, + conso_2010_ferroviaire * {{ percent }} / 100 as conso_2010_ferroviaire, + conso_2010_inconnu * {{ percent }} / 100 as conso_2010_inconnu, + conso_2011 * {{ percent }} / 100 as conso_2011, + conso_2011_activite * {{ percent }} / 100 as conso_2011_activite, + conso_2011_habitat * {{ percent }} / 100 as conso_2011_habitat, + conso_2011_mixte * {{ percent }} / 100 as conso_2011_mixte, + conso_2011_route * {{ percent }} / 100 as conso_2011_route, + conso_2011_ferroviaire * {{ percent }} / 100 as conso_2011_ferroviaire, + conso_2011_inconnu * {{ percent }} / 100 as conso_2011_inconnu, + conso_2012 * {{ percent }} / 100 as conso_2012, + conso_2012_activite * {{ percent }} / 100 as conso_2012_activite, + conso_2012_habitat * {{ percent }} / 100 as conso_2012_habitat, + conso_2012_mixte * {{ percent }} / 100 as conso_2012_mixte, + conso_2012_route * {{ percent }} / 100 as conso_2012_route, + conso_2012_ferroviaire * {{ percent }} / 100 as conso_2012_ferroviaire, + conso_2012_inconnu * {{ percent }} / 100 as conso_2012_inconnu, + conso_2013 * {{ percent }} / 100 as conso_2013, + conso_2013_activite * {{ percent }} / 100 as conso_2013_activite, + conso_2013_habitat * {{ percent }} / 100 as conso_2013_habitat, + conso_2013_mixte * {{ percent }} / 100 as conso_2013_mixte, + conso_2013_route * {{ percent }} / 100 as conso_2013_route, + conso_2013_ferroviaire * {{ percent }} / 100 as conso_2013_ferroviaire, + conso_2013_inconnu * {{ percent }} / 100 as conso_2013_inconnu, + conso_2014 * {{ percent }} / 100 as conso_2014, + conso_2014_activite * {{ percent }} / 100 as conso_2014_activite, + conso_2014_habitat * {{ percent }} / 100 as conso_2014_habitat, + conso_2014_mixte * {{ percent }} / 100 as conso_2014_mixte, + conso_2014_route * {{ percent }} / 100 as conso_2014_route, + conso_2014_ferroviaire * {{ percent }} / 100 as conso_2014_ferroviaire, + conso_2014_inconnu * {{ percent }} / 100 as conso_2014_inconnu, + conso_2015 * {{ percent }} / 100 as conso_2015, + conso_2015_activite * {{ percent }} / 100 as conso_2015_activite, + conso_2015_habitat * {{ percent }} / 100 as conso_2015_habitat, + conso_2015_mixte * {{ percent }} / 100 as conso_2015_mixte, + conso_2015_route * {{ percent }} / 100 as conso_2015_route, + conso_2015_ferroviaire * {{ percent }} / 100 as conso_2015_ferroviaire, + conso_2015_inconnu * {{ percent }} / 100 as conso_2015_inconnu, + conso_2016 * {{ percent }} / 100 as conso_2016, + conso_2016_activite * {{ percent }} / 100 as conso_2016_activite, + conso_2016_habitat * {{ percent }} / 100 as conso_2016_habitat, + conso_2016_mixte * {{ percent }} / 100 as conso_2016_mixte, + conso_2016_route * {{ percent }} / 100 as conso_2016_route, + conso_2016_ferroviaire * {{ percent }} / 100 as conso_2016_ferroviaire, + conso_2016_inconnu * {{ percent }} / 100 as conso_2016_inconnu, + conso_2017 * {{ percent }} / 100 as conso_2017, + conso_2017_activite * {{ percent }} / 100 as conso_2017_activite, + conso_2017_habitat * {{ percent }} / 100 as conso_2017_habitat, + conso_2017_mixte * {{ percent }} / 100 as conso_2017_mixte, + conso_2017_route * {{ percent }} / 100 as conso_2017_route, + conso_2017_ferroviaire * {{ percent }} / 100 as conso_2017_ferroviaire, + conso_2017_inconnu * {{ percent }} / 100 as conso_2017_inconnu, + conso_2018 * {{ percent }} / 100 as conso_2018, + conso_2018_activite * {{ percent }} / 100 as conso_2018_activite, + conso_2018_habitat * {{ percent }} / 100 as conso_2018_habitat, + conso_2018_mixte * {{ percent }} / 100 as conso_2018_mixte, + conso_2018_route * {{ percent }} / 100 as conso_2018_route, + conso_2018_ferroviaire * {{ percent }} / 100 as conso_2018_ferroviaire, + conso_2018_inconnu * {{ percent }} / 100 as conso_2018_inconnu, + conso_2019 * {{ percent }} / 100 as conso_2019, + conso_2019_activite * {{ percent }} / 100 as conso_2019_activite, + conso_2019_habitat * {{ percent }} / 100 as conso_2019_habitat, + conso_2019_mixte * {{ percent }} / 100 as conso_2019_mixte, + conso_2019_route * {{ percent }} / 100 as conso_2019_route, + conso_2019_ferroviaire * {{ percent }} / 100 as conso_2019_ferroviaire, + conso_2019_inconnu * {{ percent }} / 100 as conso_2019_inconnu, + conso_2020 * {{ percent }} / 100 as conso_2020, + conso_2020_activite * {{ percent }} / 100 as conso_2020_activite, + conso_2020_habitat * {{ percent }} / 100 as conso_2020_habitat, + conso_2020_mixte * {{ percent }} / 100 as conso_2020_mixte, + conso_2020_route * {{ percent }} / 100 as conso_2020_route, + conso_2020_ferroviaire * {{ percent }} / 100 as conso_2020_ferroviaire, + conso_2020_inconnu * {{ percent }} / 100 as conso_2020_inconnu, + conso_2021 * {{ percent }} / 100 as conso_2021, + conso_2021_activite * {{ percent }} / 100 as conso_2021_activite, + conso_2021_habitat * {{ percent }} / 100 as conso_2021_habitat, + conso_2021_mixte * {{ percent }} / 100 as conso_2021_mixte, + conso_2021_route * {{ percent }} / 100 as conso_2021_route, + conso_2021_ferroviaire * {{ percent }} / 100 as conso_2021_ferroviaire, + conso_2021_inconnu * {{ percent }} / 100 as conso_2021_inconnu, + conso_2022 * {{ percent }} / 100 as conso_2022, + conso_2022_activite * {{ percent }} / 100 as conso_2022_activite, + conso_2022_habitat * {{ percent }} / 100 as conso_2022_habitat, + conso_2022_mixte * {{ percent }} / 100 as conso_2022_mixte, + conso_2022_route * {{ percent }} / 100 as conso_2022_route, + conso_2022_ferroviaire * {{ percent }} / 100 as conso_2022_ferroviaire, + conso_2022_inconnu * {{ percent }} / 100 as conso_2022_inconnu, conso_2009_2023 * {{ percent }} / 100 as conso_2009_2023, conso_2009_2023_activite * {{ percent }} / 100 as conso_2009_2023_activite, conso_2009_2023_habitat * {{ percent }} / 100 as conso_2009_2023_habitat, diff --git a/airflow/include/sql/sparte/macros/majic/majic.sql b/airflow/include/sql/sparte/macros/majic/majic.sql index 5e8ce4638..89b486f27 100644 --- a/airflow/include/sql/sparte/macros/majic/majic.sql +++ b/airflow/include/sql/sparte/macros/majic/majic.sql @@ -4,111 +4,111 @@ SELECT idcom as commune_code, - naf09art10 as conso_2009_2010, - art09act10 as conso_2009_2010_activite, - art09hab10 as conso_2009_2010_habitat, - art09mix10 as conso_2009_2010_mixte, - art09rou10 as conso_2009_2010_route, - art09fer10 as conso_2009_2010_ferroviaire, - art09inc10 as conso_2009_2010_inconnu, - naf10art11 as conso_2010_2011, - art10act11 as conso_2010_2011_activite, - art10hab11 as conso_2010_2011_habitat, - art10mix11 as conso_2010_2011_mixte, - art10rou11 as conso_2010_2011_route, - art10fer11 as conso_2010_2011_ferroviaire, - art10inc11 as conso_2010_2011_inconnu, - naf11art12 as conso_2011_2012, - art11act12 as conso_2011_2012_activite, - art11hab12 as conso_2011_2012_habitat, - art11mix12 as conso_2011_2012_mixte, - art11rou12 as conso_2011_2012_route, - art11fer12 as conso_2011_2012_ferroviaire, - art11inc12 as conso_2011_2012_inconnu, - naf12art13 as conso_2012_2013, - art12act13 as conso_2012_2013_activite, - art12hab13 as conso_2012_2013_habitat, - art12mix13 as conso_2012_2013_mixte, - art12rou13 as conso_2012_2013_route, - art12fer13 as conso_2012_2013_ferroviaire, - art12inc13 as conso_2012_2013_inconnu, - naf13art14 as conso_2013_2014, - art13act14 as conso_2013_2014_activite, - art13hab14 as conso_2013_2014_habitat, - art13mix14 as conso_2013_2014_mixte, - art13rou14 as conso_2013_2014_route, - art13fer14 as conso_2013_2014_ferroviaire, - art13inc14 as conso_2013_2014_inconnu, - naf14art15 as conso_2014_2015, - art14act15 as conso_2014_2015_activite, - art14hab15 as conso_2014_2015_habitat, - art14mix15 as conso_2014_2015_mixte, - art14rou15 as conso_2014_2015_route, - art14fer15 as conso_2014_2015_ferroviaire, - art14inc15 as conso_2014_2015_inconnu, - naf15art16 as conso_2015_2016, - art15act16 as conso_2015_2016_activite, - art15hab16 as conso_2015_2016_habitat, - art15mix16 as conso_2015_2016_mixte, - art15rou16 as conso_2015_2016_route, - art15fer16 as conso_2015_2016_ferroviaire, - art15inc16 as conso_2015_2016_inconnu, - naf16art17 as conso_2016_2017, - art16act17 as conso_2016_2017_activite, - art16hab17 as conso_2016_2017_habitat, - art16mix17 as conso_2016_2017_mixte, - art16rou17 as conso_2016_2017_route, - art16fer17 as conso_2016_2017_ferroviaire, - art16inc17 as conso_2016_2017_inconnu, - naf17art18 as conso_2017_2018, - art17act18 as conso_2017_2018_activite, - art17hab18 as conso_2017_2018_habitat, - art17mix18 as conso_2017_2018_mixte, - art17rou18 as conso_2017_2018_route, - art17fer18 as conso_2017_2018_ferroviaire, - art17inc18 as conso_2017_2018_inconnu, - naf18art19 as conso_2018_2019, - art18act19 as conso_2018_2019_activite, - art18hab19 as conso_2018_2019_habitat, - art18mix19 as conso_2018_2019_mixte, - art18rou19 as conso_2018_2019_route, - art18fer19 as conso_2018_2019_ferroviaire, - art18inc19 as conso_2018_2019_inconnu, - naf19art20 as conso_2019_2020, - art19act20 as conso_2019_2020_activite, - art19hab20 as conso_2019_2020_habitat, - art19mix20 as conso_2019_2020_mixte, - art19rou20 as conso_2019_2020_route, - art19fer20 as conso_2019_2020_ferroviaire, - art19inc20 as conso_2019_2020_inconnu, - naf20art21 as conso_2020_2021, - art20act21 as conso_2020_2021_activite, - art20hab21 as conso_2020_2021_habitat, - art20mix21 as conso_2020_2021_mixte, - art20rou21 as conso_2020_2021_route, - art20fer21 as conso_2020_2021_ferroviaire, - art20inc21 as conso_2020_2021_inconnu, - naf21art22 as conso_2021_2022, - art21act22 as conso_2021_2022_activite, - art21hab22 as conso_2021_2022_habitat, - art21mix22 as conso_2021_2022_mixte, - art21rou22 as conso_2021_2022_route, - art21fer22 as conso_2021_2022_ferroviaire, - art21inc22 as conso_2021_2022_inconnu, - naf22art23 as conso_2022_2023, - art22act23 as conso_2022_2023_activite, - art22hab23 as conso_2022_2023_habitat, - art22mix23 as conso_2022_2023_mixte, - art22rou23 as conso_2022_2023_route, - art22fer23 as conso_2022_2023_ferroviaire, - art22inc23 as conso_2022_2023_inconnu, - naf09art23 as conso_2009_2023, - art09act23 as conso_2009_2023_activite, - art09hab23 as conso_2009_2023_habitat, - art09mix23 as conso_2009_2023_mixte, - art09rou23 as conso_2009_2023_route, - art09fer23 as conso_2009_2023_ferroviaire, - art09inc23 as conso_2009_2023_inconnu + naf09art10::int as conso_2009, + art09act10::int as conso_2009_activite, + art09hab10::int as conso_2009_habitat, + art09mix10::int as conso_2009_mixte, + art09rou10::int as conso_2009_route, + art09fer10::int as conso_2009_ferroviaire, + art09inc10::int as conso_2009_inconnu, + naf10art11::int as conso_2010, + art10act11::int as conso_2010_activite, + art10hab11::int as conso_2010_habitat, + art10mix11::int as conso_2010_mixte, + art10rou11::int as conso_2010_route, + art10fer11::int as conso_2010_ferroviaire, + art10inc11::int as conso_2010_inconnu, + naf11art12::int as conso_2011, + art11act12::int as conso_2011_activite, + art11hab12::int as conso_2011_habitat, + art11mix12::int as conso_2011_mixte, + art11rou12::int as conso_2011_route, + art11fer12::int as conso_2011_ferroviaire, + art11inc12::int as conso_2011_inconnu, + naf12art13::int as conso_2012, + art12act13::int as conso_2012_activite, + art12hab13::int as conso_2012_habitat, + art12mix13::int as conso_2012_mixte, + art12rou13::int as conso_2012_route, + art12fer13::int as conso_2012_ferroviaire, + art12inc13::int as conso_2012_inconnu, + naf13art14::int as conso_2013, + art13act14::int as conso_2013_activite, + art13hab14::int as conso_2013_habitat, + art13mix14::int as conso_2013_mixte, + art13rou14::int as conso_2013_route, + art13fer14::int as conso_2013_ferroviaire, + art13inc14::int as conso_2013_inconnu, + naf14art15::int as conso_2014, + art14act15::int as conso_2014_activite, + art14hab15::int as conso_2014_habitat, + art14mix15::int as conso_2014_mixte, + art14rou15::int as conso_2014_route, + art14fer15::int as conso_2014_ferroviaire, + art14inc15::int as conso_2014_inconnu, + naf15art16::int as conso_2015, + art15act16::int as conso_2015_activite, + art15hab16::int as conso_2015_habitat, + art15mix16::int as conso_2015_mixte, + art15rou16::int as conso_2015_route, + art15fer16::int as conso_2015_ferroviaire, + art15inc16::int as conso_2015_inconnu, + naf16art17::int as conso_2016, + art16act17::int as conso_2016_activite, + art16hab17::int as conso_2016_habitat, + art16mix17::int as conso_2016_mixte, + art16rou17::int as conso_2016_route, + art16fer17::int as conso_2016_ferroviaire, + art16inc17::int as conso_2016_inconnu, + naf17art18::int as conso_2017, + art17act18::int as conso_2017_activite, + art17hab18::int as conso_2017_habitat, + art17mix18::int as conso_2017_mixte, + art17rou18::int as conso_2017_route, + art17fer18::int as conso_2017_ferroviaire, + art17inc18::int as conso_2017_inconnu, + naf18art19::int as conso_2018, + art18act19::int as conso_2018_activite, + art18hab19::int as conso_2018_habitat, + art18mix19::int as conso_2018_mixte, + art18rou19::int as conso_2018_route, + art18fer19::int as conso_2018_ferroviaire, + art18inc19::int as conso_2018_inconnu, + naf19art20::int as conso_2019, + art19act20::int as conso_2019_activite, + art19hab20::int as conso_2019_habitat, + art19mix20::int as conso_2019_mixte, + art19rou20::int as conso_2019_route, + art19fer20::int as conso_2019_ferroviaire, + art19inc20::int as conso_2019_inconnu, + naf20art21::int as conso_2020, + art20act21::int as conso_2020_activite, + art20hab21::int as conso_2020_habitat, + art20mix21::int as conso_2020_mixte, + art20rou21::int as conso_2020_route, + art20fer21::int as conso_2020_ferroviaire, + art20inc21::int as conso_2020_inconnu, + naf21art22::int as conso_2021, + art21act22::int as conso_2021_activite, + art21hab22::int as conso_2021_habitat, + art21mix22::int as conso_2021_mixte, + art21rou22::int as conso_2021_route, + art21fer22::int as conso_2021_ferroviaire, + art21inc22::int as conso_2021_inconnu, + naf22art23::int as conso_2022, + art22act23::int as conso_2022_activite, + art22hab23::int as conso_2022_habitat, + art22mix23::int as conso_2022_mixte, + art22rou23::int as conso_2022_route, + art22fer23::int as conso_2022_ferroviaire, + art22inc23::int as conso_2022_inconnu, + naf09art23::int as conso_2009_2023, + art09act23::int as conso_2009_2023_activite, + art09hab23::int as conso_2009_2023_habitat, + art09mix23::int as conso_2009_2023_mixte, + art09rou23::int as conso_2009_2023_route, + art09fer23::int as conso_2009_2023_ferroviaire, + art09inc23::int as conso_2009_2023_inconnu FROM {{ source('public', source_table_name) }} {% endmacro %} diff --git a/airflow/include/sql/sparte/macros/majic/merge_majic.sql b/airflow/include/sql/sparte/macros/majic/merge_majic.sql index 51b522428..d3300f487 100644 --- a/airflow/include/sql/sparte/macros/majic/merge_majic.sql +++ b/airflow/include/sql/sparte/macros/majic/merge_majic.sql @@ -1,104 +1,104 @@ {% macro merge_majic(final_commune_code, communes_code_to_merge) %} SELECT min('{{ final_commune_code }}') as commune_code, - sum(conso_2009_2010) as conso_2009_2010, - sum(conso_2009_2010_activite) as conso_2009_2010_activite, - sum(conso_2009_2010_habitat) as conso_2009_2010_habitat, - sum(conso_2009_2010_mixte) as conso_2009_2010_mixte, - sum(conso_2009_2010_route) as conso_2009_2010_route, - sum(conso_2009_2010_ferroviaire) as conso_2009_2010_ferroviaire, - sum(conso_2009_2010_inconnu) as conso_2009_2010_inconnu, - sum(conso_2010_2011) as conso_2010_2011, - sum(conso_2010_2011_activite) as conso_2010_2011_activite, - sum(conso_2010_2011_habitat) as conso_2010_2011_habitat, - sum(conso_2010_2011_mixte) as conso_2010_2011_mixte, - sum(conso_2010_2011_route) as conso_2010_2011_route, - sum(conso_2010_2011_ferroviaire) as conso_2010_2011_ferroviaire, - sum(conso_2010_2011_inconnu) as conso_2010_2011_inconnu, - sum(conso_2011_2012) as conso_2011_2012, - sum(conso_2011_2012_activite) as conso_2011_2012_activite, - sum(conso_2011_2012_habitat) as conso_2011_2012_habitat, - sum(conso_2011_2012_mixte) as conso_2011_2012_mixte, - sum(conso_2011_2012_route) as conso_2011_2012_route, - sum(conso_2011_2012_ferroviaire) as conso_2011_2012_ferroviaire, - sum(conso_2011_2012_inconnu) as conso_2011_2012_inconnu, - sum(conso_2012_2013) as conso_2012_2013, - sum(conso_2012_2013_activite) as conso_2012_2013_activite, - sum(conso_2012_2013_habitat) as conso_2012_2013_habitat, - sum(conso_2012_2013_mixte) as conso_2012_2013_mixte, - sum(conso_2012_2013_route) as conso_2012_2013_route, - sum(conso_2012_2013_ferroviaire) as conso_2012_2013_ferroviaire, - sum(conso_2012_2013_inconnu) as conso_2012_2013_inconnu, - sum(conso_2013_2014) as conso_2013_2014, - sum(conso_2013_2014_activite) as conso_2013_2014_activite, - sum(conso_2013_2014_habitat) as conso_2013_2014_habitat, - sum(conso_2013_2014_mixte) as conso_2013_2014_mixte, - sum(conso_2013_2014_route) as conso_2013_2014_route, - sum(conso_2013_2014_ferroviaire) as conso_2013_2014_ferroviaire, - sum(conso_2013_2014_inconnu) as conso_2013_2014_inconnu, - sum(conso_2014_2015) as conso_2014_2015, - sum(conso_2014_2015_activite) as conso_2014_2015_activite, - sum(conso_2014_2015_habitat) as conso_2014_2015_habitat, - sum(conso_2014_2015_mixte) as conso_2014_2015_mixte, - sum(conso_2014_2015_route) as conso_2014_2015_route, - sum(conso_2014_2015_ferroviaire) as conso_2014_2015_ferroviaire, - sum(conso_2014_2015_inconnu) as conso_2014_2015_inconnu, - sum(conso_2015_2016) as conso_2015_2016, - sum(conso_2015_2016_activite) as conso_2015_2016_activite, - sum(conso_2015_2016_habitat) as conso_2015_2016_habitat, - sum(conso_2015_2016_mixte) as conso_2015_2016_mixte, - sum(conso_2015_2016_route) as conso_2015_2016_route, - sum(conso_2015_2016_ferroviaire) as conso_2015_2016_ferroviaire, - sum(conso_2015_2016_inconnu) as conso_2015_2016_inconnu, - sum(conso_2016_2017) as conso_2016_2017, - sum(conso_2016_2017_activite) as conso_2016_2017_activite, - sum(conso_2016_2017_habitat) as conso_2016_2017_habitat, - sum(conso_2016_2017_mixte) as conso_2016_2017_mixte, - sum(conso_2016_2017_route) as conso_2016_2017_route, - sum(conso_2016_2017_ferroviaire) as conso_2016_2017_ferroviaire, - sum(conso_2016_2017_inconnu) as conso_2016_2017_inconnu, - sum(conso_2017_2018) as conso_2017_2018, - sum(conso_2017_2018_activite) as conso_2017_2018_activite, - sum(conso_2017_2018_habitat) as conso_2017_2018_habitat, - sum(conso_2017_2018_mixte) as conso_2017_2018_mixte, - sum(conso_2017_2018_route) as conso_2017_2018_route, - sum(conso_2017_2018_ferroviaire) as conso_2017_2018_ferroviaire, - sum(conso_2017_2018_inconnu) as conso_2017_2018_inconnu, - sum(conso_2018_2019) as conso_2018_2019, - sum(conso_2018_2019_activite) as conso_2018_2019_activite, - sum(conso_2018_2019_habitat) as conso_2018_2019_habitat, - sum(conso_2018_2019_mixte) as conso_2018_2019_mixte, - sum(conso_2018_2019_route) as conso_2018_2019_route, - sum(conso_2018_2019_ferroviaire) as conso_2018_2019_ferroviaire, - sum(conso_2018_2019_inconnu) as conso_2018_2019_inconnu, - sum(conso_2019_2020) as conso_2019_2020, - sum(conso_2019_2020_activite) as conso_2019_2020_activite, - sum(conso_2019_2020_habitat) as conso_2019_2020_habitat, - sum(conso_2019_2020_mixte) as conso_2019_2020_mixte, - sum(conso_2019_2020_route) as conso_2019_2020_route, - sum(conso_2019_2020_ferroviaire) as conso_2019_2020_ferroviaire, - sum(conso_2019_2020_inconnu) as conso_2019_2020_inconnu, - sum(conso_2020_2021) as conso_2020_2021, - sum(conso_2020_2021_activite) as conso_2020_2021_activite, - sum(conso_2020_2021_habitat) as conso_2020_2021_habitat, - sum(conso_2020_2021_mixte) as conso_2020_2021_mixte, - sum(conso_2020_2021_route) as conso_2020_2021_route, - sum(conso_2020_2021_ferroviaire) as conso_2020_2021_ferroviaire, - sum(conso_2020_2021_inconnu) as conso_2020_2021_inconnu, - sum(conso_2021_2022) as conso_2021_2022, - sum(conso_2021_2022_activite) as conso_2021_2022_activite, - sum(conso_2021_2022_habitat) as conso_2021_2022_habitat, - sum(conso_2021_2022_mixte) as conso_2021_2022_mixte, - sum(conso_2021_2022_route) as conso_2021_2022_route, - sum(conso_2021_2022_ferroviaire) as conso_2021_2022_ferroviaire, - sum(conso_2021_2022_inconnu) as conso_2021_2022_inconnu, - sum(conso_2022_2023) as conso_2022_2023, - sum(conso_2022_2023_activite) as conso_2022_2023_activite, - sum(conso_2022_2023_habitat) as conso_2022_2023_habitat, - sum(conso_2022_2023_mixte) as conso_2022_2023_mixte, - sum(conso_2022_2023_route) as conso_2022_2023_route, - sum(conso_2022_2023_ferroviaire) as conso_2022_2023_ferroviaire, - sum(conso_2022_2023_inconnu) as conso_2022_2023_inconnu, + sum(conso_2009) as conso_2009, + sum(conso_2009_activite) as conso_2009_activite, + sum(conso_2009_habitat) as conso_2009_habitat, + sum(conso_2009_mixte) as conso_2009_mixte, + sum(conso_2009_route) as conso_2009_route, + sum(conso_2009_ferroviaire) as conso_2009_ferroviaire, + sum(conso_2009_inconnu) as conso_2009_inconnu, + sum(conso_2010) as conso_2010, + sum(conso_2010_activite) as conso_2010_activite, + sum(conso_2010_habitat) as conso_2010_habitat, + sum(conso_2010_mixte) as conso_2010_mixte, + sum(conso_2010_route) as conso_2010_route, + sum(conso_2010_ferroviaire) as conso_2010_ferroviaire, + sum(conso_2010_inconnu) as conso_2010_inconnu, + sum(conso_2011) as conso_2011, + sum(conso_2011_activite) as conso_2011_activite, + sum(conso_2011_habitat) as conso_2011_habitat, + sum(conso_2011_mixte) as conso_2011_mixte, + sum(conso_2011_route) as conso_2011_route, + sum(conso_2011_ferroviaire) as conso_2011_ferroviaire, + sum(conso_2011_inconnu) as conso_2011_inconnu, + sum(conso_2012) as conso_2012, + sum(conso_2012_activite) as conso_2012_activite, + sum(conso_2012_habitat) as conso_2012_habitat, + sum(conso_2012_mixte) as conso_2012_mixte, + sum(conso_2012_route) as conso_2012_route, + sum(conso_2012_ferroviaire) as conso_2012_ferroviaire, + sum(conso_2012_inconnu) as conso_2012_inconnu, + sum(conso_2013) as conso_2013, + sum(conso_2013_activite) as conso_2013_activite, + sum(conso_2013_habitat) as conso_2013_habitat, + sum(conso_2013_mixte) as conso_2013_mixte, + sum(conso_2013_route) as conso_2013_route, + sum(conso_2013_ferroviaire) as conso_2013_ferroviaire, + sum(conso_2013_inconnu) as conso_2013_inconnu, + sum(conso_2014) as conso_2014, + sum(conso_2014_activite) as conso_2014_activite, + sum(conso_2014_habitat) as conso_2014_habitat, + sum(conso_2014_mixte) as conso_2014_mixte, + sum(conso_2014_route) as conso_2014_route, + sum(conso_2014_ferroviaire) as conso_2014_ferroviaire, + sum(conso_2014_inconnu) as conso_2014_inconnu, + sum(conso_2015) as conso_2015, + sum(conso_2015_activite) as conso_2015_activite, + sum(conso_2015_habitat) as conso_2015_habitat, + sum(conso_2015_mixte) as conso_2015_mixte, + sum(conso_2015_route) as conso_2015_route, + sum(conso_2015_ferroviaire) as conso_2015_ferroviaire, + sum(conso_2015_inconnu) as conso_2015_inconnu, + sum(conso_2016) as conso_2016, + sum(conso_2016_activite) as conso_2016_activite, + sum(conso_2016_habitat) as conso_2016_habitat, + sum(conso_2016_mixte) as conso_2016_mixte, + sum(conso_2016_route) as conso_2016_route, + sum(conso_2016_ferroviaire) as conso_2016_ferroviaire, + sum(conso_2016_inconnu) as conso_2016_inconnu, + sum(conso_2017) as conso_2017, + sum(conso_2017_activite) as conso_2017_activite, + sum(conso_2017_habitat) as conso_2017_habitat, + sum(conso_2017_mixte) as conso_2017_mixte, + sum(conso_2017_route) as conso_2017_route, + sum(conso_2017_ferroviaire) as conso_2017_ferroviaire, + sum(conso_2017_inconnu) as conso_2017_inconnu, + sum(conso_2018) as conso_2018, + sum(conso_2018_activite) as conso_2018_activite, + sum(conso_2018_habitat) as conso_2018_habitat, + sum(conso_2018_mixte) as conso_2018_mixte, + sum(conso_2018_route) as conso_2018_route, + sum(conso_2018_ferroviaire) as conso_2018_ferroviaire, + sum(conso_2018_inconnu) as conso_2018_inconnu, + sum(conso_2019) as conso_2019, + sum(conso_2019_activite) as conso_2019_activite, + sum(conso_2019_habitat) as conso_2019_habitat, + sum(conso_2019_mixte) as conso_2019_mixte, + sum(conso_2019_route) as conso_2019_route, + sum(conso_2019_ferroviaire) as conso_2019_ferroviaire, + sum(conso_2019_inconnu) as conso_2019_inconnu, + sum(conso_2020) as conso_2020, + sum(conso_2020_activite) as conso_2020_activite, + sum(conso_2020_habitat) as conso_2020_habitat, + sum(conso_2020_mixte) as conso_2020_mixte, + sum(conso_2020_route) as conso_2020_route, + sum(conso_2020_ferroviaire) as conso_2020_ferroviaire, + sum(conso_2020_inconnu) as conso_2020_inconnu, + sum(conso_2021) as conso_2021, + sum(conso_2021_activite) as conso_2021_activite, + sum(conso_2021_habitat) as conso_2021_habitat, + sum(conso_2021_mixte) as conso_2021_mixte, + sum(conso_2021_route) as conso_2021_route, + sum(conso_2021_ferroviaire) as conso_2021_ferroviaire, + sum(conso_2021_inconnu) as conso_2021_inconnu, + sum(conso_2022) as conso_2022, + sum(conso_2022_activite) as conso_2022_activite, + sum(conso_2022_habitat) as conso_2022_habitat, + sum(conso_2022_mixte) as conso_2022_mixte, + sum(conso_2022_route) as conso_2022_route, + sum(conso_2022_ferroviaire) as conso_2022_ferroviaire, + sum(conso_2022_inconnu) as conso_2022_inconnu, sum(conso_2009_2023) as conso_2009_2023, sum(conso_2009_2023_activite) as conso_2009_2023_activite, sum(conso_2009_2023_habitat) as conso_2009_2023_habitat, diff --git a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql index 4fe17576f..531398332 100644 --- a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql +++ b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql @@ -7,21 +7,17 @@ {% endif %} SELECT {{ group_by_column}} as {{ code_name }}, - {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} - {% for type_conso_suffix in type_conso_suffixes %} - {% call(start_year, end_year) cumulative_flux( - first_available_year=2009, - last_available_year=2022 - ) %} - sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) - as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}, - sum(conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}) * 100 / sum(commune.surface) - as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}_percent - {% endcall %} - {% if not loop.last -%}, {% endif %} - {% endfor %} + from_year, + to_year, + {{ sum_percent_median_avg('total', 'commune.surface') }}, + {{ sum_percent_median_avg('activite', 'commune.surface') }}, + {{ sum_percent_median_avg('habitat', 'commune.surface') }}, + {{ sum_percent_median_avg('mixte', 'commune.surface') }}, + {{ sum_percent_median_avg('route', 'commune.surface') }}, + {{ sum_percent_median_avg('ferroviaire', 'commune.surface') }}, + {{ sum_percent_median_avg('inconnu', 'commune.surface') }} FROM - {{ ref('consommation_cog_2024') }} as consommation + {{ ref('consommation_commune') }} as consommation LEFT JOIN {{ ref('commune') }} as commune ON commune.code = consommation.commune_code @@ -33,5 +29,7 @@ LEFT JOIN WHERE {{ group_by_column }} IS NOT NULL GROUP BY - {{ group_by_column }} + {{ group_by_column }}, + from_year, + to_year {% endmacro %} diff --git a/airflow/include/sql/sparte/macros/utils/sum_percent.sql b/airflow/include/sql/sparte/macros/utils/sum_percent.sql new file mode 100644 index 000000000..01d38cb8c --- /dev/null +++ b/airflow/include/sql/sparte/macros/utils/sum_percent.sql @@ -0,0 +1,10 @@ + +{% macro sum_percent(field, total_field) %} + sum({{ field }}) as {{ field }}, + ( -- prevents division by zero + CASE + WHEN sum({{ total_field }}) = 0 THEN sum({{ field }}) * 100 / 1 + ELSE sum({{ field }}) * 100 / sum({{ total_field }}) + END + ) as {{ field }}_percent +{% endmacro %} diff --git a/airflow/include/sql/sparte/macros/utils/sum_percent_median_avg.sql b/airflow/include/sql/sparte/macros/utils/sum_percent_median_avg.sql new file mode 100644 index 000000000..1990c78a7 --- /dev/null +++ b/airflow/include/sql/sparte/macros/utils/sum_percent_median_avg.sql @@ -0,0 +1,16 @@ + +{% macro sum_percent_median_avg(field, total_field) %} + sum({{ field }}) as {{ field }}, + ( + sum({{ field }}) * 100 / ( + {# Les valeurs à 0 sont remplacées par 1 pour éviter les divisions par 0 #} + CASE + WHEN sum({{ total_field }}) = 0 THEN 1 + ELSE sum({{ total_field }}) + END + ) + ) as {{ field }}_percent, + PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY {{ field }}) as {{ field }}_median, + PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY {{ field }}_percent) as {{ field }}_median_percent, + avg({{ field }}) as {{ field }}_avg +{% endmacro %} diff --git a/airflow/include/sql/sparte/models/for_app/for_app_cerema.sql b/airflow/include/sql/sparte/models/for_app/for_app_cerema.sql index 3d4d50e86..9c3a2a7ad 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_cerema.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_cerema.sql @@ -21,108 +21,112 @@ SELECT conso_2009_2023_mixte as art09mix23, conso_2009_2023_habitat as art09hab23, conso_2009_2023_activite as art09act23, - conso_2022_2023_inconnu as art22inc23, - conso_2022_2023_ferroviaire as art22fer23, - conso_2022_2023_route as art22rou23, - conso_2022_2023_mixte as art22mix23, - conso_2022_2023_habitat as art22hab23, - conso_2022_2023_activite as art22act23, - conso_2022_2023 as naf22art23, - conso_2021_2022_inconnu as art21inc22, - conso_2021_2022_ferroviaire as art21fer22, - conso_2021_2022_route as art21rou22, - conso_2021_2022_mixte as art21mix22, - conso_2021_2022_habitat as art21hab22, - conso_2021_2022_activite as art21act22, - conso_2021_2022 as naf21art22, - conso_2020_2021_inconnu as art20inc21, - conso_2020_2021_ferroviaire as art20fer21, - conso_2020_2021_route as art20rou21, - conso_2020_2021_mixte as art20mix21, - conso_2020_2021_habitat as art20hab21, - conso_2020_2021_activite as art20act21, - conso_2020_2021 as naf20art21, - conso_2019_2020_inconnu as art19inc20, - conso_2019_2020_ferroviaire as art19fer20, - conso_2019_2020_route as art19rou20, - conso_2019_2020_mixte as art19mix20, - conso_2019_2020_habitat as art19hab20, - conso_2019_2020_activite as art19act20, - conso_2019_2020 as naf19art20, - conso_2018_2019_inconnu as art18inc19, - conso_2018_2019_ferroviaire as art18fer19, - conso_2018_2019_route as art18rou19, - conso_2018_2019_mixte as art18mix19, - conso_2018_2019_habitat as art18hab19, - conso_2018_2019_activite as art18act19, - conso_2018_2019 as naf18art19, - conso_2017_2018_inconnu as art17inc18, - conso_2017_2018_ferroviaire as art17fer18, - conso_2017_2018_route as art17rou18, - conso_2017_2018_mixte as art17mix18, - conso_2017_2018_habitat as art17hab18, - conso_2017_2018_activite as art17act18, - conso_2017_2018 as naf17art18, - conso_2016_2017_inconnu as art16inc17, - conso_2016_2017_ferroviaire as art16fer17, - conso_2016_2017_route as art16rou17, - conso_2016_2017_mixte as art16mix17, - conso_2016_2017_habitat as art16hab17, - conso_2016_2017_activite as art16act17, - conso_2016_2017 as naf16art17, - conso_2015_2016_inconnu as art15inc16, - conso_2015_2016_ferroviaire as art15fer16, - conso_2015_2016_route as art15rou16, - conso_2015_2016_mixte as art15mix16, - conso_2015_2016_habitat as art15hab16, - conso_2015_2016_activite as art15act16, - conso_2015_2016 as naf15art16, - conso_2014_2015_inconnu as art14inc15, - conso_2014_2015_ferroviaire as art14fer15, - conso_2014_2015_route as art14rou15, - conso_2014_2015_mixte as art14mix15, - conso_2014_2015_habitat as art14hab15, - conso_2014_2015_activite as art14act15, - conso_2014_2015 as naf14art15, - conso_2013_2014_inconnu as art13inc14, - conso_2013_2014_ferroviaire as art13fer14, - conso_2013_2014_route as art13rou14, - conso_2013_2014_mixte as art13mix14, - conso_2013_2014_habitat as art13hab14, - conso_2013_2014_activite as art13act14, - conso_2013_2014 as naf13art14, - conso_2012_2013_inconnu as art12inc13, - conso_2012_2013_ferroviaire as art12fer13, - conso_2012_2013_route as art12rou13, - conso_2012_2013_mixte as art12mix13, - conso_2012_2013_habitat as art12hab13, - conso_2012_2013_activite as art12act13, - conso_2012_2013 as naf12art13, - conso_2011_2012_inconnu as art11inc12, - conso_2011_2012_ferroviaire as art11fer12, - conso_2011_2012_route as art11rou12, - conso_2011_2012_mixte as art11mix12, - conso_2011_2012_habitat as art11hab12, - conso_2011_2012_activite as art11act12, - conso_2011_2012 as naf11art12, - conso_2010_2011_inconnu as art10inc11, - conso_2010_2011_ferroviaire as art10fer11, - conso_2010_2011_route as art10rou11, - conso_2010_2011_mixte as art10mix11, - conso_2010_2011_habitat as art10hab11, - conso_2010_2011_activite as art10act11, - conso_2010_2011 as naf10art11, - conso_2009_2010_inconnu as art09inc10, - conso_2009_2010_ferroviaire as art09fer10, - conso_2009_2010_route as art09rou10, - conso_2009_2010_mixte as art09mix10, - conso_2009_2010_habitat as art09hab10, - conso_2009_2010_activite as art09act10, - conso_2009_2010 as naf09art10, - commune.srid_source as srid_source, + conso_2022_inconnu as art22inc23, + conso_2022_ferroviaire as art22fer23, + conso_2022_route as art22rou23, + conso_2022_mixte as art22mix23, + conso_2022_habitat as art22hab23, + conso_2022_activite as art22act23, + conso_2022 as naf22art23, + conso_2021_inconnu as art21inc22, + conso_2021_ferroviaire as art21fer22, + conso_2021_route as art21rou22, + conso_2021_mixte as art21mix22, + conso_2021_habitat as art21hab22, + conso_2021_activite as art21act22, + conso_2021 as naf21art22, + conso_2020_inconnu as art20inc21, + conso_2020_ferroviaire as art20fer21, + conso_2020_route as art20rou21, + conso_2020_mixte as art20mix21, + conso_2020_habitat as art20hab21, + conso_2020_activite as art20act21, + conso_2020 as naf20art21, + conso_2019_inconnu as art19inc20, + conso_2019_ferroviaire as art19fer20, + conso_2019_route as art19rou20, + conso_2019_mixte as art19mix20, + conso_2019_habitat as art19hab20, + conso_2019_activite as art19act20, + conso_2019 as naf19art20, + conso_2018_inconnu as art18inc19, + conso_2018_ferroviaire as art18fer19, + conso_2018_route as art18rou19, + conso_2018_mixte as art18mix19, + conso_2018_habitat as art18hab19, + conso_2018_activite as art18act19, + conso_2018 as naf18art19, + conso_2017_inconnu as art17inc18, + conso_2017_ferroviaire as art17fer18, + conso_2017_route as art17rou18, + conso_2017_mixte as art17mix18, + conso_2017_habitat as art17hab18, + conso_2017_activite as art17act18, + conso_2017 as naf17art18, + conso_2016_inconnu as art16inc17, + conso_2016_ferroviaire as art16fer17, + conso_2016_route as art16rou17, + conso_2016_mixte as art16mix17, + conso_2016_habitat as art16hab17, + conso_2016_activite as art16act17, + conso_2016 as naf16art17, + conso_2015_inconnu as art15inc16, + conso_2015_ferroviaire as art15fer16, + conso_2015_route as art15rou16, + conso_2015_mixte as art15mix16, + conso_2015_habitat as art15hab16, + conso_2015_activite as art15act16, + conso_2015 as naf15art16, + conso_2014_inconnu as art14inc15, + conso_2014_ferroviaire as art14fer15, + conso_2014_route as art14rou15, + conso_2014_mixte as art14mix15, + conso_2014_habitat as art14hab15, + conso_2014_activite as art14act15, + conso_2014 as naf14art15, + conso_2013_inconnu as art13inc14, + conso_2013_ferroviaire as art13fer14, + conso_2013_route as art13rou14, + conso_2013_mixte as art13mix14, + conso_2013_habitat as art13hab14, + conso_2013_activite as art13act14, + conso_2013 as naf13art14, + conso_2012_inconnu as art12inc13, + conso_2012_ferroviaire as art12fer13, + conso_2012_route as art12rou13, + conso_2012_mixte as art12mix13, + conso_2012_habitat as art12hab13, + conso_2012_activite as art12act13, + conso_2012 as naf12art13, + conso_2011_inconnu as art11inc12, + conso_2011_ferroviaire as art11fer12, + conso_2011_route as art11rou12, + conso_2011_mixte as art11mix12, + conso_2011_habitat as art11hab12, + conso_2011_activite as art11act12, + conso_2011 as naf11art12, + conso_2010_inconnu as art10inc11, + conso_2010_ferroviaire as art10fer11, + conso_2010_route as art10rou11, + conso_2010_mixte as art10mix11, + conso_2010_habitat as art10hab11, + conso_2010_activite as art10act11, + conso_2010 as naf10art11, + conso_2009_inconnu as art09inc10, + conso_2009_ferroviaire as art09fer10, + conso_2009_route as art09rou10, + conso_2009_mixte as art09mix10, + conso_2009_habitat as art09hab10, + conso_2009_activite as art09act10, + conso_2009 as naf09art10, conso_2011_2021 as naf11art21, + conso_2011_2021_inconnu as naf11inc21, + conso_2011_2021_ferroviaire as naf11fer21, + conso_2011_2021_route as naf11rou21, + conso_2011_2021_mixte as naf11mix21, conso_2011_2021_activite as naf11act21, conso_2011_2021_habitat as naf11hab21, + commune.srid_source as srid_source, {{ make_valid_multipolygon('ST_Transform(commune.geom, 4326)') }} as mpoly FROM {{ ref("consommation_cog_2024") }} as consommation diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql b/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql new file mode 100644 index 000000000..27b491e44 --- /dev/null +++ b/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql @@ -0,0 +1,77 @@ +{{ + config( + materialized='table', + docs={'node_color': 'purple'} + ) +}} + +{% set fields_to_query = """ + from_year, + to_year, + total, + total_percent, + activite, + activite_percent, + habitat, + habitat_percent, + mixte, + mixte_percent, + route, + route_percent, + ferroviaire, + ferroviaire_percent, + inconnu, + inconnu_percent +""" %} + +SELECT + commune_code as land_id, + 'COMMUNE' as land_type, + 'EPCI' as comparison_level, + commune.epci as comparison_id, + {{ fields_to_query }} + +FROM + {{ ref('consommation_commune') }} +LEFT JOIN + {{ ref('commune') }} as commune + ON commune_code = commune.code +UNION +SELECT + epci as land_id, + 'EPCI' as land_type, + 'NATION' as comparison_level, + 'NATION' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('consommation_epci') }} +UNION +SELECT + departement as land_id, + 'DEPARTEMENT' as land_type, + 'REGION' as comparison_level, + departement.region as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('consommation_departement') }} +LEFT JOIN + {{ ref('departement') }} as departement + ON departement.code = consommation_departement.departement +UNION +SELECT + region as land_id, + 'REGION' as land_type, + 'NATION' as comparison_level, + 'NATION' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('consommation_region') }} +UNION +SELECT + scot as land_id, + 'SCOT' as land_type, + 'NATION' as comparison_level, + 'NATION' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('consommation_scot') }} diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql b/airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql new file mode 100644 index 000000000..e1378a821 --- /dev/null +++ b/airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql @@ -0,0 +1,80 @@ +{{ + config( + materialized='table', + docs={'node_color': 'purple'} + ) +}} + +{% set fields_to_query = """ + from_year, + to_year, + total_median, + total_median_percent, + total_avg, + total_percent, + activite_median, + activite_median_percent, + activite_avg, + activite_percent, + habitat_median, + habitat_median_percent, + habitat_avg, + habitat_percent, + mixte_median, + mixte_median_percent, + mixte_avg, + mixte_percent, + route_median, + route_median_percent, + route_avg, + route_percent, + ferroviaire_median, + ferroviaire_median_percent, + ferroviaire_avg, + ferroviaire_percent, + inconnu_median, + inconnu_median_percent, + inconnu_avg, + inconnu_percent +""" %} + + +SELECT + 'COMMUNE' as relevance_level, + 'EPCI' as land_type, + epci_stats_commune.epci as land_id, + {{ fields_to_query }} +FROM + {{ ref('conso_epci_stats_commune') }} as epci_stats_commune +UNION +SELECT + 'DEPARTEMENT' as relevance_level, + 'REGION' as land_type, + region_stats_departement.region as land_id, + {{ fields_to_query }} +FROM + {{ ref('conso_regional_stats_departement') }} as region_stats_departement +UNION +SELECT + 'EPCI' as relevance_level, + 'NATION' as land_type, + 'NATION' as land_id, + {{ fields_to_query }} +FROM + {{ ref('conso_national_stats_epci') }} as national_stats_epci +UNION +SELECT + 'REGION' as relevance_level, + 'NATION' as land_type, + 'NATION' as land_id, + {{ fields_to_query }} +FROM + {{ ref('conso_national_stats_region') }} as national_stats_region +UNION +SELECT + 'SCOT' as relevance_level, + 'NATION' as land_type, + 'NATION' as land_id, + {{ fields_to_query }} +FROM + {{ ref('conso_national_stats_scot') }} as national_stats_scot diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql b/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql new file mode 100644 index 000000000..c2709b344 --- /dev/null +++ b/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql @@ -0,0 +1,48 @@ +{{ + config( + materialized='table', + docs={'node_color': 'purple'} + ) +}} + +{% set fields_to_query = """ + from_year, + to_year, + evolution, + evolution_percent +""" %} + +SELECT + code_commune as land_id, + 'COMMUNE' as land_type, + {{ fields_to_query }} +FROM + {{ ref('flux_population_commune') }} +UNION +SELECT + epci as land_id, + 'EPCI' as land_type, + {{ fields_to_query }} +FROM + {{ ref('flux_population_epci') }} +UNION +SELECT + departement as land_id, + 'DEPARTEMENT' as land_type, + {{ fields_to_query }} +FROM + {{ ref('flux_population_departement') }} +UNION +SELECT + region as land_id, + 'REGION' as land_type, + {{ fields_to_query }} +FROM + {{ ref('flux_population_region') }} +UNION +SELECT + scot as land_id, + 'SCOT' as land_type, + {{ fields_to_query }} +FROM + {{ ref('flux_population_scot') }} diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql b/airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql new file mode 100644 index 000000000..2aaf05cca --- /dev/null +++ b/airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql @@ -0,0 +1,56 @@ +{{ + config( + materialized='table', + docs={'node_color': 'purple'} + ) +}} + +{% set fields_to_query = """ + from_year, + to_year, + evolution_median, + evolution_median_percent, + evolution_avg, + evolution_percent +""" %} + + +SELECT + 'COMMUNE' as relevance_level, + 'EPCI' as land_type, + epci_stats_commune.epci as land_id, + {{ fields_to_query }} +FROM + {{ ref('pop_epci_stats_commune') }} as epci_stats_commune +UNION +SELECT + 'DEPARTEMENT' as relevance_level, + 'REGION' as land_type, + region_stats_departement.region as land_id, + {{ fields_to_query }} +FROM + {{ ref('pop_regional_stats_departement') }} as region_stats_departement +UNION +SELECT + 'EPCI' as relevance_level, + 'NATION' as land_type, + 'NATION' as land_id, + {{ fields_to_query }} +FROM + {{ ref('pop_national_stats_epci') }} as national_stats_epci +UNION +SELECT + 'REGION' as relevance_level, + 'NATION' as land_type, + 'NATION' as land_id, + {{ fields_to_query }} +FROM + {{ ref('pop_national_stats_region') }} as national_stats_region +UNION +SELECT + 'SCOT' as relevance_level, + 'NATION' as land_type, + 'NATION' as land_id, + {{ fields_to_query }} +FROM + {{ ref('pop_national_stats_scot') }} as national_stats_scot diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql new file mode 100644 index 000000000..415fa7253 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql @@ -0,0 +1,37 @@ +{{ config(materialized='table') }} + +{% for to_year in range(2010, 2023) %} + {% for from_year in range(2009, to_year) %} + {% if from_year >= to_year %} + {% break %} + {% endif %} + SELECT + flux_population.code_commune, + {{ from_year }} as from_year, + {{ to_year }} as to_year, + {{ sum_percent( + 'evolution', + 'stock_population.population_' + from_year|string + ) }}, + max(stock_population.population_{{ from_year }}) as start_population + FROM + {{ ref('flux_population') }} as flux_population + LEFT JOIN + {{ ref('commune') }} as commune + ON commune.code = flux_population.code_commune + LEFT JOIN + {{ ref('population_cog_2024') }} as stock_population + ON stock_population.code_commune = flux_population.code_commune + WHERE year BETWEEN {{ from_year }} AND {{ to_year }} + GROUP BY + flux_population.code_commune, + from_year, + to_year + {% if not loop.last %} + UNION + {% endif %} + {% endfor %} + {% if not loop.last %} + UNION + {% endif %} +{% endfor %} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml new file mode 100644 index 000000000..c7edc00f2 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml @@ -0,0 +1,52 @@ +version: 2 + +accepted_years: &accepted_years + values : [ + 2009, + 2010, + 2011, + 2012, + 2013, + 2014, + 2015, + 2016, + 2017, + 2018, + 2019, + 2020, + 2021, + 2022 + ] + +models: + - name: flux_population_commune + columns: + - name: code_commune + data_tests: + - has_all_communes + - relationships: + to: ref('commune') + field: code + - name: from_year + data_tests: + - not_null + - accepted_values: *accepted_years + - name: to_year + data_tests: + - not_null + - accepted_values: *accepted_years + - name: evolution + data_tests: + - not_null + - name: evolution_percent + data_tests: + - not_null + - name: evolution_median + data_tests: + - not_null + - name: evolution_avg + data_tests: + - not_null + - name: start_population + data_tests: + - not_null diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departements.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departements.sql rename to airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml new file mode 100644 index 000000000..0fb786704 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: flux_population_departement + columns: + - name: departement + data_tests: + - not_null + - has_all_departements + - relationships: + to: ref('departement') + field: code diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epcis.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epcis.sql rename to airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml new file mode 100644 index 000000000..9a3c1ac39 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: flux_population_epci + columns: + - name: epci + data_tests: + - not_null + - has_all_epcis + - relationships: + to: ref('epci') + field: code diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_regions.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_regions.sql rename to airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml new file mode 100644 index 000000000..4a6608b0d --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml @@ -0,0 +1,12 @@ +version: 2 + +models: + - name: flux_population_region + columns: + - name: region + data_tests: + - not_null + - has_all_regions + - relationships: + to: ref('region') + field: code diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scots.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scots.sql rename to airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml new file mode 100644 index 000000000..dcc273da6 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml @@ -0,0 +1,12 @@ +version: 2 + +models: + - name: flux_population_scot + columns: + - name: scot + data_tests: + - not_null + - has_all_scots + - relationships: + to: ref('scot') + field: id_scot diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departements.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departements.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml new file mode 100644 index 000000000..e834717b3 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: stock_population_departement + columns: + - name: departement + data_tests: + - not_null + - unique + - has_all_departements + - relationships: + to: ref('departement') + field: code diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epcis.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epcis.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml new file mode 100644 index 000000000..e5e2fc4ed --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: stock_population_epci + columns: + - name: epci + data_tests: + - not_null + - unique + - has_all_epcis + - relationships: + to: ref('epci') + field: code diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_regions.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_regions.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml new file mode 100644 index 000000000..d85d3d1aa --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: stock_population_region + columns: + - name: region + data_tests: + - not_null + - unique + - has_all_regions + - relationships: + to: ref('region') + field: code diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scots.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scots.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml new file mode 100644 index 000000000..45d0a7440 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: stock_population_scot + columns: + - name: scot + data_tests: + - not_null + - unique + - has_all_scots + - relationships: + to: ref('scot') + field: id_scot diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql new file mode 100644 index 000000000..f7d6b725d --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql @@ -0,0 +1,16 @@ +{{ config(materialized='table') }} + +SELECT + epci, + from_year, + to_year, + {{ sum_percent_median_avg('evolution', 'start_population') }} +FROM + {{ ref('flux_population_commune') }} as pop +LEFT JOIN + {{ ref('commune') }} as commune + ON commune.code = pop.code_commune +GROUP BY + commune.epci, + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.yml b/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.yml new file mode 100644 index 000000000..3258993af --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.yml @@ -0,0 +1,39 @@ +version: 2 + +accepted_years: &accepted_years + values : [ + 2009, + 2010, + 2011, + 2012, + 2013, + 2014, + 2015, + 2016, + 2017, + 2018, + 2019, + 2020, + 2021, + 2022 + ] + +models: + - name: pop_epci_stats_commune + columns: + - name: epci + data_tests: + - not_null + - unique + - has_all_epcis + - relationships: + to: ref('epci') + field: code + - name: from_year + data_tests: + - not_null + - accepted_values: *accepted_years + - name: to_year + data_tests: + - not_null + - accepted_values: *accepted_years diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql new file mode 100644 index 000000000..aea8946d8 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +SELECT + from_year, + to_year, + {{ sum_percent_median_avg('evolution', 'start_population') }} +FROM + {{ ref('flux_population_epci') }} as pop +GROUP BY + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql new file mode 100644 index 000000000..caea0a3d6 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +SELECT + from_year, + to_year, + {{ sum_percent_median_avg('evolution', 'start_population') }} +FROM + {{ ref('flux_population_region') }} as pop +GROUP BY + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql new file mode 100644 index 000000000..ba36a7ee4 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +SELECT + from_year, + to_year, + {{ sum_percent_median_avg('evolution', 'start_population') }} +FROM + {{ ref('flux_population_scot') }} as pop +GROUP BY + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql new file mode 100644 index 000000000..14b737fcc --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql @@ -0,0 +1,16 @@ +{{ config(materialized='table') }} + +SELECT + commune.region, + from_year, + to_year, + {{ sum_percent_median_avg('evolution', 'start_population') }} +FROM + {{ ref('flux_population_departement') }} as pop +LEFT JOIN + {{ ref('commune') }} as commune + ON commune.departement = pop.departement +GROUP BY + commune.region, + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/insee/cog_changes_2024.yml b/airflow/include/sql/sparte/models/insee/cog_changes_2024.yml new file mode 100644 index 000000000..43333bb36 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/cog_changes_2024.yml @@ -0,0 +1,40 @@ + +version: 2 + +type_de_changement: &type_de_changement + values : [ + 10, # Changement de nom + 20, # Création + 21, # Rétablissement + 30, # Suppression + 31, # Fusion simple + 32, # Création de commune nouvelle + 33, # Fusion association + 34, # Transformation de fusion association en fusion simple + 35, # Suppression de commune déléguée + 41, # Changement de code dû à un changement de département + 50, # Changement de code dû à un transfert de chef-lieu + 70, # Transformation de commune associée en commune déléguée + 71 # 71 is not in the official list but is present in the data + ] + +type_commune: &type_commune + values : [ + "COM", # Commune + "COMA", # Commune associée + "COMD", # Commune déléguée + "ARM" # Arrondissement municipal + ] + +models: + - name: cog_changes_2024 + columns: + - name: type_de_changement + data_tests: + - accepted_values: *type_de_changement + - name: type_commune_avant + data_tests: + - accepted_values: *type_commune + - name: type_commune_apres + data_tests: + - accepted_values: *type_commune diff --git a/airflow/include/sql/sparte/models/insee/flux_population.sql b/airflow/include/sql/sparte/models/insee/flux_population.sql index bfbb912ff..c400236ce 100644 --- a/airflow/include/sql/sparte/models/insee/flux_population.sql +++ b/airflow/include/sql/sparte/models/insee/flux_population.sql @@ -4,50 +4,54 @@ indexes=[{'columns': ['code_commune'], 'type': 'btree'}] ) }} -with flux as ( +with known_years as ( + {% for year in range(2009, 2021) %} + {% set next_year = year + 1 %} + SELECT + code_commune, + {{ year }} as year, + (population_{{ next_year }} - population_{{ year }}) as evolution, + population_{{ year }} as population, + 'INSEE' as source + FROM + {{ ref('population_cog_2024') }} + {% if not loop.last %} + UNION + {% endif %} + {% endfor %} +), average_evolution as ( SELECT - *, -- keep stock columns - (population_2010 - population_2009) as population_2009_2010, - (population_2011 - population_2010) as population_2010_2011, - (population_2012 - population_2011) as population_2011_2012, - (population_2013 - population_2012) as population_2012_2013, - (population_2014 - population_2013) as population_2013_2014, - (population_2015 - population_2014) as population_2014_2015, - (population_2016 - population_2015) as population_2015_2016, - (population_2017 - population_2016) as population_2016_2017, - (population_2018 - population_2017) as population_2017_2018, - (population_2019 - population_2018) as population_2018_2019, - (population_2020 - population_2019) as population_2019_2020, - (population_2021 - population_2020) as population_2020_2021 + code_commune, + AVG(evolution) as average_evolution FROM - {{ ref('population_cog_2024') }} + known_years + GROUP BY + code_commune +), predictions as ( + {% set first_unknown_year = 2021 %} + {% for year in range(first_unknown_year, 2023) %} + {% set next_year = year + 1 %} + SELECT + code_commune, + {{ year }} as year, + ( + SELECT round(average_evolution)::numeric + FROM average_evolution + WHERE code_commune = pop.code_commune + ) as evolution, + population_{{ first_unknown_year }} + ( + SELECT round(average_evolution)::numeric + FROM average_evolution + WHERE code_commune = pop.code_commune + ) * ({{ year }} - 2020) as population, + 'PROJECTION' as source + FROM + {{ ref('population_cog_2024') }} as pop + {% if not loop.last %} + UNION + {% endif %} + {% endfor %} ) -SELECT - code_commune, - population_2009, - population_2010, - population_2011, - population_2012, - population_2013, - population_2014, - population_2015, - population_2016, - population_2017, - population_2018, - population_2019, - population_2020, - population_2021, - {% call(start_year, end_year) cumulative_flux( - first_available_year=2009, - last_available_year=2020 - ) %} - ( - {% for first_year in range(start_year, end_year + 1) -%} - {% set next_year = first_year + 1 -%} - population_{{ first_year }}_{{ next_year }} - {% if not loop.last -%} + {% endif %} - {% endfor %} - ) as population_{{ start_year }}_{{ end_year + 1 }} - {% endcall %} -FROM - flux +SELECT * FROM known_years +UNION +SELECT * FROM predictions diff --git a/airflow/include/sql/sparte/models/insee/flux_population.yml b/airflow/include/sql/sparte/models/insee/flux_population.yml new file mode 100644 index 000000000..ef21beb1b --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/flux_population.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: flux_population + columns: + - name: code_commune + data_tests: + - has_all_communes + - relationships: + to: ref('commune') + field: code diff --git a/airflow/include/sql/sparte/models/insee/population.yml b/airflow/include/sql/sparte/models/insee/population.yml new file mode 100644 index 000000000..82846b497 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/population.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: population + columns: + - name: code_commune + data_tests: + - not_null + - unique + - name: population_2021 + data_tests: + - not_null + - name: population_2020 + data_tests: + - not_null + - name: population_2019 + data_tests: + - not_null + - name: population_2018 + data_tests: + - not_null + - name: population_2017 + data_tests: + - not_null + - name: population_2016 + data_tests: + - not_null diff --git a/airflow/include/sql/sparte/models/insee/population_cog_2024.yml b/airflow/include/sql/sparte/models/insee/population_cog_2024.yml new file mode 100644 index 000000000..8b70e1a66 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/population_cog_2024.yml @@ -0,0 +1,13 @@ +version: 2 + +models: + - name: population_cog_2024 + columns: + - name: code_commune + data_tests: + - not_null + - unique + - has_all_communes + - relationships: + to: ref('commune') + field: code diff --git a/airflow/include/sql/sparte/models/insee/schema.yml b/airflow/include/sql/sparte/models/insee/schema.yml deleted file mode 100644 index 303370abc..000000000 --- a/airflow/include/sql/sparte/models/insee/schema.yml +++ /dev/null @@ -1,146 +0,0 @@ - -version: 2 - -type_de_changement: &type_de_changement - values : [ - 10, # Changement de nom - 20, # Création - 21, # Rétablissement - 30, # Suppression - 31, # Fusion simple - 32, # Création de commune nouvelle - 33, # Fusion association - 34, # Transformation de fusion association en fusion simple - 35, # Suppression de commune déléguée - 41, # Changement de code dû à un changement de département - 50, # Changement de code dû à un transfert de chef-lieu - 70, # Transformation de commune associée en commune déléguée - 71 # 71 is not in the official list but is present in the data - ] - -type_commune: &type_commune - values : [ - "COM", # Commune - "COMA", # Commune associée - "COMD", # Commune déléguée - "ARM" # Arrondissement municipal - ] - -models: - - name: cog_changes_2024 - columns: - - name: type_de_changement - data_tests: - - accepted_values: *type_de_changement - - name: type_commune_avant - data_tests: - - accepted_values: *type_commune - - name: type_commune_apres - data_tests: - - accepted_values: *type_commune - - name: population_cog_2024 - columns: - - name: code_commune - data_tests: - - not_null - - unique - - has_all_communes - - relationships: - to: ref('commune') - field: code - - name: flux_population_departements - columns: - - name: departement - data_tests: - - not_null - - unique - - has_all_departements - - relationships: - to: ref('departement') - field: code - - name: flux_population_regions - columns: - - name: region - data_tests: - - not_null - - unique - - has_all_regions - - relationships: - to: ref('region') - field: code - - name: flux_population_scots - columns: - - name: scot - data_tests: - - not_null - - unique - - has_all_scots - - relationships: - to: ref('scot') - field: id_scot - - name: flux_population_epcis - columns: - - name: epci - data_tests: - - not_null - - unique - - has_all_epcis - - relationships: - to: ref('epci') - field: code - - name: flux_population - columns: - - name: code_commune - data_tests: - - not_null - - unique - - has_all_communes - - relationships: - to: ref('commune') - field: code - - name: stock_population_departements - columns: - - name: departement - data_tests: - - not_null - - unique - - has_all_departements - - relationships: - to: ref('departement') - field: code - - name: stock_population_regions - columns: - - name: region - data_tests: - - not_null - - unique - - has_all_regions - - relationships: - to: ref('region') - field: code - - name: stock_population_epcis - columns: - - name: epci - data_tests: - - not_null - - unique - - has_all_epcis - - relationships: - to: ref('epci') - field: code - - name: stock_population_scots - columns: - - name: scot - data_tests: - - not_null - - unique - - has_all_scots - - relationships: - to: ref('scot') - field: id_scot - -sources: - - name: public - tables: - - name: insee_cog_changes_2024 - - name: insee_population diff --git a/airflow/include/sql/sparte/models/insee/sources.yml b/airflow/include/sql/sparte/models/insee/sources.yml new file mode 100644 index 000000000..fd42732f5 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/sources.yml @@ -0,0 +1,6 @@ +version: 2 +sources: + - name: public + tables: + - name: insee_cog_changes_2024 + - name: insee_population diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql b/airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql new file mode 100644 index 000000000..742be835e --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql @@ -0,0 +1,37 @@ +{{ config(materialized='table') }} + +{% for to_year in range(2010, 2023) %} + {% for from_year in range(2009, to_year) %} + {% if from_year >= to_year %} + {% break %} + {% endif %} + SELECT + commune_code, + {{ from_year }} as from_year, + {{ to_year }} as to_year, + max(commune.surface) as commune_surface, + {{ sum_percent('total', 'commune.surface') }}, + {{ sum_percent('activite', 'commune.surface') }}, + {{ sum_percent('habitat', 'commune.surface') }}, + {{ sum_percent('mixte', 'commune.surface') }}, + {{ sum_percent('route', 'commune.surface') }}, + {{ sum_percent('ferroviaire', 'commune.surface') }}, + {{ sum_percent('inconnu', 'commune.surface') }} + + FROM {{ ref('consommation_en_lignes') }} as conso + LEFT JOIN + {{ ref('commune') }} as commune + ON commune.code = conso.commune_code + WHERE year BETWEEN {{ from_year }} AND {{ to_year }} + GROUP BY + commune_code, + from_year, + to_year + {% if not loop.last %} + UNION + {% endif %} + {% endfor %} + {% if not loop.last %} + UNION + {% endif %} +{% endfor %} diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql new file mode 100644 index 000000000..b23993d24 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +SELECT + epci, + from_year, + to_year, + {{ sum_percent_median_avg('total', 'commune.surface') }}, + {{ sum_percent_median_avg('activite', 'commune.surface') }}, + {{ sum_percent_median_avg('habitat', 'commune.surface') }}, + {{ sum_percent_median_avg('mixte', 'commune.surface') }}, + {{ sum_percent_median_avg('route', 'commune.surface') }}, + {{ sum_percent_median_avg('ferroviaire', 'commune.surface') }}, + {{ sum_percent_median_avg('inconnu', 'commune.surface') }} +FROM + {{ ref('consommation_commune') }} as conso +LEFT JOIN + {{ ref('commune') }} as commune + ON commune.code = conso.commune_code +GROUP BY + commune.epci, + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql new file mode 100644 index 000000000..eddcdf614 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql @@ -0,0 +1,21 @@ +{{ config(materialized='table') }} + +SELECT + + from_year, + to_year, + {{ sum_percent_median_avg('total', 'epci.surface') }}, + {{ sum_percent_median_avg('activite', 'epci.surface') }}, + {{ sum_percent_median_avg('habitat', 'epci.surface') }}, + {{ sum_percent_median_avg('mixte', 'epci.surface') }}, + {{ sum_percent_median_avg('route', 'epci.surface') }}, + {{ sum_percent_median_avg('ferroviaire', 'epci.surface') }}, + {{ sum_percent_median_avg('inconnu', 'epci.surface') }} +FROM + {{ ref('consommation_epci') }} as conso +LEFT JOIN + {{ ref('epci') }} as epci + ON epci.code = conso.epci +GROUP BY + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql new file mode 100644 index 000000000..bc6a090d0 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql @@ -0,0 +1,20 @@ +{{ config(materialized='table') }} + +SELECT + from_year, + to_year, + {{ sum_percent_median_avg('total', 'region.surface') }}, + {{ sum_percent_median_avg('activite', 'region.surface') }}, + {{ sum_percent_median_avg('habitat', 'region.surface') }}, + {{ sum_percent_median_avg('mixte', 'region.surface') }}, + {{ sum_percent_median_avg('route', 'region.surface') }}, + {{ sum_percent_median_avg('ferroviaire', 'region.surface') }}, + {{ sum_percent_median_avg('inconnu', 'region.surface') }} +FROM + {{ ref('consommation_region') }} as conso +LEFT JOIN + {{ ref('region') }} as region + ON region.code = conso.region +GROUP BY + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql new file mode 100644 index 000000000..897b6fe81 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql @@ -0,0 +1,20 @@ +{{ config(materialized='table') }} + +SELECT + from_year, + to_year, + {{ sum_percent_median_avg('total', 'scot.surface') }}, + {{ sum_percent_median_avg('activite', 'scot.surface') }}, + {{ sum_percent_median_avg('habitat', 'scot.surface') }}, + {{ sum_percent_median_avg('mixte', 'scot.surface') }}, + {{ sum_percent_median_avg('route', 'scot.surface') }}, + {{ sum_percent_median_avg('ferroviaire', 'scot.surface') }}, + {{ sum_percent_median_avg('inconnu', 'scot.surface') }} +FROM + {{ ref('consommation_scot') }} as conso +LEFT JOIN + {{ ref('scot') }} as scot + ON scot.id_scot = conso.scot +GROUP BY + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql new file mode 100644 index 000000000..06379f392 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +SELECT + departement.region, + from_year, + to_year, + {{ sum_percent_median_avg('total', 'departement.surface') }}, + {{ sum_percent_median_avg('activite', 'departement.surface') }}, + {{ sum_percent_median_avg('habitat', 'departement.surface') }}, + {{ sum_percent_median_avg('mixte', 'departement.surface') }}, + {{ sum_percent_median_avg('route', 'departement.surface') }}, + {{ sum_percent_median_avg('ferroviaire', 'departement.surface') }}, + {{ sum_percent_median_avg('inconnu', 'departement.surface') }} +FROM + {{ ref('consommation_departement') }} as conso +LEFT JOIN + {{ ref('departement') }} as departement + ON departement.code = conso.departement +GROUP BY + departement.region, + from_year, + to_year diff --git a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql index c9fea1e9f..0cbfdafb7 100644 --- a/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql +++ b/airflow/include/sql/sparte/models/majic/consommation_cog_2024.sql @@ -129,33 +129,27 @@ together as ( select *, 'MISSING_FROM_SOURCE' as correction_status from missing_from_source ) select - commune_code, - correction_status, - {% set type_conso_suffixes = ["", "_activite", "_habitat"] %} - {% for type_conso_suffix in type_conso_suffixes %} - {% call(start_year, end_year) cumulative_flux( - first_available_year=2009, - last_available_year=2022 - ) %} - ( - {% for first_year in range(start_year, end_year + 1) -%} - {% set next_year = first_year + 1 -%} - conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} - {% if not loop.last -%} + {% endif %} - {% endfor %} - ) as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}, - ( - {% for first_year in range(start_year, end_year + 1) -%} - {% set next_year = first_year + 1 -%} - conso_{{ first_year }}_{{ next_year }}{{ type_conso_suffix }} - {% if not loop.last -%} + {% endif %} - {% endfor %} - ) * 100 / commune.surface as conso_{{ start_year }}_{{ end_year + 1 }}{{ type_conso_suffix }}_percent - {% endcall %} - {% if not loop.last -%}, {% endif %} - {% endfor %} + *, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }} {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }}_inconnu {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021_inconnu, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }}_ferroviaire {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021_ferroviaire, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }}_route {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021_route, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }}_mixte {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021_mixte, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }}_activite {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021_activite, + ( + {%- for year in range(2011, 2022) %} conso_{{ year }}_habitat {% if not loop.last %} + {% endif %}{%- endfor %} + ) as conso_2011_2021_habitat from together -LEFT JOIN - {{ ref('commune') }} as commune - ON commune.code = together.commune_code diff --git a/airflow/include/sql/sparte/models/majic/consommation_en_lignes.sql b/airflow/include/sql/sparte/models/majic/consommation_en_lignes.sql new file mode 100644 index 000000000..8f3b836b1 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/consommation_en_lignes.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +{% for year in range(2009, 2023) %} + SELECT + commune_code, + correction_status, + commune.surface as surface, + {{ year }} as year, + conso_{{ year }} as total, + conso_{{ year }}_activite as activite, + conso_{{ year }}_habitat as habitat, + conso_{{ year }}_mixte as mixte, + conso_{{ year }}_route as route, + conso_{{ year }}_ferroviaire as ferroviaire, + conso_{{ year }}_inconnu as inconnu + FROM {{ ref('consommation_cog_2024') }} as conso + LEFT JOIN {{ ref('commune') }} as commune + ON commune.code = conso.commune_code + {% if not loop.last %} + UNION + {% endif %} +{% endfor %} diff --git a/airflow/include/sql/sparte/models/sudocuh/scot.sql b/airflow/include/sql/sparte/models/sudocuh/scot.sql index 10549a98a..a1e368915 100644 --- a/airflow/include/sql/sparte/models/sudocuh/scot.sql +++ b/airflow/include/sql/sparte/models/sudocuh/scot.sql @@ -54,7 +54,8 @@ SELECT perimetre_en_cours_pcpopulationtotale, perimetre_en_cours_pcsuperficie, scot_geom.geom as geom, - scot_geom.srid_source as srid_source + scot_geom.srid_source as srid_source, + ST_Area(scot_geom.geom) as surface FROM {{ source('public', 'sudocuh_scot') }} as scot LEFT JOIN {{ ref('scot_geom') }} as scot_geom ON scot_geom.id_scot::text = scot.id_scot::text diff --git a/airflow/include/sql/sparte/tests/generic/has_all_communes.sql b/airflow/include/sql/sparte/tests/generic/has_all_communes.sql index 80791b1f1..8041015f5 100644 --- a/airflow/include/sql/sparte/tests/generic/has_all_communes.sql +++ b/airflow/include/sql/sparte/tests/generic/has_all_communes.sql @@ -4,7 +4,7 @@ with validation_errors as ( SELECT code from {{ ref('commune') }} WHERE code NOT IN ( - select {{ column_name }} + select distinct {{ column_name }} from {{ model }} ) diff --git a/airflow/include/sql/sparte/tests/generic/has_all_departements.sql b/airflow/include/sql/sparte/tests/generic/has_all_departements.sql index f7d90db58..c2f1764ed 100644 --- a/airflow/include/sql/sparte/tests/generic/has_all_departements.sql +++ b/airflow/include/sql/sparte/tests/generic/has_all_departements.sql @@ -4,7 +4,7 @@ with validation_errors as ( SELECT code from {{ ref('departement') }} WHERE code NOT IN ( - select {{ column_name }} + select distinct {{ column_name }} from {{ model }} ) diff --git a/airflow/include/sql/sparte/tests/generic/has_all_epcis.sql b/airflow/include/sql/sparte/tests/generic/has_all_epcis.sql index c380952b0..5eb405034 100644 --- a/airflow/include/sql/sparte/tests/generic/has_all_epcis.sql +++ b/airflow/include/sql/sparte/tests/generic/has_all_epcis.sql @@ -4,7 +4,7 @@ with validation_errors as ( SELECT code from {{ ref('epci') }} WHERE code NOT IN ( - select {{ column_name }} + select distinct {{ column_name }} from {{ model }} ) diff --git a/airflow/include/sql/sparte/tests/generic/has_all_regions.sql b/airflow/include/sql/sparte/tests/generic/has_all_regions.sql index 621bd7d29..6167b38e0 100644 --- a/airflow/include/sql/sparte/tests/generic/has_all_regions.sql +++ b/airflow/include/sql/sparte/tests/generic/has_all_regions.sql @@ -4,7 +4,7 @@ with validation_errors as ( SELECT code from {{ ref('region') }} WHERE code NOT IN ( - select {{ column_name }} + select distinct {{ column_name }} from {{ model }} ) diff --git a/airflow/include/sql/sparte/tests/generic/has_all_scots.sql b/airflow/include/sql/sparte/tests/generic/has_all_scots.sql index b42a7dc3d..877b95a1a 100644 --- a/airflow/include/sql/sparte/tests/generic/has_all_scots.sql +++ b/airflow/include/sql/sparte/tests/generic/has_all_scots.sql @@ -4,7 +4,7 @@ with validation_errors as ( SELECT id_scot from {{ ref('scot') }} WHERE id_scot NOT IN ( - select {{ column_name }} + select distinct {{ column_name }} from {{ model }} ) ) From f28f7b16a5b81ddc00aa1991b4e03a1919a06350 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 15:51:19 +0100 Subject: [PATCH 06/14] feat(landpop): add comparison level --- .../sparte/models/for_app/for_app_landpop.sql | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql b/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql index c2709b344..674f6807b 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql @@ -15,13 +15,20 @@ SELECT code_commune as land_id, 'COMMUNE' as land_type, + 'EPCI' as comparison_level, + commune.epci as comparison_id, {{ fields_to_query }} FROM - {{ ref('flux_population_commune') }} + {{ ref('flux_population_commune') }} as flux_population_commune +LEFT JOIN + {{ ref('commune') }} as commune + ON flux_population_commune.code_commune = commune.code UNION SELECT epci as land_id, 'EPCI' as land_type, + 'NATION' as comparison_level, + 'NATION' as comparison_id, {{ fields_to_query }} FROM {{ ref('flux_population_epci') }} @@ -29,13 +36,20 @@ UNION SELECT departement as land_id, 'DEPARTEMENT' as land_type, + 'REGION' as comparison_level, + departement.region as comparison_id, {{ fields_to_query }} FROM - {{ ref('flux_population_departement') }} + {{ ref('flux_population_departement') }} as flux_population_departement +LEFT JOIN + {{ ref('departement') }} as departement + ON departement.code = flux_population_departement.departement UNION SELECT region as land_id, 'REGION' as land_type, + 'NATION' as comparison_level, + 'NATION' as comparison_id, {{ fields_to_query }} FROM {{ ref('flux_population_region') }} @@ -43,6 +57,8 @@ UNION SELECT scot as land_id, 'SCOT' as land_type, + 'NATION' as comparison_level, + 'NATION' as comparison_id, {{ fields_to_query }} FROM {{ ref('flux_population_scot') }} From 4654b3fffb99966e733a1b1c05a0d4a1451925da Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 22:41:09 +0100 Subject: [PATCH 07/14] feat(conso/pop): add models to django --- airflow/dags/update_app.py | 78 ++ airflow/include/sql/sparte/dbt_project.yml | 10 + .../merge_flux_population_by_admin_level.sql | 2 +- .../majic/merge_majic_by_admin_level.sql | 2 +- .../models/for_app/for_app_landconso.sql | 83 +- ...cs.sql => for_app_landconsocomparison.sql} | 26 +- .../models/for_app/for_app_landconsostats.sql | 77 ++ .../sparte/models/for_app/for_app_landpop.sql | 71 +- ...tics.sql => for_app_landpopcomparison.sql} | 26 +- .../models/for_app/for_app_landpopstats.sql | 64 + .../flux/flux_population_commune.sql | 80 +- .../flux/flux_population_commune.yml | 41 - .../flux/flux_population_departement.sql | 16 +- .../aggregated/flux/flux_population_epci.sql | 16 +- .../flux/flux_population_region.sql | 16 +- .../aggregated/flux/flux_population_scot.sql | 26 +- .../period/period_flux_population_commune.sql | 37 + .../period/period_flux_population_commune.yml | 52 + .../period_flux_population_departement.sql | 3 + .../period_flux_population_departement.yml} | 2 +- .../period/period_flux_population_epci.sql | 3 + .../period_flux_population_epci.yml} | 2 +- .../period/period_flux_population_region.sql | 3 + .../period_flux_population_region.yml} | 2 +- .../period/period_flux_population_scot.sql | 8 + .../period_flux_population_scot.yml} | 2 +- .../period_stock_population_departement.sql} | 0 .../period_stock_population_departement.yml} | 2 +- .../period_stock_population_epci.sql} | 0 .../period_stock_population_epci.yml} | 2 +- .../period_stock_population_region.sql} | 0 .../period_stock_population_region.yml} | 2 +- .../period_stock_population_scot.sql} | 0 .../period_stock_population_scot.yml} | 2 +- .../calculated/pop_epci_stats_commune.sql | 2 +- .../calculated/pop_national_stats_epci.sql | 2 +- .../calculated/pop_national_stats_region.sql | 2 +- .../calculated/pop_national_stats_scot.sql | 2 +- .../pop_regional_stats_departement.sql | 2 +- .../sparte/models/insee/flux_population.sql | 57 - .../sparte/models/insee/flux_population.yml | 11 - .../flux/flux_consommation_commune.sql} | 0 .../flux/flux_consommation_departement.sql | 22 + .../flux/flux_consommation_epci.sql | 22 + .../flux/flux_consommation_region.sql | 22 + .../flux/flux_consommation_scot.sql | 27 + .../period/period_consommation_commune.sql} | 2 +- .../period_consommation_departement.sql} | 0 .../period/period_consommation_epci.sql} | 0 .../period/period_consommation_region.sql} | 0 .../period/period_consommation_scot.sql} | 0 .../calculated/conso_epci_stats_commune.sql | 2 +- .../calculated/conso_national_stats_epci.sql | 2 +- .../conso_national_stats_region.sql | 2 +- .../calculated/conso_national_stats_scot.sql | 2 +- .../conso_regional_stats_departement.sql | 2 +- .../sql/sparte/models/majic/schema.yml | 8 +- models.temp.py | 1100 +++++++++++++++++ public_data/models/__init__.py | 2 + public_data/models/consommation/LandConso.py | 24 + .../consommation/LandConsoComparison.py | 43 + .../models/consommation/LandConsoStats.py | 30 + public_data/models/consommation/__init__.py | 3 + public_data/models/demography/LandPop.py | 20 + .../models/demography/LandPopComparison.py | 19 + public_data/models/demography/LandPopStats.py | 18 + public_data/models/demography/__init__.py | 3 + utils/schema.py | 12 + 68 files changed, 1926 insertions(+), 295 deletions(-) rename airflow/include/sql/sparte/models/for_app/{for_app_landconsostatistics.sql => for_app_landconsocomparison.sql} (72%) create mode 100644 airflow/include/sql/sparte/models/for_app/for_app_landconsostats.sql rename airflow/include/sql/sparte/models/for_app/{for_app_landpopstatistics.sql => for_app_landpopcomparison.sql} (62%) create mode 100644 airflow/include/sql/sparte/models/for_app/for_app_landpopstats.sql create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.sql create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.yml create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.sql rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_departement.yml => period/period_flux_population_departement.yml} (83%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.sql rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_epci.yml => period/period_flux_population_epci.yml} (84%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.sql rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_region.yml => period/period_flux_population_region.yml} (83%) create mode 100644 airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.sql rename airflow/include/sql/sparte/models/insee/aggregated/flux/{flux_population_scot.yml => period/period_flux_population_scot.yml} (84%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_departement.sql => period/period_stock_population_departement.sql} (100%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_departement.yml => period/period_stock_population_departement.yml} (84%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_epci.sql => period/period_stock_population_epci.sql} (100%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_epci.yml => period/period_stock_population_epci.yml} (85%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_region.sql => period/period_stock_population_region.sql} (100%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_region.yml => period/period_stock_population_region.yml} (84%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_scot.sql => period/period_stock_population_scot.sql} (100%) rename airflow/include/sql/sparte/models/insee/aggregated/stock/{stock_population_scot.yml => period/period_stock_population_scot.yml} (85%) delete mode 100644 airflow/include/sql/sparte/models/insee/flux_population.sql delete mode 100644 airflow/include/sql/sparte/models/insee/flux_population.yml rename airflow/include/sql/sparte/models/majic/{consommation_en_lignes.sql => aggregated/flux/flux_consommation_commune.sql} (100%) create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_departement.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_epci.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_region.sql create mode 100644 airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_scot.sql rename airflow/include/sql/sparte/models/majic/aggregated/{consommation_commune.sql => flux/period/period_consommation_commune.sql} (95%) rename airflow/include/sql/sparte/models/majic/aggregated/{consommation_departement.sql => flux/period/period_consommation_departement.sql} (100%) rename airflow/include/sql/sparte/models/majic/aggregated/{consommation_epci.sql => flux/period/period_consommation_epci.sql} (100%) rename airflow/include/sql/sparte/models/majic/aggregated/{consommation_region.sql => flux/period/period_consommation_region.sql} (100%) rename airflow/include/sql/sparte/models/majic/aggregated/{consommation_scot.sql => flux/period/period_consommation_scot.sql} (100%) create mode 100644 models.temp.py create mode 100644 public_data/models/consommation/LandConso.py create mode 100644 public_data/models/consommation/LandConsoComparison.py create mode 100644 public_data/models/consommation/LandConsoStats.py create mode 100644 public_data/models/consommation/__init__.py create mode 100644 public_data/models/demography/LandPop.py create mode 100644 public_data/models/demography/LandPopComparison.py create mode 100644 public_data/models/demography/LandPopStats.py create mode 100644 public_data/models/demography/__init__.py diff --git a/airflow/dags/update_app.py b/airflow/dags/update_app.py index 3bfbd8894..4f6cb4803 100644 --- a/airflow/dags/update_app.py +++ b/airflow/dags/update_app.py @@ -106,6 +106,12 @@ def copy_table_from_dw_to_app( "copy_public_data_scot_departements", "copy_public_data_scot_regions", "copy_public_data_cerema", + "copy_public_data_landconso", + "copy_public_data_landconsocomparison", + "copy_public_data_landconsostats", + "copy_public_data_landpop", + "copy_public_data_landpopcomparison", + "copy_public_data_landpopstats", ], type="array", ), @@ -339,6 +345,72 @@ def copy_public_data_cerema(**context): ], ) + @task.python + def copy_public_data_landconso(**context): + return copy_table_from_dw_to_app( + from_table="public_for_app.for_app_landconso", + to_table="public.public_data_landconso", + environment=context["params"]["environment"], + btree_index_columns=[ + ["land_id", "land_type", "year"], + ], + ) + + @task.python + def copy_public_data_landconsocomparison(**context): + return copy_table_from_dw_to_app( + from_table="public_for_app.for_app_landconsocomparison", + to_table="public.public_data_landconsocomparison", + environment=context["params"]["environment"], + btree_index_columns=[ + ["land_id", "land_type", "from_year", "to_year"], + ], + ) + + @task.python + def copy_public_data_landconsostats(**context): + return copy_table_from_dw_to_app( + from_table="public_for_app.for_app_landconsostats", + to_table="public.public_data_landconsostats", + environment=context["params"]["environment"], + btree_index_columns=[ + ["land_id", "land_type", "from_year", "to_year"], + ], + ) + + @task.python + def copy_public_data_landpop(**context): + return copy_table_from_dw_to_app( + from_table="public_for_app.for_app_landpop", + to_table="public.public_data_landpop", + environment=context["params"]["environment"], + btree_index_columns=[ + ["land_id", "land_type", "year"], + ], + ) + + @task.python + def copy_public_data_landpopcomparison(**context): + return copy_table_from_dw_to_app( + from_table="public_for_app.for_app_landpopcomparison", + to_table="public.public_data_landpopcomparison", + environment=context["params"]["environment"], + btree_index_columns=[ + ["land_id", "land_type", "from_year", "to_year"], + ], + ) + + @task.python + def copy_public_data_landpopstats(**context): + return copy_table_from_dw_to_app( + from_table="public_for_app.for_app_landpopstats", + to_table="public.public_data_landpopstats", + environment=context["params"]["environment"], + btree_index_columns=[ + ["land_id", "land_type", "from_year", "to_year"], + ], + ) + @task.branch def copy_public_data_branch(**context): return context["params"]["tasks"] @@ -361,6 +433,12 @@ def copy_public_data_branch(**context): copy_public_data_scot_departements(), copy_public_data_scot_regions(), copy_public_data_cerema(), + copy_public_data_landconso(), + copy_public_data_landconsocomparison(), + copy_public_data_landconsostats(), + copy_public_data_landpop(), + copy_public_data_landpopcomparison(), + copy_public_data_landpopstats(), ] diff --git a/airflow/include/sql/sparte/dbt_project.yml b/airflow/include/sql/sparte/dbt_project.yml index cf23b2f78..df943b885 100644 --- a/airflow/include/sql/sparte/dbt_project.yml +++ b/airflow/include/sql/sparte/dbt_project.yml @@ -13,6 +13,16 @@ clean-targets: - "target" - "dbt_packages" +vars: + sparte: + for_app: + REGION: "REGION" + DEPARTEMENT: "DEPART" + SCOT: "SCOT" + EPCI: "EPCI" + COMMUNE: "COMM" + COMPOSITE: "COMP" + NATION: "NATION" models: sparte: diff --git a/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql b/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql index 519a277b5..1c56eff80 100644 --- a/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql +++ b/airflow/include/sql/sparte/macros/insee/merge_flux_population_by_admin_level.sql @@ -12,7 +12,7 @@ SELECT {{ sum_percent_median_avg('evolution', 'start_population') }}, sum(start_population) as start_population FROM - {{ ref('flux_population_commune') }} as population + {{ ref('period_flux_population_commune') }} as population LEFT JOIN {{ ref('commune') }} as commune ON commune.code = population.code_commune diff --git a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql index 531398332..d6216f43f 100644 --- a/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql +++ b/airflow/include/sql/sparte/macros/majic/merge_majic_by_admin_level.sql @@ -17,7 +17,7 @@ SELECT {{ sum_percent_median_avg('ferroviaire', 'commune.surface') }}, {{ sum_percent_median_avg('inconnu', 'commune.surface') }} FROM - {{ ref('consommation_commune') }} as consommation + {{ ref('period_consommation_commune') }} as consommation LEFT JOIN {{ ref('commune') }} as commune ON commune.code = consommation.commune_code diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql b/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql index 27b491e44..bf177fa5d 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_landconso.sql @@ -6,72 +6,49 @@ }} {% set fields_to_query = """ - from_year, - to_year, + surface, + year, total, - total_percent, activite, - activite_percent, habitat, - habitat_percent, mixte, - mixte_percent, route, - route_percent, ferroviaire, - ferroviaire_percent, - inconnu, - inconnu_percent + inconnu """ %} -SELECT - commune_code as land_id, - 'COMMUNE' as land_type, - 'EPCI' as comparison_level, - commune.epci as comparison_id, - {{ fields_to_query }} -FROM - {{ ref('consommation_commune') }} -LEFT JOIN - {{ ref('commune') }} as commune - ON commune_code = commune.code -UNION -SELECT - epci as land_id, - 'EPCI' as land_type, - 'NATION' as comparison_level, - 'NATION' as comparison_id, +select + commune_code as land_id, + '{{ var('COMMUNE') }}' as land_type, {{ fields_to_query }} -FROM - {{ ref('consommation_epci') }} -UNION -SELECT +from + {{ ref('flux_consommation_commune') }} +union +select departement as land_id, - 'DEPARTEMENT' as land_type, - 'REGION' as comparison_level, - departement.region as comparison_id, + '{{ var('DEPARTEMENT') }}' as land_type, {{ fields_to_query }} -FROM - {{ ref('consommation_departement') }} -LEFT JOIN - {{ ref('departement') }} as departement - ON departement.code = consommation_departement.departement -UNION -SELECT +from + {{ ref('flux_consommation_departement') }} +union +select region as land_id, - 'REGION' as land_type, - 'NATION' as comparison_level, - 'NATION' as comparison_id, + '{{ var('REGION') }}' as land_type, + {{ fields_to_query }} +from + {{ ref('flux_consommation_region') }} +union +select + epci as land_id, + '{{ var('EPCI') }}' as land_type, {{ fields_to_query }} -FROM - {{ ref('consommation_region') }} -UNION -SELECT +from + {{ ref('flux_consommation_epci') }} +union +select scot as land_id, - 'SCOT' as land_type, - 'NATION' as comparison_level, - 'NATION' as comparison_id, + '{{ var('SCOT') }}' as land_type, {{ fields_to_query }} -FROM - {{ ref('consommation_scot') }} +from + {{ ref('flux_consommation_scot') }} diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql b/airflow/include/sql/sparte/models/for_app/for_app_landconsocomparison.sql similarity index 72% rename from airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql rename to airflow/include/sql/sparte/models/for_app/for_app_landconsocomparison.sql index e1378a821..c826fc8fb 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_landconsostatistics.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_landconsocomparison.sql @@ -40,41 +40,41 @@ SELECT - 'COMMUNE' as relevance_level, - 'EPCI' as land_type, + '{{ var('COMMUNE') }}' as relevance_level, + '{{ var('EPCI') }}' as land_type, epci_stats_commune.epci as land_id, {{ fields_to_query }} FROM {{ ref('conso_epci_stats_commune') }} as epci_stats_commune UNION SELECT - 'DEPARTEMENT' as relevance_level, - 'REGION' as land_type, + '{{ var('DEPARTEMENT') }}' as relevance_level, + '{{ var('REGION') }}' as land_type, region_stats_departement.region as land_id, {{ fields_to_query }} FROM {{ ref('conso_regional_stats_departement') }} as region_stats_departement UNION SELECT - 'EPCI' as relevance_level, - 'NATION' as land_type, - 'NATION' as land_id, + '{{ var('EPCI') }}' as relevance_level, + '{{ var('NATION') }}' as land_type, + '{{ var('NATION') }}' as land_id, {{ fields_to_query }} FROM {{ ref('conso_national_stats_epci') }} as national_stats_epci UNION SELECT - 'REGION' as relevance_level, - 'NATION' as land_type, - 'NATION' as land_id, + '{{ var('REGION') }}' as relevance_level, + '{{ var('NATION') }}' as land_type, + '{{ var('NATION') }}' as land_id, {{ fields_to_query }} FROM {{ ref('conso_national_stats_region') }} as national_stats_region UNION SELECT - 'SCOT' as relevance_level, - 'NATION' as land_type, - 'NATION' as land_id, + '{{ var('SCOT') }}' as relevance_level, + '{{ var('NATION') }}' as land_type, + '{{ var('NATION') }}' as land_id, {{ fields_to_query }} FROM {{ ref('conso_national_stats_scot') }} as national_stats_scot diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landconsostats.sql b/airflow/include/sql/sparte/models/for_app/for_app_landconsostats.sql new file mode 100644 index 000000000..1d2adef97 --- /dev/null +++ b/airflow/include/sql/sparte/models/for_app/for_app_landconsostats.sql @@ -0,0 +1,77 @@ +{{ + config( + materialized='table', + docs={'node_color': 'purple'} + ) +}} + +{% set fields_to_query = """ + from_year, + to_year, + total, + total_percent, + activite, + activite_percent, + habitat, + habitat_percent, + mixte, + mixte_percent, + route, + route_percent, + ferroviaire, + ferroviaire_percent, + inconnu, + inconnu_percent +""" %} + +SELECT + commune_code as land_id, + '{{ var('COMMUNE') }}' as land_type, + '{{ var('EPCI') }}' as comparison_level, + commune.epci as comparison_id, + {{ fields_to_query }} + +FROM + {{ ref('period_consommation_commune') }} +LEFT JOIN + {{ ref('commune') }} as commune + ON commune_code = commune.code +UNION +SELECT + epci as land_id, + '{{ var('EPCI') }}' as land_type, + '{{ var('NATION') }}' as comparison_level, + '{{ var('NATION') }}' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_consommation_epci') }} +UNION +SELECT + departement as land_id, + '{{ var('DEPARTEMENT') }}' as land_type, + '{{ var('REGION') }}' as comparison_level, + departement.region as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_consommation_departement') }} as consommation_departement +LEFT JOIN + {{ ref('departement') }} as departement + ON departement.code = consommation_departement.departement +UNION +SELECT + region as land_id, + '{{ var('REGION') }}' as land_type, + '{{ var('NATION') }}' as comparison_level, + '{{ var('NATION') }}' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_consommation_region') }} +UNION +SELECT + scot as land_id, + '{{ var('SCOT') }}' as land_type, + '{{ var('NATION') }}' as comparison_level, + '{{ var('NATION') }}' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_consommation_scot') }} diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql b/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql index 674f6807b..d450e0c8d 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_landpop.sql @@ -6,59 +6,44 @@ }} {% set fields_to_query = """ - from_year, - to_year, + year, evolution, - evolution_percent + population, + source """ %} -SELECT + +select code_commune as land_id, - 'COMMUNE' as land_type, - 'EPCI' as comparison_level, - commune.epci as comparison_id, - {{ fields_to_query }} -FROM - {{ ref('flux_population_commune') }} as flux_population_commune -LEFT JOIN - {{ ref('commune') }} as commune - ON flux_population_commune.code_commune = commune.code -UNION -SELECT - epci as land_id, - 'EPCI' as land_type, - 'NATION' as comparison_level, - 'NATION' as comparison_id, + '{{ var('COMMUNE') }}' as land_type, {{ fields_to_query }} -FROM - {{ ref('flux_population_epci') }} -UNION -SELECT +from + {{ ref('flux_population_commune') }} +union +select departement as land_id, - 'DEPARTEMENT' as land_type, - 'REGION' as comparison_level, - departement.region as comparison_id, + '{{ var('DEPARTEMENT') }}' as land_type, {{ fields_to_query }} -FROM - {{ ref('flux_population_departement') }} as flux_population_departement -LEFT JOIN - {{ ref('departement') }} as departement - ON departement.code = flux_population_departement.departement -UNION -SELECT +from + {{ ref('flux_population_departement') }} +union +select region as land_id, - 'REGION' as land_type, - 'NATION' as comparison_level, - 'NATION' as comparison_id, + '{{ var('REGION') }}' as land_type, {{ fields_to_query }} -FROM +from {{ ref('flux_population_region') }} -UNION -SELECT +union +select + epci as land_id, + '{{ var('EPCI') }}' as land_type, + {{ fields_to_query }} +from + {{ ref('flux_population_epci') }} +union +select scot as land_id, - 'SCOT' as land_type, - 'NATION' as comparison_level, - 'NATION' as comparison_id, + '{{ var('SCOT') }}' as land_type, {{ fields_to_query }} -FROM +from {{ ref('flux_population_scot') }} diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql b/airflow/include/sql/sparte/models/for_app/for_app_landpopcomparison.sql similarity index 62% rename from airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql rename to airflow/include/sql/sparte/models/for_app/for_app_landpopcomparison.sql index 2aaf05cca..f444416c1 100644 --- a/airflow/include/sql/sparte/models/for_app/for_app_landpopstatistics.sql +++ b/airflow/include/sql/sparte/models/for_app/for_app_landpopcomparison.sql @@ -16,41 +16,41 @@ SELECT - 'COMMUNE' as relevance_level, - 'EPCI' as land_type, + '{{ var('COMMUNE') }}' as relevance_level, + '{{ var('EPCI') }}' as land_type, epci_stats_commune.epci as land_id, {{ fields_to_query }} FROM {{ ref('pop_epci_stats_commune') }} as epci_stats_commune UNION SELECT - 'DEPARTEMENT' as relevance_level, - 'REGION' as land_type, + '{{ var('DEPARTEMENT') }}' as relevance_level, + '{{ var('REGION') }}' as land_type, region_stats_departement.region as land_id, {{ fields_to_query }} FROM {{ ref('pop_regional_stats_departement') }} as region_stats_departement UNION SELECT - 'EPCI' as relevance_level, - 'NATION' as land_type, - 'NATION' as land_id, + '{{ var('EPCI') }}' as relevance_level, + '{{ var('NATION') }}' as land_type, + '{{ var('NATION') }}' as land_id, {{ fields_to_query }} FROM {{ ref('pop_national_stats_epci') }} as national_stats_epci UNION SELECT - 'REGION' as relevance_level, - 'NATION' as land_type, - 'NATION' as land_id, + '{{ var('REGION') }}' as relevance_level, + '{{ var('NATION') }}' as land_type, + '{{ var('NATION') }}' as land_id, {{ fields_to_query }} FROM {{ ref('pop_national_stats_region') }} as national_stats_region UNION SELECT - 'SCOT' as relevance_level, - 'NATION' as land_type, - 'NATION' as land_id, + '{{ var('SCOT') }}' as relevance_level, + '{{ var('NATION') }}' as land_type, + '{{ var('NATION') }}' as land_id, {{ fields_to_query }} FROM {{ ref('pop_national_stats_scot') }} as national_stats_scot diff --git a/airflow/include/sql/sparte/models/for_app/for_app_landpopstats.sql b/airflow/include/sql/sparte/models/for_app/for_app_landpopstats.sql new file mode 100644 index 000000000..22bf02f1a --- /dev/null +++ b/airflow/include/sql/sparte/models/for_app/for_app_landpopstats.sql @@ -0,0 +1,64 @@ +{{ + config( + materialized='table', + docs={'node_color': 'purple'} + ) +}} + +{% set fields_to_query = """ + from_year, + to_year, + evolution, + evolution_percent +""" %} + +SELECT + code_commune as land_id, + '{{ var('COMMUNE') }}' as land_type, + '{{ var('EPCI') }}' as comparison_level, + commune.epci as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_flux_population_commune') }} as flux_population_commune +LEFT JOIN + {{ ref('commune') }} as commune + ON flux_population_commune.code_commune = commune.code +UNION +SELECT + epci as land_id, + '{{ var('EPCI') }}' as land_type, + '{{ var('NATION') }}' as comparison_level, + '{{ var('NATION') }}' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_flux_population_epci') }} +UNION +SELECT + departement as land_id, + '{{ var('DEPARTEMENT') }}' as land_type, + '{{ var('REGION') }}' as comparison_level, + departement.region as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_flux_population_departement') }} as flux_population_departement +LEFT JOIN + {{ ref('departement') }} as departement + ON departement.code = flux_population_departement.departement +UNION +SELECT + region as land_id, + '{{ var('REGION') }}' as land_type, + '{{ var('NATION') }}' as comparison_level, + '{{ var('NATION') }}' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_flux_population_region') }} +UNION +SELECT + scot as land_id, + '{{ var('SCOT') }}' as land_type, + '{{ var('NATION') }}' as comparison_level, + '{{ var('NATION') }}' as comparison_id, + {{ fields_to_query }} +FROM + {{ ref('period_flux_population_scot') }} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql index 415fa7253..c400236ce 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.sql @@ -1,37 +1,57 @@ -{{ config(materialized='table') }} - -{% for to_year in range(2010, 2023) %} - {% for from_year in range(2009, to_year) %} - {% if from_year >= to_year %} - {% break %} - {% endif %} +{{ + config( + materialized='table', + indexes=[{'columns': ['code_commune'], 'type': 'btree'}] + ) +}} +with known_years as ( + {% for year in range(2009, 2021) %} + {% set next_year = year + 1 %} SELECT - flux_population.code_commune, - {{ from_year }} as from_year, - {{ to_year }} as to_year, - {{ sum_percent( - 'evolution', - 'stock_population.population_' + from_year|string - ) }}, - max(stock_population.population_{{ from_year }}) as start_population + code_commune, + {{ year }} as year, + (population_{{ next_year }} - population_{{ year }}) as evolution, + population_{{ year }} as population, + 'INSEE' as source FROM - {{ ref('flux_population') }} as flux_population - LEFT JOIN - {{ ref('commune') }} as commune - ON commune.code = flux_population.code_commune - LEFT JOIN - {{ ref('population_cog_2024') }} as stock_population - ON stock_population.code_commune = flux_population.code_commune - WHERE year BETWEEN {{ from_year }} AND {{ to_year }} - GROUP BY - flux_population.code_commune, - from_year, - to_year + {{ ref('population_cog_2024') }} {% if not loop.last %} UNION {% endif %} {% endfor %} +), average_evolution as ( + SELECT + code_commune, + AVG(evolution) as average_evolution + FROM + known_years + GROUP BY + code_commune +), predictions as ( + {% set first_unknown_year = 2021 %} + {% for year in range(first_unknown_year, 2023) %} + {% set next_year = year + 1 %} + SELECT + code_commune, + {{ year }} as year, + ( + SELECT round(average_evolution)::numeric + FROM average_evolution + WHERE code_commune = pop.code_commune + ) as evolution, + population_{{ first_unknown_year }} + ( + SELECT round(average_evolution)::numeric + FROM average_evolution + WHERE code_commune = pop.code_commune + ) * ({{ year }} - 2020) as population, + 'PROJECTION' as source + FROM + {{ ref('population_cog_2024') }} as pop {% if not loop.last %} - UNION - {% endif %} -{% endfor %} + UNION + {% endif %} + {% endfor %} +) +SELECT * FROM known_years +UNION +SELECT * FROM predictions diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml index c7edc00f2..fe3cd7b35 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_commune.yml @@ -1,23 +1,5 @@ version: 2 -accepted_years: &accepted_years - values : [ - 2009, - 2010, - 2011, - 2012, - 2013, - 2014, - 2015, - 2016, - 2017, - 2018, - 2019, - 2020, - 2021, - 2022 - ] - models: - name: flux_population_commune columns: @@ -27,26 +9,3 @@ models: - relationships: to: ref('commune') field: code - - name: from_year - data_tests: - - not_null - - accepted_values: *accepted_years - - name: to_year - data_tests: - - not_null - - accepted_values: *accepted_years - - name: evolution - data_tests: - - not_null - - name: evolution_percent - data_tests: - - not_null - - name: evolution_median - data_tests: - - not_null - - name: evolution_avg - data_tests: - - not_null - - name: start_population - data_tests: - - not_null diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.sql index 6c96b2415..061f1d011 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.sql +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.sql @@ -1,3 +1,17 @@ {{ config(materialized='table') }} -{{ merge_flux_population_by_admin_level('departement') }} +select + commune.departement, + year as year, + sum(evolution) as evolution, + sum(flux_population.population) as population, + max(source) as source +FROM + {{ ref('flux_population_commune') }} as flux_population +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_population.code_commune +GROUP BY + commune.departement, + year diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.sql index 2e79f8f26..64255dca1 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.sql +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.sql @@ -1,3 +1,17 @@ {{ config(materialized='table') }} -{{ merge_flux_population_by_admin_level('epci') }} +select + commune.epci, + year as year, + sum(evolution) as evolution, + sum(flux_population.population) as population, + max(source) as source +FROM + {{ ref('flux_population_commune') }} as flux_population +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_population.code_commune +GROUP BY + commune.epci, + year diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.sql index bdffdbfd1..26e4621bb 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.sql +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.sql @@ -1,3 +1,17 @@ {{ config(materialized='table') }} -{{ merge_flux_population_by_admin_level('region') }} +select + commune.region, + year as year, + sum(evolution) as evolution, + sum(flux_population.population) as population, + max(source) as source +FROM + {{ ref('flux_population_commune') }} as flux_population +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_population.code_commune +GROUP BY + commune.region, + year diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.sql index 8e6862a95..32844a135 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.sql +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.sql @@ -1,8 +1,22 @@ {{ config(materialized='table') }} -{{ - merge_flux_population_by_admin_level( - 'id_scot', - 'scot' - ) -}} +select + scot.id_scot as scot, + year as year, + sum(evolution) as evolution, + sum(flux_population.population) as population, + max(source) as source +FROM + {{ ref('flux_population_commune') }} as flux_population +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_population.code_commune +LEFT JOIN + {{ ref('scot_communes') }} as scot +ON + commune.code = scot.commune_code + +GROUP BY + scot.id_scot, + year diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.sql new file mode 100644 index 000000000..607594bc2 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.sql @@ -0,0 +1,37 @@ +{{ config(materialized='table') }} + +{% for to_year in range(2010, 2023) %} + {% for from_year in range(2009, to_year) %} + {% if from_year >= to_year %} + {% break %} + {% endif %} + SELECT + flux_population.code_commune, + {{ from_year }} as from_year, + {{ to_year }} as to_year, + {{ sum_percent( + 'evolution', + 'stock_population.population_' + from_year|string + ) }}, + max(stock_population.population_{{ from_year }}) as start_population + FROM + {{ ref('flux_population_commune') }} as flux_population + LEFT JOIN + {{ ref('commune') }} as commune + ON commune.code = flux_population.code_commune + LEFT JOIN + {{ ref('population_cog_2024') }} as stock_population + ON stock_population.code_commune = flux_population.code_commune + WHERE year BETWEEN {{ from_year }} AND {{ to_year }} + GROUP BY + flux_population.code_commune, + from_year, + to_year + {% if not loop.last %} + UNION + {% endif %} + {% endfor %} + {% if not loop.last %} + UNION + {% endif %} +{% endfor %} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.yml new file mode 100644 index 000000000..beaf69eab --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_commune.yml @@ -0,0 +1,52 @@ +version: 2 + +accepted_years: &accepted_years + values : [ + 2009, + 2010, + 2011, + 2012, + 2013, + 2014, + 2015, + 2016, + 2017, + 2018, + 2019, + 2020, + 2021, + 2022 + ] + +models: + - name: period_flux_population_commune + columns: + - name: code_commune + data_tests: + - has_all_communes + - relationships: + to: ref('commune') + field: code + - name: from_year + data_tests: + - not_null + - accepted_values: *accepted_years + - name: to_year + data_tests: + - not_null + - accepted_values: *accepted_years + - name: evolution + data_tests: + - not_null + - name: evolution_percent + data_tests: + - not_null + - name: evolution_median + data_tests: + - not_null + - name: evolution_avg + data_tests: + - not_null + - name: start_population + data_tests: + - not_null diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.sql new file mode 100644 index 000000000..6c96b2415 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.sql @@ -0,0 +1,3 @@ +{{ config(materialized='table') }} + +{{ merge_flux_population_by_admin_level('departement') }} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.yml similarity index 83% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml rename to airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.yml index 0fb786704..7fe09acab 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_departement.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_departement.yml @@ -1,6 +1,6 @@ version: 2 models: - - name: flux_population_departement + - name: period_flux_population_departement columns: - name: departement data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.sql new file mode 100644 index 000000000..2e79f8f26 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.sql @@ -0,0 +1,3 @@ +{{ config(materialized='table') }} + +{{ merge_flux_population_by_admin_level('epci') }} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.yml similarity index 84% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml rename to airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.yml index 9a3c1ac39..ff466c0e2 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_epci.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_epci.yml @@ -1,6 +1,6 @@ version: 2 models: - - name: flux_population_epci + - name: period_flux_population_epci columns: - name: epci data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.sql new file mode 100644 index 000000000..bdffdbfd1 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.sql @@ -0,0 +1,3 @@ +{{ config(materialized='table') }} + +{{ merge_flux_population_by_admin_level('region') }} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.yml similarity index 83% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml rename to airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.yml index 4a6608b0d..354564862 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_region.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_region.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: flux_population_region + - name: period_flux_population_region columns: - name: region data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.sql b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.sql new file mode 100644 index 000000000..8e6862a95 --- /dev/null +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} + +{{ + merge_flux_population_by_admin_level( + 'id_scot', + 'scot' + ) +}} diff --git a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.yml similarity index 84% rename from airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml rename to airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.yml index dcc273da6..72fcd57f6 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/flux/flux_population_scot.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/flux/period/period_flux_population_scot.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: flux_population_scot + - name: period_flux_population_scot columns: - name: scot data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_departement.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_departement.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_departement.yml similarity index 84% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_departement.yml index e834717b3..7200510ae 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_departement.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_departement.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: stock_population_departement + - name: period_stock_population_departement columns: - name: departement data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_epci.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_epci.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_epci.yml similarity index 85% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_epci.yml index e5e2fc4ed..472a15118 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_epci.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_epci.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: stock_population_epci + - name: period_stock_population_epci columns: - name: epci data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_region.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_region.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_region.yml similarity index 84% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_region.yml index d85d3d1aa..270371bcc 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_region.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_region.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: stock_population_region + - name: period_stock_population_region columns: - name: region data_tests: diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.sql b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_scot.sql similarity index 100% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.sql rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_scot.sql diff --git a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_scot.yml similarity index 85% rename from airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml rename to airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_scot.yml index 45d0a7440..140a300a5 100644 --- a/airflow/include/sql/sparte/models/insee/aggregated/stock/stock_population_scot.yml +++ b/airflow/include/sql/sparte/models/insee/aggregated/stock/period/period_stock_population_scot.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: stock_population_scot + - name: period_stock_population_scot columns: - name: scot data_tests: diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql index f7d6b725d..9431eeb01 100644 --- a/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_epci_stats_commune.sql @@ -6,7 +6,7 @@ SELECT to_year, {{ sum_percent_median_avg('evolution', 'start_population') }} FROM - {{ ref('flux_population_commune') }} as pop + {{ ref('period_flux_population_commune') }} as pop LEFT JOIN {{ ref('commune') }} as commune ON commune.code = pop.code_commune diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql index aea8946d8..a3af85e78 100644 --- a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_epci.sql @@ -5,7 +5,7 @@ SELECT to_year, {{ sum_percent_median_avg('evolution', 'start_population') }} FROM - {{ ref('flux_population_epci') }} as pop + {{ ref('period_flux_population_epci') }} as pop GROUP BY from_year, to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql index caea0a3d6..8a9aa2ed1 100644 --- a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_region.sql @@ -5,7 +5,7 @@ SELECT to_year, {{ sum_percent_median_avg('evolution', 'start_population') }} FROM - {{ ref('flux_population_region') }} as pop + {{ ref('period_flux_population_region') }} as pop GROUP BY from_year, to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql index ba36a7ee4..010cf131d 100644 --- a/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_national_stats_scot.sql @@ -5,7 +5,7 @@ SELECT to_year, {{ sum_percent_median_avg('evolution', 'start_population') }} FROM - {{ ref('flux_population_scot') }} as pop + {{ ref('period_flux_population_scot') }} as pop GROUP BY from_year, to_year diff --git a/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql b/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql index 14b737fcc..8297fa394 100644 --- a/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql +++ b/airflow/include/sql/sparte/models/insee/calculated/pop_regional_stats_departement.sql @@ -6,7 +6,7 @@ SELECT to_year, {{ sum_percent_median_avg('evolution', 'start_population') }} FROM - {{ ref('flux_population_departement') }} as pop + {{ ref('period_flux_population_departement') }} as pop LEFT JOIN {{ ref('commune') }} as commune ON commune.departement = pop.departement diff --git a/airflow/include/sql/sparte/models/insee/flux_population.sql b/airflow/include/sql/sparte/models/insee/flux_population.sql deleted file mode 100644 index c400236ce..000000000 --- a/airflow/include/sql/sparte/models/insee/flux_population.sql +++ /dev/null @@ -1,57 +0,0 @@ -{{ - config( - materialized='table', - indexes=[{'columns': ['code_commune'], 'type': 'btree'}] - ) -}} -with known_years as ( - {% for year in range(2009, 2021) %} - {% set next_year = year + 1 %} - SELECT - code_commune, - {{ year }} as year, - (population_{{ next_year }} - population_{{ year }}) as evolution, - population_{{ year }} as population, - 'INSEE' as source - FROM - {{ ref('population_cog_2024') }} - {% if not loop.last %} - UNION - {% endif %} - {% endfor %} -), average_evolution as ( - SELECT - code_commune, - AVG(evolution) as average_evolution - FROM - known_years - GROUP BY - code_commune -), predictions as ( - {% set first_unknown_year = 2021 %} - {% for year in range(first_unknown_year, 2023) %} - {% set next_year = year + 1 %} - SELECT - code_commune, - {{ year }} as year, - ( - SELECT round(average_evolution)::numeric - FROM average_evolution - WHERE code_commune = pop.code_commune - ) as evolution, - population_{{ first_unknown_year }} + ( - SELECT round(average_evolution)::numeric - FROM average_evolution - WHERE code_commune = pop.code_commune - ) * ({{ year }} - 2020) as population, - 'PROJECTION' as source - FROM - {{ ref('population_cog_2024') }} as pop - {% if not loop.last %} - UNION - {% endif %} - {% endfor %} -) -SELECT * FROM known_years -UNION -SELECT * FROM predictions diff --git a/airflow/include/sql/sparte/models/insee/flux_population.yml b/airflow/include/sql/sparte/models/insee/flux_population.yml deleted file mode 100644 index ef21beb1b..000000000 --- a/airflow/include/sql/sparte/models/insee/flux_population.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -models: - - name: flux_population - columns: - - name: code_commune - data_tests: - - has_all_communes - - relationships: - to: ref('commune') - field: code diff --git a/airflow/include/sql/sparte/models/majic/consommation_en_lignes.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_commune.sql similarity index 100% rename from airflow/include/sql/sparte/models/majic/consommation_en_lignes.sql rename to airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_commune.sql diff --git a/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_departement.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_departement.sql new file mode 100644 index 000000000..0c9e88eb0 --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_departement.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +select + commune.departement, + sum(commune.surface) as surface, + year, + sum(total) as total, + sum(activite) as activite, + sum(habitat) as habitat, + sum(mixte) as mixte, + sum(route) as route, + sum(ferroviaire) as ferroviaire, + sum(inconnu) as inconnu +FROM + {{ ref('flux_consommation_commune') }} as flux_consommation +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_consommation.commune_code +GROUP BY + commune.departement, + year diff --git a/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_epci.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_epci.sql new file mode 100644 index 000000000..5bf1b765f --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_epci.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +select + commune.epci, + sum(commune.surface) as surface, + year, + sum(total) as total, + sum(activite) as activite, + sum(habitat) as habitat, + sum(mixte) as mixte, + sum(route) as route, + sum(ferroviaire) as ferroviaire, + sum(inconnu) as inconnu +FROM + {{ ref('flux_consommation_commune') }} as flux_consommation +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_consommation.commune_code +GROUP BY + commune.epci, + year diff --git a/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_region.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_region.sql new file mode 100644 index 000000000..81d29d6bb --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_region.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +select + commune.region, + sum(commune.surface) as surface, + year, + sum(total) as total, + sum(activite) as activite, + sum(habitat) as habitat, + sum(mixte) as mixte, + sum(route) as route, + sum(ferroviaire) as ferroviaire, + sum(inconnu) as inconnu +FROM + {{ ref('flux_consommation_commune') }} as flux_consommation +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_consommation.commune_code +GROUP BY + commune.region, + year diff --git a/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_scot.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_scot.sql new file mode 100644 index 000000000..ebd43bb4a --- /dev/null +++ b/airflow/include/sql/sparte/models/majic/aggregated/flux/flux_consommation_scot.sql @@ -0,0 +1,27 @@ +{{ config(materialized='table') }} + +select + scot.id_scot as scot, + sum(commune.surface) as surface, + year, + sum(total) as total, + sum(activite) as activite, + sum(habitat) as habitat, + sum(mixte) as mixte, + sum(route) as route, + sum(ferroviaire) as ferroviaire, + sum(inconnu) as inconnu +FROM + {{ ref('flux_consommation_commune') }} as flux_consommation +LEFT JOIN + {{ ref('commune') }} as commune +ON + commune.code = flux_consommation.commune_code +LEFT JOIN + {{ ref('scot_communes') }} as scot +ON + commune.code = scot.commune_code + +GROUP BY + scot.id_scot, + year diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_commune.sql similarity index 95% rename from airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql rename to airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_commune.sql index 742be835e..c41d8fb47 100644 --- a/airflow/include/sql/sparte/models/majic/aggregated/consommation_commune.sql +++ b/airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_commune.sql @@ -18,7 +18,7 @@ {{ sum_percent('ferroviaire', 'commune.surface') }}, {{ sum_percent('inconnu', 'commune.surface') }} - FROM {{ ref('consommation_en_lignes') }} as conso + FROM {{ ref('flux_consommation_commune') }} as conso LEFT JOIN {{ ref('commune') }} as commune ON commune.code = conso.commune_code diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_departement.sql similarity index 100% rename from airflow/include/sql/sparte/models/majic/aggregated/consommation_departement.sql rename to airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_departement.sql diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_epci.sql similarity index 100% rename from airflow/include/sql/sparte/models/majic/aggregated/consommation_epci.sql rename to airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_epci.sql diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_region.sql similarity index 100% rename from airflow/include/sql/sparte/models/majic/aggregated/consommation_region.sql rename to airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_region.sql diff --git a/airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql b/airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_scot.sql similarity index 100% rename from airflow/include/sql/sparte/models/majic/aggregated/consommation_scot.sql rename to airflow/include/sql/sparte/models/majic/aggregated/flux/period/period_consommation_scot.sql diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql index b23993d24..1e65cf94c 100644 --- a/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_epci_stats_commune.sql @@ -12,7 +12,7 @@ SELECT {{ sum_percent_median_avg('ferroviaire', 'commune.surface') }}, {{ sum_percent_median_avg('inconnu', 'commune.surface') }} FROM - {{ ref('consommation_commune') }} as conso + {{ ref('period_consommation_commune') }} as conso LEFT JOIN {{ ref('commune') }} as commune ON commune.code = conso.commune_code diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql index eddcdf614..191ce1f07 100644 --- a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_epci.sql @@ -12,7 +12,7 @@ SELECT {{ sum_percent_median_avg('ferroviaire', 'epci.surface') }}, {{ sum_percent_median_avg('inconnu', 'epci.surface') }} FROM - {{ ref('consommation_epci') }} as conso + {{ ref('period_consommation_epci') }} as conso LEFT JOIN {{ ref('epci') }} as epci ON epci.code = conso.epci diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql index bc6a090d0..0780ae75b 100644 --- a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_region.sql @@ -11,7 +11,7 @@ SELECT {{ sum_percent_median_avg('ferroviaire', 'region.surface') }}, {{ sum_percent_median_avg('inconnu', 'region.surface') }} FROM - {{ ref('consommation_region') }} as conso + {{ ref('period_consommation_region') }} as conso LEFT JOIN {{ ref('region') }} as region ON region.code = conso.region diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql index 897b6fe81..d282c4d3a 100644 --- a/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_national_stats_scot.sql @@ -11,7 +11,7 @@ SELECT {{ sum_percent_median_avg('ferroviaire', 'scot.surface') }}, {{ sum_percent_median_avg('inconnu', 'scot.surface') }} FROM - {{ ref('consommation_scot') }} as conso + {{ ref('period_consommation_scot') }} as conso LEFT JOIN {{ ref('scot') }} as scot ON scot.id_scot = conso.scot diff --git a/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql b/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql index 06379f392..571e0b10c 100644 --- a/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql +++ b/airflow/include/sql/sparte/models/majic/calculated/conso_regional_stats_departement.sql @@ -12,7 +12,7 @@ SELECT {{ sum_percent_median_avg('ferroviaire', 'departement.surface') }}, {{ sum_percent_median_avg('inconnu', 'departement.surface') }} FROM - {{ ref('consommation_departement') }} as conso + {{ ref('period_consommation_departement') }} as conso LEFT JOIN {{ ref('departement') }} as departement ON departement.code = conso.departement diff --git a/airflow/include/sql/sparte/models/majic/schema.yml b/airflow/include/sql/sparte/models/majic/schema.yml index 1492841c7..ce8b60065 100644 --- a/airflow/include/sql/sparte/models/majic/schema.yml +++ b/airflow/include/sql/sparte/models/majic/schema.yml @@ -29,7 +29,7 @@ models: - relationships: to: ref('commune') field: code - - name: consommation_departement + - name: period_consommation_departement columns: - name: departement data_tests: @@ -39,7 +39,7 @@ models: - relationships: to: ref('departement') field: code - - name: consommation_epci + - name: period_consommation_epci columns: - name: epci data_tests: @@ -49,7 +49,7 @@ models: - relationships: to: ref('epci') field: code - - name: commation_region + - name: period_consommation_region columns: - name: region data_tests: @@ -59,7 +59,7 @@ models: - relationships: to: ref('region') field: code - - name: consommation_scot + - name: period_consommation_scot columns: - name: scot data_tests: diff --git a/models.temp.py b/models.temp.py new file mode 100644 index 000000000..cce9708cc --- /dev/null +++ b/models.temp.py @@ -0,0 +1,1100 @@ +# This is an auto-generated Django model module. +# You'll have to do the following manually to clean this up: +# * Rearrange models' order +# * Make sure each model has one field with primary_key=True +# * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior +# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table +# Feel free to rename the models, but don't rename db_table values or field names. +from django.contrib.gis.db import models + + +class AppParameterParameter(models.Model): + id = models.BigAutoField(primary_key=True) + name = models.CharField(max_length=100) + slug = models.CharField(unique=True, max_length=40) + value_type = models.CharField(max_length=3) + description = models.TextField() + value = models.CharField(max_length=250) + + class Meta: + managed = False + db_table = "app_parameter_parameter" + + +class AuthGroup(models.Model): + name = models.CharField(unique=True, max_length=150) + + class Meta: + managed = False + db_table = "auth_group" + + +class AuthGroupPermissions(models.Model): + id = models.BigAutoField(primary_key=True) + group = models.ForeignKey(AuthGroup, models.DO_NOTHING) + permission = models.ForeignKey("AuthPermission", models.DO_NOTHING) + + class Meta: + managed = False + db_table = "auth_group_permissions" + unique_together = (("group", "permission"),) + + +class AuthPermission(models.Model): + name = models.CharField(max_length=255) + content_type = models.ForeignKey("DjangoContentType", models.DO_NOTHING) + codename = models.CharField(max_length=100) + + class Meta: + managed = False + db_table = "auth_permission" + unique_together = (("content_type", "codename"),) + + +class CrispCrispwebhooknotification(models.Model): + id = models.BigAutoField(primary_key=True) + event = models.CharField(max_length=255) + timestamp = models.DateTimeField() + data = models.JSONField() + + class Meta: + managed = False + db_table = "crisp_crispwebhooknotification" + + +class DiagnosticWordWordtemplate(models.Model): + slug = models.CharField(primary_key=True, max_length=50) + description = models.TextField() + docx = models.CharField(max_length=100) + last_update = models.DateTimeField() + filename_mask = models.CharField(max_length=255) + + class Meta: + managed = False + db_table = "diagnostic_word_wordtemplate" + + +class DjangoAdminLog(models.Model): + action_time = models.DateTimeField() + object_id = models.TextField(blank=True, null=True) + object_repr = models.CharField(max_length=200) + action_flag = models.SmallIntegerField() + change_message = models.TextField() + content_type = models.ForeignKey("DjangoContentType", models.DO_NOTHING, blank=True, null=True) + user = models.ForeignKey("UsersUser", models.DO_NOTHING) + + class Meta: + managed = False + db_table = "django_admin_log" + + +class DjangoAppParameterParameter(models.Model): + id = models.BigAutoField(primary_key=True) + name = models.CharField(max_length=100) + slug = models.CharField(unique=True, max_length=40) + value_type = models.CharField(max_length=3) + description = models.TextField() + value = models.CharField(max_length=250) + is_global = models.BooleanField() + + class Meta: + managed = False + db_table = "django_app_parameter_parameter" + + +class DjangoCeleryResultsChordcounter(models.Model): + group_id = models.CharField(unique=True, max_length=255) + sub_tasks = models.TextField() + count = models.IntegerField() + + class Meta: + managed = False + db_table = "django_celery_results_chordcounter" + + +class DjangoCeleryResultsGroupresult(models.Model): + group_id = models.CharField(unique=True, max_length=255) + date_created = models.DateTimeField() + date_done = models.DateTimeField() + content_type = models.CharField(max_length=128) + content_encoding = models.CharField(max_length=64) + result = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = "django_celery_results_groupresult" + + +class DjangoCeleryResultsTaskresult(models.Model): + task_id = models.CharField(unique=True, max_length=255) + status = models.CharField(max_length=50) + content_type = models.CharField(max_length=128) + content_encoding = models.CharField(max_length=64) + result = models.TextField(blank=True, null=True) + date_done = models.DateTimeField() + traceback = models.TextField(blank=True, null=True) + meta = models.TextField(blank=True, null=True) + task_args = models.TextField(blank=True, null=True) + task_kwargs = models.TextField(blank=True, null=True) + task_name = models.CharField(max_length=255, blank=True, null=True) + worker = models.CharField(max_length=100, blank=True, null=True) + date_created = models.DateTimeField() + + class Meta: + managed = False + db_table = "django_celery_results_taskresult" + + +class DjangoContentType(models.Model): + app_label = models.CharField(max_length=100) + model = models.CharField(max_length=100) + + class Meta: + managed = False + db_table = "django_content_type" + unique_together = (("app_label", "model"),) + + +class DjangoMigrations(models.Model): + id = models.BigAutoField(primary_key=True) + app = models.CharField(max_length=255) + name = models.CharField(max_length=255) + applied = models.DateTimeField() + + class Meta: + managed = False + db_table = "django_migrations" + + +class DjangoSession(models.Model): + session_key = models.CharField(primary_key=True, max_length=40) + session_data = models.TextField() + expire_date = models.DateTimeField() + + class Meta: + managed = False + db_table = "django_session" + + +class DsfrDsfrconfig(models.Model): + id = models.BigAutoField(primary_key=True) + header_brand = models.CharField(max_length=200) + header_brand_html = models.CharField(max_length=200) + footer_brand = models.CharField(max_length=200) + footer_brand_html = models.CharField(max_length=200) + site_title = models.CharField(max_length=200) + site_tagline = models.CharField(max_length=200) + footer_description = models.TextField() + mourning = models.BooleanField() + accessibility_status = models.CharField(max_length=4) + + class Meta: + managed = False + db_table = "dsfr_dsfrconfig" + + +class HomeAlivetimestamp(models.Model): + id = models.BigAutoField() + timestamp = models.DateTimeField() + queue_name = models.CharField(max_length=20) + + class Meta: + managed = False + db_table = "home_alivetimestamp" + + +class HomeContactform(models.Model): + id = models.BigAutoField(primary_key=True) + email = models.CharField(max_length=254) + content = models.TextField() + status = models.CharField(max_length=10) + created_date = models.DateTimeField() + processed_date = models.DateTimeField(blank=True, null=True) + error = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = "home_contactform" + + +class HomeNewsletter(models.Model): + id = models.BigAutoField(primary_key=True) + email = models.CharField(max_length=254) + created_date = models.DateTimeField() + confirm_token = models.CharField(unique=True, max_length=25) + confirmation_date = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = "home_newsletter" + + +class MetabaseStatdiagnostic(models.Model): + id = models.BigAutoField(primary_key=True) + created_date = models.DateTimeField() + is_anonymouse = models.BooleanField() + is_public = models.BooleanField() + administrative_level = models.CharField(max_length=255, blank=True, null=True) + analysis_level = models.CharField(max_length=255) + start_date = models.DateField() + end_date = models.DateField() + link = models.CharField(max_length=255) + city = models.CharField(max_length=255, blank=True, null=True) + epci = models.CharField(max_length=255, blank=True, null=True) + scot = models.CharField(max_length=255, blank=True, null=True) + departement = models.CharField(max_length=255, blank=True, null=True) + region = models.CharField(max_length=255, blank=True, null=True) + project = models.OneToOneField("ProjectProject", models.DO_NOTHING) + date_first_download = models.DateTimeField(blank=True, null=True) + is_downaloaded = models.BooleanField() + organism = models.CharField(max_length=255, blank=True, null=True) + request = models.ForeignKey("ProjectRequest", models.DO_NOTHING, blank=True, null=True) + group_organism = models.CharField(max_length=50, blank=True, null=True) + + class Meta: + managed = False + db_table = "metabase_statdiagnostic" + + +class ProjectEmprise(models.Model): + id = models.BigAutoField(primary_key=True) + mpoly = models.MultiPolygonField() + project = models.ForeignKey("ProjectProject", models.DO_NOTHING) + srid_source = models.IntegerField() + + class Meta: + managed = False + db_table = "project_emprise" + + +class ProjectErrortracking(models.Model): + id = models.BigAutoField(primary_key=True) + created_date = models.DateTimeField() + exception = models.TextField() + request = models.ForeignKey("ProjectRequest", models.DO_NOTHING) + + class Meta: + managed = False + db_table = "project_errortracking" + + +class ProjectHistoricalproject(models.Model): + id = models.BigIntegerField() + name = models.CharField(max_length=100) + is_public = models.BooleanField() + analyse_start_date = models.CharField(max_length=4) + analyse_end_date = models.CharField(max_length=4) + level = models.CharField(max_length=7) + land_type = models.CharField(max_length=7, blank=True, null=True) + land_id = models.CharField(max_length=255, blank=True, null=True) + look_a_like = models.CharField(max_length=250, blank=True, null=True) + target_2031 = models.IntegerField() + created_date = models.DateTimeField() + updated_date = models.DateTimeField() + first_year_ocsge = models.IntegerField(blank=True, null=True) + last_year_ocsge = models.IntegerField(blank=True, null=True) + folder_name = models.CharField(max_length=15, blank=True, null=True) + territory_name = models.CharField(max_length=250, blank=True, null=True) + cover_image = models.TextField(blank=True, null=True) + theme_map_conso = models.TextField(blank=True, null=True) + theme_map_artif = models.TextField(blank=True, null=True) + theme_map_understand_artif = models.TextField(blank=True, null=True) + async_add_city_done = models.BooleanField() + async_cover_image_done = models.BooleanField() + async_find_first_and_last_ocsge_done = models.BooleanField() + async_add_comparison_lands_done = models.BooleanField() + async_generate_theme_map_conso_done = models.BooleanField() + async_generate_theme_map_artif_done = models.BooleanField() + async_theme_map_understand_artif_done = models.BooleanField() + history_id = models.AutoField(primary_key=True) + history_date = models.DateTimeField() + history_change_reason = models.CharField(max_length=100, blank=True, null=True) + history_type = models.CharField(max_length=1) + history_user_id = models.BigIntegerField(blank=True, null=True) + user_id = models.BigIntegerField(blank=True, null=True) + async_set_combined_emprise_done = models.BooleanField() + available_millesimes = models.CharField(max_length=255, blank=True, null=True) + async_theme_map_gpu_done = models.BooleanField() + theme_map_gpu = models.TextField(blank=True, null=True) + public_keys = models.CharField(max_length=255, blank=True, null=True) + async_theme_map_fill_gpu_done = models.BooleanField() + theme_map_fill_gpu = models.TextField(blank=True, null=True) + async_ocsge_coverage_status_done = models.BooleanField() + ocsge_coverage_status = models.CharField(max_length=20) + + class Meta: + managed = False + db_table = "project_historicalproject" + + +class ProjectHistoricalrequest(models.Model): + id = models.BigIntegerField() + first_name = models.CharField(max_length=150, blank=True, null=True) + last_name = models.CharField(max_length=150, blank=True, null=True) + function = models.CharField(max_length=250, blank=True, null=True) + organism = models.CharField(max_length=30) + email = models.CharField(max_length=254) + created_date = models.DateTimeField() + updated_date = models.DateTimeField() + sent_date = models.DateTimeField(blank=True, null=True) + done = models.BooleanField() + sent_file = models.TextField(blank=True, null=True) + history_id = models.AutoField(primary_key=True) + history_date = models.DateTimeField() + history_change_reason = models.CharField(max_length=100, blank=True, null=True) + history_type = models.CharField(max_length=1) + history_user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) + project_id = models.BigIntegerField(blank=True, null=True) + user_id = models.BigIntegerField(blank=True, null=True) + requested_document = models.CharField(max_length=30) + competence_urba = models.BooleanField() + du_en_cours = models.BooleanField() + + class Meta: + managed = False + db_table = "project_historicalrequest" + + +class ProjectProject(models.Model): + id = models.BigAutoField(primary_key=True) + name = models.CharField(max_length=100) + analyse_start_date = models.CharField(max_length=4) + analyse_end_date = models.CharField(max_length=4) + user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) + is_public = models.BooleanField() + look_a_like = models.CharField(max_length=250, blank=True, null=True) + created_date = models.DateTimeField() + updated_date = models.DateTimeField() + first_year_ocsge = models.IntegerField(blank=True, null=True) + last_year_ocsge = models.IntegerField(blank=True, null=True) + level = models.CharField(max_length=7) + land_id = models.CharField(max_length=255, blank=True, null=True) + land_type = models.CharField(max_length=7, blank=True, null=True) + cover_image = models.CharField(max_length=100, blank=True, null=True) + folder_name = models.CharField(max_length=15, blank=True, null=True) + territory_name = models.CharField(max_length=250, blank=True, null=True) + target_2031 = models.IntegerField() + theme_map_artif = models.CharField(max_length=100, blank=True, null=True) + theme_map_conso = models.CharField(max_length=100, blank=True, null=True) + theme_map_understand_artif = models.CharField(max_length=100, blank=True, null=True) + async_add_comparison_lands_done = models.BooleanField() + async_add_city_done = models.BooleanField() + async_cover_image_done = models.BooleanField() + async_find_first_and_last_ocsge_done = models.BooleanField() + async_generate_theme_map_artif_done = models.BooleanField() + async_generate_theme_map_conso_done = models.BooleanField() + async_theme_map_understand_artif_done = models.BooleanField() + async_set_combined_emprise_done = models.BooleanField() + available_millesimes = models.CharField(max_length=255, blank=True, null=True) + async_theme_map_gpu_done = models.BooleanField() + theme_map_gpu = models.CharField(max_length=100, blank=True, null=True) + public_keys = models.CharField(max_length=255, blank=True, null=True) + async_theme_map_fill_gpu_done = models.BooleanField() + theme_map_fill_gpu = models.CharField(max_length=100, blank=True, null=True) + async_ocsge_coverage_status_done = models.BooleanField() + ocsge_coverage_status = models.CharField(max_length=20) + + class Meta: + managed = False + db_table = "project_project" + + +class ProjectProjectcommune(models.Model): + id = models.BigAutoField(primary_key=True) + project = models.ForeignKey(ProjectProject, models.DO_NOTHING) + group_name = models.CharField(max_length=100, blank=True, null=True) + commune_id = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = "project_projectcommune" + + +class ProjectRequest(models.Model): + id = models.BigAutoField(primary_key=True) + first_name = models.CharField(max_length=150, blank=True, null=True) + last_name = models.CharField(max_length=150, blank=True, null=True) + function = models.CharField(max_length=250, blank=True, null=True) + organism = models.CharField(max_length=30) + email = models.CharField(max_length=254) + created_date = models.DateTimeField() + updated_date = models.DateTimeField() + sent_date = models.DateTimeField(blank=True, null=True) + done = models.BooleanField() + project = models.ForeignKey(ProjectProject, models.DO_NOTHING, blank=True, null=True) + user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) + sent_file = models.CharField(max_length=100, blank=True, null=True) + requested_document = models.CharField(max_length=30) + competence_urba = models.BooleanField() + du_en_cours = models.BooleanField() + + class Meta: + managed = False + db_table = "project_request" + + +class ProjectRnupackage(models.Model): + id = models.BigAutoField(primary_key=True) + file = models.CharField(max_length=100, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + departement_official_id = models.CharField(unique=True, max_length=10) + app_version = models.CharField(max_length=10) + + class Meta: + managed = False + db_table = "project_rnupackage" + + +class ProjectRnupackagerequest(models.Model): + id = models.BigAutoField(primary_key=True) + departement_official_id = models.CharField(max_length=10) + email = models.CharField(max_length=254) + requested_at = models.DateTimeField() + requested_diagnostics_before_package_request = models.IntegerField() + account_created_for_package = models.BooleanField() + rnu_package = models.ForeignKey(ProjectRnupackage, models.DO_NOTHING, blank=True, null=True) + user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) + + class Meta: + managed = False + db_table = "project_rnupackagerequest" + + +class PublicDataArtifareazoneurba(models.Model): + zone_urba = models.CharField(blank=True, null=True) + year = models.IntegerField(blank=True, null=True) + departement = models.CharField(blank=True, null=True) + area = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_artifareazoneurba" + + +class PublicDataArtificialarea(models.Model): + commune_year_id = models.CharField(blank=True, null=True) + year = models.IntegerField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + departement = models.CharField(blank=True, null=True) + city = models.CharField(max_length=5, blank=True, null=True) + surface = models.FloatField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_artificialarea" + + +class PublicDataCerema(models.Model): + city_insee = models.CharField(blank=True, null=True) + city_name = models.CharField(max_length=50, blank=True, null=True) + region_name = models.CharField(max_length=35, blank=True, null=True) + region_id = models.CharField(max_length=2, blank=True, null=True) + dept_id = models.CharField(max_length=3, blank=True, null=True) + dept_name = models.CharField(max_length=30, blank=True, null=True) + epci_id = models.CharField(blank=True, null=True) + epci_name = models.CharField(max_length=230, blank=True, null=True) + scot = models.CharField(blank=True, null=True) + art09inc23 = models.FloatField(blank=True, null=True) + art09fer23 = models.FloatField(blank=True, null=True) + art09rou23 = models.FloatField(blank=True, null=True) + art09mix23 = models.FloatField(blank=True, null=True) + art09hab23 = models.FloatField(blank=True, null=True) + art09act23 = models.FloatField(blank=True, null=True) + art22inc23 = models.FloatField(blank=True, null=True) + art22fer23 = models.FloatField(blank=True, null=True) + art22rou23 = models.FloatField(blank=True, null=True) + art22mix23 = models.FloatField(blank=True, null=True) + art22hab23 = models.FloatField(blank=True, null=True) + art22act23 = models.FloatField(blank=True, null=True) + naf22art23 = models.FloatField(blank=True, null=True) + art21inc22 = models.FloatField(blank=True, null=True) + art21fer22 = models.FloatField(blank=True, null=True) + art21rou22 = models.FloatField(blank=True, null=True) + art21mix22 = models.FloatField(blank=True, null=True) + art21hab22 = models.FloatField(blank=True, null=True) + art21act22 = models.FloatField(blank=True, null=True) + naf21art22 = models.FloatField(blank=True, null=True) + art20inc21 = models.FloatField(blank=True, null=True) + art20fer21 = models.FloatField(blank=True, null=True) + art20rou21 = models.FloatField(blank=True, null=True) + art20mix21 = models.FloatField(blank=True, null=True) + art20hab21 = models.FloatField(blank=True, null=True) + art20act21 = models.FloatField(blank=True, null=True) + naf20art21 = models.FloatField(blank=True, null=True) + art19inc20 = models.FloatField(blank=True, null=True) + art19fer20 = models.FloatField(blank=True, null=True) + art19rou20 = models.FloatField(blank=True, null=True) + art19mix20 = models.FloatField(blank=True, null=True) + art19hab20 = models.FloatField(blank=True, null=True) + art19act20 = models.FloatField(blank=True, null=True) + naf19art20 = models.FloatField(blank=True, null=True) + art18inc19 = models.FloatField(blank=True, null=True) + art18fer19 = models.FloatField(blank=True, null=True) + art18rou19 = models.FloatField(blank=True, null=True) + art18mix19 = models.FloatField(blank=True, null=True) + art18hab19 = models.FloatField(blank=True, null=True) + art18act19 = models.FloatField(blank=True, null=True) + naf18art19 = models.FloatField(blank=True, null=True) + art17inc18 = models.FloatField(blank=True, null=True) + art17fer18 = models.FloatField(blank=True, null=True) + art17rou18 = models.FloatField(blank=True, null=True) + art17mix18 = models.FloatField(blank=True, null=True) + art17hab18 = models.FloatField(blank=True, null=True) + art17act18 = models.FloatField(blank=True, null=True) + naf17art18 = models.FloatField(blank=True, null=True) + art16inc17 = models.FloatField(blank=True, null=True) + art16fer17 = models.FloatField(blank=True, null=True) + art16rou17 = models.FloatField(blank=True, null=True) + art16mix17 = models.FloatField(blank=True, null=True) + art16hab17 = models.FloatField(blank=True, null=True) + art16act17 = models.FloatField(blank=True, null=True) + naf16art17 = models.FloatField(blank=True, null=True) + art15inc16 = models.FloatField(blank=True, null=True) + art15fer16 = models.FloatField(blank=True, null=True) + art15rou16 = models.FloatField(blank=True, null=True) + art15mix16 = models.FloatField(blank=True, null=True) + art15hab16 = models.FloatField(blank=True, null=True) + art15act16 = models.FloatField(blank=True, null=True) + naf15art16 = models.FloatField(blank=True, null=True) + art14inc15 = models.FloatField(blank=True, null=True) + art14fer15 = models.FloatField(blank=True, null=True) + art14rou15 = models.FloatField(blank=True, null=True) + art14mix15 = models.FloatField(blank=True, null=True) + art14hab15 = models.FloatField(blank=True, null=True) + art14act15 = models.FloatField(blank=True, null=True) + naf14art15 = models.FloatField(blank=True, null=True) + art13inc14 = models.FloatField(blank=True, null=True) + art13fer14 = models.FloatField(blank=True, null=True) + art13rou14 = models.FloatField(blank=True, null=True) + art13mix14 = models.FloatField(blank=True, null=True) + art13hab14 = models.FloatField(blank=True, null=True) + art13act14 = models.FloatField(blank=True, null=True) + naf13art14 = models.FloatField(blank=True, null=True) + art12inc13 = models.FloatField(blank=True, null=True) + art12fer13 = models.FloatField(blank=True, null=True) + art12rou13 = models.FloatField(blank=True, null=True) + art12mix13 = models.FloatField(blank=True, null=True) + art12hab13 = models.FloatField(blank=True, null=True) + art12act13 = models.FloatField(blank=True, null=True) + naf12art13 = models.FloatField(blank=True, null=True) + art11inc12 = models.FloatField(blank=True, null=True) + art11fer12 = models.FloatField(blank=True, null=True) + art11rou12 = models.FloatField(blank=True, null=True) + art11mix12 = models.FloatField(blank=True, null=True) + art11hab12 = models.FloatField(blank=True, null=True) + art11act12 = models.FloatField(blank=True, null=True) + naf11art12 = models.FloatField(blank=True, null=True) + art10inc11 = models.FloatField(blank=True, null=True) + art10fer11 = models.FloatField(blank=True, null=True) + art10rou11 = models.FloatField(blank=True, null=True) + art10mix11 = models.FloatField(blank=True, null=True) + art10hab11 = models.FloatField(blank=True, null=True) + art10act11 = models.FloatField(blank=True, null=True) + naf10art11 = models.FloatField(blank=True, null=True) + art09inc10 = models.FloatField(blank=True, null=True) + art09fer10 = models.FloatField(blank=True, null=True) + art09rou10 = models.FloatField(blank=True, null=True) + art09mix10 = models.FloatField(blank=True, null=True) + art09hab10 = models.FloatField(blank=True, null=True) + art09act10 = models.FloatField(blank=True, null=True) + naf09art10 = models.FloatField(blank=True, null=True) + naf11art21 = models.FloatField(blank=True, null=True) + naf11inc21 = models.FloatField(blank=True, null=True) + naf11fer21 = models.FloatField(blank=True, null=True) + naf11rou21 = models.FloatField(blank=True, null=True) + naf11mix21 = models.FloatField(blank=True, null=True) + naf11act21 = models.FloatField(blank=True, null=True) + naf11hab21 = models.FloatField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_cerema" + + +class PublicDataCommune(models.Model): + insee = models.CharField(max_length=5, blank=True, null=True) + departement_id = models.CharField(max_length=3, blank=True, null=True) + epci_id = models.CharField(blank=True, null=True) + ept_id = models.CharField(blank=True, null=True) + scot_id = models.CharField(blank=True, null=True) + first_millesime = models.IntegerField(blank=True, null=True) + last_millesime = models.IntegerField(blank=True, null=True) + name = models.CharField(max_length=50, blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + ocsge_available = models.BooleanField(blank=True, null=True) + surface_artif = models.FloatField(blank=True, null=True) + area = models.FloatField(blank=True, null=True) + consommation_correction_status = models.CharField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_commune" + + +class PublicDataCommuneEpci(models.Model): + commune_id = models.CharField(max_length=5, blank=True, null=True) + epci_id = models.CharField(max_length=9, blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_commune_epci" + + +class PublicDataCommunediff(models.Model): + year_old = models.IntegerField(blank=True, null=True) + year_new = models.IntegerField(blank=True, null=True) + new_artif = models.FloatField(blank=True, null=True) + new_natural = models.FloatField(blank=True, null=True) + city_id = models.CharField(max_length=5, blank=True, null=True) + net_artif = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_communediff" + + +class PublicDataCommunepop(models.Model): + id = models.BigAutoField(primary_key=True) + year = models.IntegerField() + pop = models.IntegerField(blank=True, null=True) + city_id = models.BigIntegerField() + household = models.IntegerField(blank=True, null=True) + household_change = models.IntegerField(blank=True, null=True) + pop_change = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_communepop" + + +class PublicDataCommunesol(models.Model): + year = models.IntegerField(blank=True, null=True) + surface = models.FloatField(blank=True, null=True) + city_id = models.CharField(max_length=5, blank=True, null=True) + matrix_id = models.BigIntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_communesol" + + +class PublicDataCouverturesol(models.Model): + id = models.BigAutoField(primary_key=True) + code = models.CharField(unique=True, max_length=8) + label = models.CharField(max_length=250) + parent = models.ForeignKey("self", models.DO_NOTHING, blank=True, null=True) + code_prefix = models.CharField(unique=True, max_length=10) + map_color = models.CharField(max_length=8, blank=True, null=True) + label_short = models.CharField(max_length=50, blank=True, null=True) + is_key = models.BooleanField() + + class Meta: + managed = False + db_table = "public_data_couverturesol" + + +class PublicDataCouvertureusagematrix(models.Model): + id = models.BigAutoField(primary_key=True) + is_artificial = models.BooleanField(blank=True, null=True) + couverture = models.ForeignKey(PublicDataCouverturesol, models.DO_NOTHING, blank=True, null=True) + usage = models.ForeignKey("PublicDataUsagesol", models.DO_NOTHING, blank=True, null=True) + is_natural = models.BooleanField(blank=True, null=True) + label = models.CharField(max_length=20) + is_impermeable = models.BooleanField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_couvertureusagematrix" + unique_together = (("couverture", "usage"),) + + +class PublicDataDepartement(models.Model): + source_id = models.CharField(max_length=3, blank=True, null=True) + name = models.CharField(max_length=30, blank=True, null=True) + region_id = models.CharField(max_length=2, blank=True, null=True) + ocsge_millesimes = models.TextField(blank=True, null=True) # This field type is a guess. + srid_source = models.IntegerField(blank=True, null=True) + is_artif_ready = models.BooleanField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_departement" + + +class PublicDataEpci(models.Model): + source_id = models.CharField(max_length=9, blank=True, null=True) + name = models.CharField(max_length=230, blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_epci" + + +class PublicDataEpciDepartements(models.Model): + epci_id = models.CharField(max_length=9, blank=True, null=True) + departement_id = models.CharField(max_length=3, blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_epci_departements" + + +class PublicDataLandconso(models.Model): + land_id = models.CharField(blank=True, null=True) + land_type = models.CharField(blank=True, null=True) + surface = models.FloatField(blank=True, null=True) + year = models.IntegerField(blank=True, null=True) + total = models.FloatField(blank=True, null=True) + activite = models.FloatField(blank=True, null=True) + habitat = models.FloatField(blank=True, null=True) + mixte = models.FloatField(blank=True, null=True) + route = models.FloatField(blank=True, null=True) + ferroviaire = models.FloatField(blank=True, null=True) + inconnu = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_landconso" + + +class PublicDataLandconsocomparison(models.Model): + relevance_level = models.CharField(blank=True, null=True) + land_type = models.CharField(blank=True, null=True) + land_id = models.CharField(blank=True, null=True) + from_year = models.IntegerField(blank=True, null=True) + to_year = models.IntegerField(blank=True, null=True) + total_median = models.FloatField(blank=True, null=True) + total_median_percent = models.FloatField(blank=True, null=True) + total_avg = models.FloatField(blank=True, null=True) + total_percent = models.FloatField(blank=True, null=True) + activite_median = models.FloatField(blank=True, null=True) + activite_median_percent = models.FloatField(blank=True, null=True) + activite_avg = models.FloatField(blank=True, null=True) + activite_percent = models.FloatField(blank=True, null=True) + habitat_median = models.FloatField(blank=True, null=True) + habitat_median_percent = models.FloatField(blank=True, null=True) + habitat_avg = models.FloatField(blank=True, null=True) + habitat_percent = models.FloatField(blank=True, null=True) + mixte_median = models.FloatField(blank=True, null=True) + mixte_median_percent = models.FloatField(blank=True, null=True) + mixte_avg = models.FloatField(blank=True, null=True) + mixte_percent = models.FloatField(blank=True, null=True) + route_median = models.FloatField(blank=True, null=True) + route_median_percent = models.FloatField(blank=True, null=True) + route_avg = models.FloatField(blank=True, null=True) + route_percent = models.FloatField(blank=True, null=True) + ferroviaire_median = models.FloatField(blank=True, null=True) + ferroviaire_median_percent = models.FloatField(blank=True, null=True) + ferroviaire_avg = models.FloatField(blank=True, null=True) + ferroviaire_percent = models.FloatField(blank=True, null=True) + inconnu_median = models.FloatField(blank=True, null=True) + inconnu_median_percent = models.FloatField(blank=True, null=True) + inconnu_avg = models.FloatField(blank=True, null=True) + inconnu_percent = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_landconsocomparison" + + +class PublicDataLandconsostats(models.Model): + land_id = models.CharField(blank=True, null=True) + land_type = models.CharField(blank=True, null=True) + comparison_level = models.CharField(blank=True, null=True) + comparison_id = models.CharField(blank=True, null=True) + from_year = models.IntegerField(blank=True, null=True) + to_year = models.IntegerField(blank=True, null=True) + total = models.FloatField(blank=True, null=True) + total_percent = models.FloatField(blank=True, null=True) + activite = models.FloatField(blank=True, null=True) + activite_percent = models.FloatField(blank=True, null=True) + habitat = models.FloatField(blank=True, null=True) + habitat_percent = models.FloatField(blank=True, null=True) + mixte = models.FloatField(blank=True, null=True) + mixte_percent = models.FloatField(blank=True, null=True) + route = models.FloatField(blank=True, null=True) + route_percent = models.FloatField(blank=True, null=True) + ferroviaire = models.FloatField(blank=True, null=True) + ferroviaire_percent = models.FloatField(blank=True, null=True) + inconnu = models.FloatField(blank=True, null=True) + inconnu_percent = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_landconsostats" + + +class PublicDataLandpop(models.Model): + land_id = models.CharField(blank=True, null=True) + land_type = models.CharField(blank=True, null=True) + year = models.IntegerField(blank=True, null=True) + evolution = models.FloatField(blank=True, null=True) + population = models.FloatField(blank=True, null=True) + source = models.CharField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_landpop" + + +class PublicDataLandpopcomparison(models.Model): + relevance_level = models.CharField(blank=True, null=True) + land_type = models.CharField(blank=True, null=True) + land_id = models.CharField(blank=True, null=True) + from_year = models.IntegerField(blank=True, null=True) + to_year = models.IntegerField(blank=True, null=True) + evolution_median = models.FloatField(blank=True, null=True) + evolution_median_percent = models.FloatField(blank=True, null=True) + evolution_avg = models.FloatField(blank=True, null=True) + evolution_percent = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_landpopcomparison" + + +class PublicDataLandpopstats(models.Model): + land_id = models.CharField(blank=True, null=True) + land_type = models.CharField(blank=True, null=True) + comparison_level = models.CharField(blank=True, null=True) + comparison_id = models.CharField(blank=True, null=True) + from_year = models.IntegerField(blank=True, null=True) + to_year = models.IntegerField(blank=True, null=True) + evolution = models.FloatField(blank=True, null=True) + evolution_percent = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_landpopstats" + + +class PublicDataOcsge(models.Model): + uuid = models.CharField(blank=True, null=True) + couverture = models.CharField(blank=True, null=True) + usage = models.CharField(blank=True, null=True) + year = models.IntegerField(blank=True, null=True) + id_source = models.CharField(blank=True, null=True) + is_artificial = models.BooleanField(blank=True, null=True) + surface = models.FloatField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + departement = models.CharField(blank=True, null=True) + is_impermeable = models.BooleanField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_ocsge" + + +class PublicDataOcsgediff(models.Model): + uuid = models.CharField(blank=True, null=True) + year_old = models.IntegerField(blank=True, null=True) + year_new = models.IntegerField(blank=True, null=True) + cs_new = models.CharField(blank=True, null=True) + cs_old = models.CharField(blank=True, null=True) + us_new = models.CharField(blank=True, null=True) + us_old = models.CharField(blank=True, null=True) + surface = models.FloatField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + departement = models.CharField(blank=True, null=True) + is_new_artif = models.BooleanField(blank=True, null=True) + is_new_natural = models.BooleanField(blank=True, null=True) + is_new_impermeable = models.BooleanField(blank=True, null=True) + is_new_not_impermeable = models.BooleanField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_ocsgediff" + + +class PublicDataRegion(models.Model): + source_id = models.CharField(max_length=2, blank=True, null=True) + name = models.CharField(max_length=35, blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_region" + + +class PublicDataScot(models.Model): + siren = models.CharField(blank=True, null=True) + source_id = models.CharField(blank=True, null=True) + name = models.CharField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_scot" + + +class PublicDataScotDepartements(models.Model): + scot_id = models.CharField(blank=True, null=True) + departement_id = models.CharField(max_length=3, blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_scot_departements" + + +class PublicDataScotRegions(models.Model): + scot_id = models.CharField(blank=True, null=True) + region_id = models.CharField(max_length=2, blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_scot_regions" + + +class PublicDataSudocuh(models.Model): + code_insee = models.CharField(primary_key=True, max_length=200) + code_departement = models.CharField(max_length=200) + nom_region = models.CharField(max_length=200) + nom_commune = models.CharField(max_length=200) + collectivite_porteuse = models.CharField(max_length=200) + siren_epci = models.CharField(max_length=200, blank=True, null=True) + code_etat = models.CharField(max_length=2) + du_opposable = models.CharField(max_length=200) + du_en_cours = models.CharField(max_length=200, blank=True, null=True) + code_bcsi = models.CharField(max_length=200) + etat_commune = models.CharField(max_length=200) + etat_detaille = models.CharField(max_length=200) + prescription_du_en_vigueur = models.DateField(blank=True, null=True) + approbation_du_en_vigueur = models.DateField(blank=True, null=True) + executoire_du_en_vigueur = models.DateField(blank=True, null=True) + prescription_proc_en_cours = models.DateField(blank=True, null=True) + population_municipale = models.IntegerField() + population_totale = models.IntegerField() + superficie = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_sudocuh" + + +class PublicDataSudocuhepci(models.Model): + id = models.BigAutoField(primary_key=True) + nom_region = models.CharField(max_length=200) + code_departement = models.CharField(max_length=200) + nom_departement = models.CharField(max_length=200) + siren = models.CharField(max_length=200) + type_epci = models.CharField(max_length=200) + nom_epci = models.CharField(max_length=200) + date_creation_epci = models.DateField(blank=True, null=True) + epci_interdepartemental = models.BooleanField() + competence_plan = models.BooleanField() + competence_scot = models.BooleanField() + competence_plh = models.BooleanField() + obligation_plh = models.BooleanField() + nb_communes = models.IntegerField() + insee_pop_tot_2021 = models.IntegerField() + insee_pop_municipale = models.IntegerField() + insee_superficie = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_sudocuhepci" + + +class PublicDataUsagesol(models.Model): + id = models.BigAutoField(primary_key=True) + code = models.CharField(unique=True, max_length=8) + label = models.CharField(max_length=250) + parent = models.ForeignKey("self", models.DO_NOTHING, blank=True, null=True) + code_prefix = models.CharField(unique=True, max_length=10) + map_color = models.CharField(max_length=8, blank=True, null=True) + label_short = models.CharField(max_length=50, blank=True, null=True) + is_key = models.BooleanField() + + class Meta: + managed = False + db_table = "public_data_usagesol" + + +class PublicDataZoneconstruite(models.Model): + uuid = models.CharField(blank=True, null=True) + id_source = models.CharField(blank=True, null=True) + millesime = models.IntegerField(blank=True, null=True) + year = models.IntegerField(blank=True, null=True) + surface = models.FloatField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + departement = models.CharField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_zoneconstruite" + + +class PublicDataZoneurba(models.Model): + checksum = models.CharField(blank=True, null=True) + libelle = models.CharField(blank=True, null=True) + libelong = models.CharField(blank=True, null=True) + idurba = models.CharField(blank=True, null=True) + typezone = models.CharField(blank=True, null=True) + partition = models.CharField(blank=True, null=True) + datappro = models.CharField(blank=True, null=True) + datvalid = models.CharField(blank=True, null=True) + area = models.FloatField(blank=True, null=True) + srid_source = models.IntegerField(blank=True, null=True) + mpoly = models.GeometryField(blank=True, null=True) + + class Meta: + managed = False + db_table = "public_data_zoneurba" + + +class UsersUser(models.Model): + id = models.BigAutoField(primary_key=True) + password = models.CharField(max_length=128) + last_login = models.DateTimeField(blank=True, null=True) + is_superuser = models.BooleanField() + first_name = models.CharField(max_length=150) + last_name = models.CharField(max_length=150) + email = models.CharField(unique=True, max_length=254) + is_staff = models.BooleanField() + is_active = models.BooleanField() + date_joined = models.DateTimeField() + email_checked = models.DateTimeField(blank=True, null=True) + function = models.CharField(max_length=250) + organism = models.CharField(max_length=250) + organism_group = models.CharField(max_length=250, blank=True, null=True) + created_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = "users_user" + + +class UsersUserGroups(models.Model): + id = models.BigAutoField(primary_key=True) + user = models.ForeignKey(UsersUser, models.DO_NOTHING) + group = models.ForeignKey(AuthGroup, models.DO_NOTHING) + + class Meta: + managed = False + db_table = "users_user_groups" + unique_together = (("user", "group"),) + + +class UsersUserUserPermissions(models.Model): + id = models.BigAutoField(primary_key=True) + user = models.ForeignKey(UsersUser, models.DO_NOTHING) + permission = models.ForeignKey(AuthPermission, models.DO_NOTHING) + + class Meta: + managed = False + db_table = "users_user_user_permissions" + unique_together = (("user", "permission"),) diff --git a/public_data/models/__init__.py b/public_data/models/__init__.py index ecb876f37..b66b8de42 100644 --- a/public_data/models/__init__.py +++ b/public_data/models/__init__.py @@ -1,6 +1,8 @@ from .administration import * # noqa: F401, F403 from .cerema import Cerema # noqa: F401 +from .consommation import * # noqa: F401, F403 from .couverture_usage import * # noqa: F401, F403 +from .demography import * # noqa: F401, F403 from .gpu import * # noqa: F401, F403 from .mixins import * # noqa: F401, F403 from .ocsge import ArtificialArea, Ocsge, OcsgeDiff, ZoneConstruite # noqa: F401, F403 diff --git a/public_data/models/consommation/LandConso.py b/public_data/models/consommation/LandConso.py new file mode 100644 index 000000000..04a22cfcd --- /dev/null +++ b/public_data/models/consommation/LandConso.py @@ -0,0 +1,24 @@ +from django.db import models + +from public_data.models.administration import AdminRef + + +class LandConso(models.Model): + land_id = models.CharField() + land_type = models.CharField(choices=AdminRef.CHOICES) + surface = models.FloatField() + year = models.IntegerField() + total = models.FloatField() + activite = models.FloatField() + habitat = models.FloatField() + mixte = models.FloatField() + route = models.FloatField() + ferroviaire = models.FloatField() + inconnu = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_landconso" + + def __str__(self): + return f"{self.land_type} {self.land_id} ({self.year})" diff --git a/public_data/models/consommation/LandConsoComparison.py b/public_data/models/consommation/LandConsoComparison.py new file mode 100644 index 000000000..5733fb167 --- /dev/null +++ b/public_data/models/consommation/LandConsoComparison.py @@ -0,0 +1,43 @@ +from django.db import models + +from public_data.models.administration import AdminRef + + +class LandConsoComparison(models.Model): + relevance_level = models.CharField(choices=AdminRef.CHOICES) + land_type = models.CharField(choices=AdminRef.CHOICES) + land_id = models.CharField() + from_year = models.IntegerField() + to_year = models.IntegerField() + total_median = models.FloatField() + total_median_percent = models.FloatField() + total_avg = models.FloatField() + total_percent = models.FloatField() + activite_median = models.FloatField() + activite_median_percent = models.FloatField() + activite_avg = models.FloatField() + activite_percent = models.FloatField() + habitat_median = models.FloatField() + habitat_median_percent = models.FloatField() + habitat_avg = models.FloatField() + habitat_percent = models.FloatField() + mixte_median = models.FloatField() + mixte_median_percent = models.FloatField() + mixte_avg = models.FloatField() + mixte_percent = models.FloatField() + route_median = models.FloatField() + route_median_percent = models.FloatField() + route_avg = models.FloatField() + route_percent = models.FloatField() + ferroviaire_median = models.FloatField() + ferroviaire_median_percent = models.FloatField() + ferroviaire_avg = models.FloatField() + ferroviaire_percent = models.FloatField() + inconnu_median = models.FloatField() + inconnu_median_percent = models.FloatField() + inconnu_avg = models.FloatField() + inconnu_percent = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_landconsocomparison" diff --git a/public_data/models/consommation/LandConsoStats.py b/public_data/models/consommation/LandConsoStats.py new file mode 100644 index 000000000..2a900e04a --- /dev/null +++ b/public_data/models/consommation/LandConsoStats.py @@ -0,0 +1,30 @@ +from django.db import models + +from public_data.models.administration import AdminRef + + +class LandConsoStats(models.Model): + land_id = models.CharField() + land_type = models.CharField(choices=AdminRef.CHOICES) + comparison_level = models.CharField(choices=AdminRef.CHOICES) + comparison_id = models.CharField(choices=AdminRef.CHOICES) + from_year = models.IntegerField() + to_year = models.IntegerField() + total = models.FloatField() + total_percent = models.FloatField() + activite = models.FloatField() + activite_percent = models.FloatField() + habitat = models.FloatField() + habitat_percent = models.FloatField() + mixte = models.FloatField() + mixte_percent = models.FloatField() + route = models.FloatField() + route_percent = models.FloatField() + ferroviaire = models.FloatField() + ferroviaire_percent = models.FloatField() + inconnu = models.FloatField() + inconnu_percent = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_landconsostats" diff --git a/public_data/models/consommation/__init__.py b/public_data/models/consommation/__init__.py new file mode 100644 index 000000000..2baadf453 --- /dev/null +++ b/public_data/models/consommation/__init__.py @@ -0,0 +1,3 @@ +from .LandConso import LandConso # noqa: F401 +from .LandConsoComparison import LandConsoComparison # noqa: F401 +from .LandConsoStats import LandConsoStats # noqa: F401 diff --git a/public_data/models/demography/LandPop.py b/public_data/models/demography/LandPop.py new file mode 100644 index 000000000..d12873ab7 --- /dev/null +++ b/public_data/models/demography/LandPop.py @@ -0,0 +1,20 @@ +from django.db import models + +from public_data.models.administration import AdminRef + + +class LandPop(models.Model): + class Source(models.TextChoices): + INSEE = "INSEE" + PROJECTION = "PROJECTION" + + land_id = models.CharField() + land_type = models.CharField(choices=AdminRef.CHOICES) + year = models.IntegerField() + evolution = models.IntegerField() + population = models.IntegerField() + source = models.CharField(choices=Source.choices) + + class Meta: + managed = False + db_table = "public_data_landpop" diff --git a/public_data/models/demography/LandPopComparison.py b/public_data/models/demography/LandPopComparison.py new file mode 100644 index 000000000..04c4df08d --- /dev/null +++ b/public_data/models/demography/LandPopComparison.py @@ -0,0 +1,19 @@ +from django.db import models + +from public_data.models.administration import AdminRef + + +class LandPopComparison(models.Model): + relevance_level = models.CharField(choices=AdminRef.CHOICES) + land_type = models.CharField(choices=AdminRef.CHOICES) + land_id = models.CharField() + from_year = models.IntegerField() + to_year = models.IntegerField() + evolution_median = models.FloatField() + evolution_median_percent = models.FloatField() + evolution_avg = models.FloatField() + evolution_percent = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_landpopcomparison" diff --git a/public_data/models/demography/LandPopStats.py b/public_data/models/demography/LandPopStats.py new file mode 100644 index 000000000..d4b8a5df1 --- /dev/null +++ b/public_data/models/demography/LandPopStats.py @@ -0,0 +1,18 @@ +from django.db import models + +from public_data.models.administration import AdminRef + + +class LandPopStats(models.Model): + land_id = models.CharField() + land_type = models.CharField(choices=AdminRef.CHOICES) + comparison_level = models.CharField() + comparison_id = models.CharField(choices=AdminRef.CHOICES) + from_year = models.IntegerField() + to_year = models.IntegerField() + evolution = models.FloatField() + evolution_percent = models.FloatField() + + class Meta: + managed = False + db_table = "public_data_landpopstats" diff --git a/public_data/models/demography/__init__.py b/public_data/models/demography/__init__.py new file mode 100644 index 000000000..fb971e245 --- /dev/null +++ b/public_data/models/demography/__init__.py @@ -0,0 +1,3 @@ +from .LandPop import LandPop # noqa : F401 +from .LandPopComparison import LandPopComparison # noqa : F401 +from .LandPopStats import LandPopStats # noqa : F401 diff --git a/utils/schema.py b/utils/schema.py index 2d5d24fe2..2bf5da23a 100644 --- a/utils/schema.py +++ b/utils/schema.py @@ -13,6 +13,12 @@ CommuneSol, Departement, Epci, + LandConso, + LandConsoComparison, + LandConsoStats, + LandPop, + LandPopComparison, + LandPopStats, Region, Scot, ZoneUrba, @@ -56,3 +62,9 @@ def init_unmanaged_schema_for_tests() -> None: drop_and_create_model(ZoneUrba, schema_editor) drop_and_create_model(ArtificialArea, schema_editor) drop_and_create_model(Cerema, schema_editor) + drop_and_create_model(LandConso, schema_editor) + drop_and_create_model(LandConsoComparison, schema_editor) + drop_and_create_model(LandConsoStats, schema_editor) + drop_and_create_model(LandPop, schema_editor) + drop_and_create_model(LandPopStats, schema_editor) + drop_and_create_model(LandPopComparison, schema_editor) From d95d72316b8d3189c737128ab60e31574d8f3887 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 22:43:00 +0100 Subject: [PATCH 08/14] chore(models.temp): remove file --- models.temp.py | 1100 ------------------------------------------------ 1 file changed, 1100 deletions(-) delete mode 100644 models.temp.py diff --git a/models.temp.py b/models.temp.py deleted file mode 100644 index cce9708cc..000000000 --- a/models.temp.py +++ /dev/null @@ -1,1100 +0,0 @@ -# This is an auto-generated Django model module. -# You'll have to do the following manually to clean this up: -# * Rearrange models' order -# * Make sure each model has one field with primary_key=True -# * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior -# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table -# Feel free to rename the models, but don't rename db_table values or field names. -from django.contrib.gis.db import models - - -class AppParameterParameter(models.Model): - id = models.BigAutoField(primary_key=True) - name = models.CharField(max_length=100) - slug = models.CharField(unique=True, max_length=40) - value_type = models.CharField(max_length=3) - description = models.TextField() - value = models.CharField(max_length=250) - - class Meta: - managed = False - db_table = "app_parameter_parameter" - - -class AuthGroup(models.Model): - name = models.CharField(unique=True, max_length=150) - - class Meta: - managed = False - db_table = "auth_group" - - -class AuthGroupPermissions(models.Model): - id = models.BigAutoField(primary_key=True) - group = models.ForeignKey(AuthGroup, models.DO_NOTHING) - permission = models.ForeignKey("AuthPermission", models.DO_NOTHING) - - class Meta: - managed = False - db_table = "auth_group_permissions" - unique_together = (("group", "permission"),) - - -class AuthPermission(models.Model): - name = models.CharField(max_length=255) - content_type = models.ForeignKey("DjangoContentType", models.DO_NOTHING) - codename = models.CharField(max_length=100) - - class Meta: - managed = False - db_table = "auth_permission" - unique_together = (("content_type", "codename"),) - - -class CrispCrispwebhooknotification(models.Model): - id = models.BigAutoField(primary_key=True) - event = models.CharField(max_length=255) - timestamp = models.DateTimeField() - data = models.JSONField() - - class Meta: - managed = False - db_table = "crisp_crispwebhooknotification" - - -class DiagnosticWordWordtemplate(models.Model): - slug = models.CharField(primary_key=True, max_length=50) - description = models.TextField() - docx = models.CharField(max_length=100) - last_update = models.DateTimeField() - filename_mask = models.CharField(max_length=255) - - class Meta: - managed = False - db_table = "diagnostic_word_wordtemplate" - - -class DjangoAdminLog(models.Model): - action_time = models.DateTimeField() - object_id = models.TextField(blank=True, null=True) - object_repr = models.CharField(max_length=200) - action_flag = models.SmallIntegerField() - change_message = models.TextField() - content_type = models.ForeignKey("DjangoContentType", models.DO_NOTHING, blank=True, null=True) - user = models.ForeignKey("UsersUser", models.DO_NOTHING) - - class Meta: - managed = False - db_table = "django_admin_log" - - -class DjangoAppParameterParameter(models.Model): - id = models.BigAutoField(primary_key=True) - name = models.CharField(max_length=100) - slug = models.CharField(unique=True, max_length=40) - value_type = models.CharField(max_length=3) - description = models.TextField() - value = models.CharField(max_length=250) - is_global = models.BooleanField() - - class Meta: - managed = False - db_table = "django_app_parameter_parameter" - - -class DjangoCeleryResultsChordcounter(models.Model): - group_id = models.CharField(unique=True, max_length=255) - sub_tasks = models.TextField() - count = models.IntegerField() - - class Meta: - managed = False - db_table = "django_celery_results_chordcounter" - - -class DjangoCeleryResultsGroupresult(models.Model): - group_id = models.CharField(unique=True, max_length=255) - date_created = models.DateTimeField() - date_done = models.DateTimeField() - content_type = models.CharField(max_length=128) - content_encoding = models.CharField(max_length=64) - result = models.TextField(blank=True, null=True) - - class Meta: - managed = False - db_table = "django_celery_results_groupresult" - - -class DjangoCeleryResultsTaskresult(models.Model): - task_id = models.CharField(unique=True, max_length=255) - status = models.CharField(max_length=50) - content_type = models.CharField(max_length=128) - content_encoding = models.CharField(max_length=64) - result = models.TextField(blank=True, null=True) - date_done = models.DateTimeField() - traceback = models.TextField(blank=True, null=True) - meta = models.TextField(blank=True, null=True) - task_args = models.TextField(blank=True, null=True) - task_kwargs = models.TextField(blank=True, null=True) - task_name = models.CharField(max_length=255, blank=True, null=True) - worker = models.CharField(max_length=100, blank=True, null=True) - date_created = models.DateTimeField() - - class Meta: - managed = False - db_table = "django_celery_results_taskresult" - - -class DjangoContentType(models.Model): - app_label = models.CharField(max_length=100) - model = models.CharField(max_length=100) - - class Meta: - managed = False - db_table = "django_content_type" - unique_together = (("app_label", "model"),) - - -class DjangoMigrations(models.Model): - id = models.BigAutoField(primary_key=True) - app = models.CharField(max_length=255) - name = models.CharField(max_length=255) - applied = models.DateTimeField() - - class Meta: - managed = False - db_table = "django_migrations" - - -class DjangoSession(models.Model): - session_key = models.CharField(primary_key=True, max_length=40) - session_data = models.TextField() - expire_date = models.DateTimeField() - - class Meta: - managed = False - db_table = "django_session" - - -class DsfrDsfrconfig(models.Model): - id = models.BigAutoField(primary_key=True) - header_brand = models.CharField(max_length=200) - header_brand_html = models.CharField(max_length=200) - footer_brand = models.CharField(max_length=200) - footer_brand_html = models.CharField(max_length=200) - site_title = models.CharField(max_length=200) - site_tagline = models.CharField(max_length=200) - footer_description = models.TextField() - mourning = models.BooleanField() - accessibility_status = models.CharField(max_length=4) - - class Meta: - managed = False - db_table = "dsfr_dsfrconfig" - - -class HomeAlivetimestamp(models.Model): - id = models.BigAutoField() - timestamp = models.DateTimeField() - queue_name = models.CharField(max_length=20) - - class Meta: - managed = False - db_table = "home_alivetimestamp" - - -class HomeContactform(models.Model): - id = models.BigAutoField(primary_key=True) - email = models.CharField(max_length=254) - content = models.TextField() - status = models.CharField(max_length=10) - created_date = models.DateTimeField() - processed_date = models.DateTimeField(blank=True, null=True) - error = models.TextField(blank=True, null=True) - - class Meta: - managed = False - db_table = "home_contactform" - - -class HomeNewsletter(models.Model): - id = models.BigAutoField(primary_key=True) - email = models.CharField(max_length=254) - created_date = models.DateTimeField() - confirm_token = models.CharField(unique=True, max_length=25) - confirmation_date = models.DateTimeField(blank=True, null=True) - - class Meta: - managed = False - db_table = "home_newsletter" - - -class MetabaseStatdiagnostic(models.Model): - id = models.BigAutoField(primary_key=True) - created_date = models.DateTimeField() - is_anonymouse = models.BooleanField() - is_public = models.BooleanField() - administrative_level = models.CharField(max_length=255, blank=True, null=True) - analysis_level = models.CharField(max_length=255) - start_date = models.DateField() - end_date = models.DateField() - link = models.CharField(max_length=255) - city = models.CharField(max_length=255, blank=True, null=True) - epci = models.CharField(max_length=255, blank=True, null=True) - scot = models.CharField(max_length=255, blank=True, null=True) - departement = models.CharField(max_length=255, blank=True, null=True) - region = models.CharField(max_length=255, blank=True, null=True) - project = models.OneToOneField("ProjectProject", models.DO_NOTHING) - date_first_download = models.DateTimeField(blank=True, null=True) - is_downaloaded = models.BooleanField() - organism = models.CharField(max_length=255, blank=True, null=True) - request = models.ForeignKey("ProjectRequest", models.DO_NOTHING, blank=True, null=True) - group_organism = models.CharField(max_length=50, blank=True, null=True) - - class Meta: - managed = False - db_table = "metabase_statdiagnostic" - - -class ProjectEmprise(models.Model): - id = models.BigAutoField(primary_key=True) - mpoly = models.MultiPolygonField() - project = models.ForeignKey("ProjectProject", models.DO_NOTHING) - srid_source = models.IntegerField() - - class Meta: - managed = False - db_table = "project_emprise" - - -class ProjectErrortracking(models.Model): - id = models.BigAutoField(primary_key=True) - created_date = models.DateTimeField() - exception = models.TextField() - request = models.ForeignKey("ProjectRequest", models.DO_NOTHING) - - class Meta: - managed = False - db_table = "project_errortracking" - - -class ProjectHistoricalproject(models.Model): - id = models.BigIntegerField() - name = models.CharField(max_length=100) - is_public = models.BooleanField() - analyse_start_date = models.CharField(max_length=4) - analyse_end_date = models.CharField(max_length=4) - level = models.CharField(max_length=7) - land_type = models.CharField(max_length=7, blank=True, null=True) - land_id = models.CharField(max_length=255, blank=True, null=True) - look_a_like = models.CharField(max_length=250, blank=True, null=True) - target_2031 = models.IntegerField() - created_date = models.DateTimeField() - updated_date = models.DateTimeField() - first_year_ocsge = models.IntegerField(blank=True, null=True) - last_year_ocsge = models.IntegerField(blank=True, null=True) - folder_name = models.CharField(max_length=15, blank=True, null=True) - territory_name = models.CharField(max_length=250, blank=True, null=True) - cover_image = models.TextField(blank=True, null=True) - theme_map_conso = models.TextField(blank=True, null=True) - theme_map_artif = models.TextField(blank=True, null=True) - theme_map_understand_artif = models.TextField(blank=True, null=True) - async_add_city_done = models.BooleanField() - async_cover_image_done = models.BooleanField() - async_find_first_and_last_ocsge_done = models.BooleanField() - async_add_comparison_lands_done = models.BooleanField() - async_generate_theme_map_conso_done = models.BooleanField() - async_generate_theme_map_artif_done = models.BooleanField() - async_theme_map_understand_artif_done = models.BooleanField() - history_id = models.AutoField(primary_key=True) - history_date = models.DateTimeField() - history_change_reason = models.CharField(max_length=100, blank=True, null=True) - history_type = models.CharField(max_length=1) - history_user_id = models.BigIntegerField(blank=True, null=True) - user_id = models.BigIntegerField(blank=True, null=True) - async_set_combined_emprise_done = models.BooleanField() - available_millesimes = models.CharField(max_length=255, blank=True, null=True) - async_theme_map_gpu_done = models.BooleanField() - theme_map_gpu = models.TextField(blank=True, null=True) - public_keys = models.CharField(max_length=255, blank=True, null=True) - async_theme_map_fill_gpu_done = models.BooleanField() - theme_map_fill_gpu = models.TextField(blank=True, null=True) - async_ocsge_coverage_status_done = models.BooleanField() - ocsge_coverage_status = models.CharField(max_length=20) - - class Meta: - managed = False - db_table = "project_historicalproject" - - -class ProjectHistoricalrequest(models.Model): - id = models.BigIntegerField() - first_name = models.CharField(max_length=150, blank=True, null=True) - last_name = models.CharField(max_length=150, blank=True, null=True) - function = models.CharField(max_length=250, blank=True, null=True) - organism = models.CharField(max_length=30) - email = models.CharField(max_length=254) - created_date = models.DateTimeField() - updated_date = models.DateTimeField() - sent_date = models.DateTimeField(blank=True, null=True) - done = models.BooleanField() - sent_file = models.TextField(blank=True, null=True) - history_id = models.AutoField(primary_key=True) - history_date = models.DateTimeField() - history_change_reason = models.CharField(max_length=100, blank=True, null=True) - history_type = models.CharField(max_length=1) - history_user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) - project_id = models.BigIntegerField(blank=True, null=True) - user_id = models.BigIntegerField(blank=True, null=True) - requested_document = models.CharField(max_length=30) - competence_urba = models.BooleanField() - du_en_cours = models.BooleanField() - - class Meta: - managed = False - db_table = "project_historicalrequest" - - -class ProjectProject(models.Model): - id = models.BigAutoField(primary_key=True) - name = models.CharField(max_length=100) - analyse_start_date = models.CharField(max_length=4) - analyse_end_date = models.CharField(max_length=4) - user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) - is_public = models.BooleanField() - look_a_like = models.CharField(max_length=250, blank=True, null=True) - created_date = models.DateTimeField() - updated_date = models.DateTimeField() - first_year_ocsge = models.IntegerField(blank=True, null=True) - last_year_ocsge = models.IntegerField(blank=True, null=True) - level = models.CharField(max_length=7) - land_id = models.CharField(max_length=255, blank=True, null=True) - land_type = models.CharField(max_length=7, blank=True, null=True) - cover_image = models.CharField(max_length=100, blank=True, null=True) - folder_name = models.CharField(max_length=15, blank=True, null=True) - territory_name = models.CharField(max_length=250, blank=True, null=True) - target_2031 = models.IntegerField() - theme_map_artif = models.CharField(max_length=100, blank=True, null=True) - theme_map_conso = models.CharField(max_length=100, blank=True, null=True) - theme_map_understand_artif = models.CharField(max_length=100, blank=True, null=True) - async_add_comparison_lands_done = models.BooleanField() - async_add_city_done = models.BooleanField() - async_cover_image_done = models.BooleanField() - async_find_first_and_last_ocsge_done = models.BooleanField() - async_generate_theme_map_artif_done = models.BooleanField() - async_generate_theme_map_conso_done = models.BooleanField() - async_theme_map_understand_artif_done = models.BooleanField() - async_set_combined_emprise_done = models.BooleanField() - available_millesimes = models.CharField(max_length=255, blank=True, null=True) - async_theme_map_gpu_done = models.BooleanField() - theme_map_gpu = models.CharField(max_length=100, blank=True, null=True) - public_keys = models.CharField(max_length=255, blank=True, null=True) - async_theme_map_fill_gpu_done = models.BooleanField() - theme_map_fill_gpu = models.CharField(max_length=100, blank=True, null=True) - async_ocsge_coverage_status_done = models.BooleanField() - ocsge_coverage_status = models.CharField(max_length=20) - - class Meta: - managed = False - db_table = "project_project" - - -class ProjectProjectcommune(models.Model): - id = models.BigAutoField(primary_key=True) - project = models.ForeignKey(ProjectProject, models.DO_NOTHING) - group_name = models.CharField(max_length=100, blank=True, null=True) - commune_id = models.TextField(blank=True, null=True) - - class Meta: - managed = False - db_table = "project_projectcommune" - - -class ProjectRequest(models.Model): - id = models.BigAutoField(primary_key=True) - first_name = models.CharField(max_length=150, blank=True, null=True) - last_name = models.CharField(max_length=150, blank=True, null=True) - function = models.CharField(max_length=250, blank=True, null=True) - organism = models.CharField(max_length=30) - email = models.CharField(max_length=254) - created_date = models.DateTimeField() - updated_date = models.DateTimeField() - sent_date = models.DateTimeField(blank=True, null=True) - done = models.BooleanField() - project = models.ForeignKey(ProjectProject, models.DO_NOTHING, blank=True, null=True) - user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) - sent_file = models.CharField(max_length=100, blank=True, null=True) - requested_document = models.CharField(max_length=30) - competence_urba = models.BooleanField() - du_en_cours = models.BooleanField() - - class Meta: - managed = False - db_table = "project_request" - - -class ProjectRnupackage(models.Model): - id = models.BigAutoField(primary_key=True) - file = models.CharField(max_length=100, blank=True, null=True) - created_at = models.DateTimeField() - updated_at = models.DateTimeField() - departement_official_id = models.CharField(unique=True, max_length=10) - app_version = models.CharField(max_length=10) - - class Meta: - managed = False - db_table = "project_rnupackage" - - -class ProjectRnupackagerequest(models.Model): - id = models.BigAutoField(primary_key=True) - departement_official_id = models.CharField(max_length=10) - email = models.CharField(max_length=254) - requested_at = models.DateTimeField() - requested_diagnostics_before_package_request = models.IntegerField() - account_created_for_package = models.BooleanField() - rnu_package = models.ForeignKey(ProjectRnupackage, models.DO_NOTHING, blank=True, null=True) - user = models.ForeignKey("UsersUser", models.DO_NOTHING, blank=True, null=True) - - class Meta: - managed = False - db_table = "project_rnupackagerequest" - - -class PublicDataArtifareazoneurba(models.Model): - zone_urba = models.CharField(blank=True, null=True) - year = models.IntegerField(blank=True, null=True) - departement = models.CharField(blank=True, null=True) - area = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_artifareazoneurba" - - -class PublicDataArtificialarea(models.Model): - commune_year_id = models.CharField(blank=True, null=True) - year = models.IntegerField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - departement = models.CharField(blank=True, null=True) - city = models.CharField(max_length=5, blank=True, null=True) - surface = models.FloatField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_artificialarea" - - -class PublicDataCerema(models.Model): - city_insee = models.CharField(blank=True, null=True) - city_name = models.CharField(max_length=50, blank=True, null=True) - region_name = models.CharField(max_length=35, blank=True, null=True) - region_id = models.CharField(max_length=2, blank=True, null=True) - dept_id = models.CharField(max_length=3, blank=True, null=True) - dept_name = models.CharField(max_length=30, blank=True, null=True) - epci_id = models.CharField(blank=True, null=True) - epci_name = models.CharField(max_length=230, blank=True, null=True) - scot = models.CharField(blank=True, null=True) - art09inc23 = models.FloatField(blank=True, null=True) - art09fer23 = models.FloatField(blank=True, null=True) - art09rou23 = models.FloatField(blank=True, null=True) - art09mix23 = models.FloatField(blank=True, null=True) - art09hab23 = models.FloatField(blank=True, null=True) - art09act23 = models.FloatField(blank=True, null=True) - art22inc23 = models.FloatField(blank=True, null=True) - art22fer23 = models.FloatField(blank=True, null=True) - art22rou23 = models.FloatField(blank=True, null=True) - art22mix23 = models.FloatField(blank=True, null=True) - art22hab23 = models.FloatField(blank=True, null=True) - art22act23 = models.FloatField(blank=True, null=True) - naf22art23 = models.FloatField(blank=True, null=True) - art21inc22 = models.FloatField(blank=True, null=True) - art21fer22 = models.FloatField(blank=True, null=True) - art21rou22 = models.FloatField(blank=True, null=True) - art21mix22 = models.FloatField(blank=True, null=True) - art21hab22 = models.FloatField(blank=True, null=True) - art21act22 = models.FloatField(blank=True, null=True) - naf21art22 = models.FloatField(blank=True, null=True) - art20inc21 = models.FloatField(blank=True, null=True) - art20fer21 = models.FloatField(blank=True, null=True) - art20rou21 = models.FloatField(blank=True, null=True) - art20mix21 = models.FloatField(blank=True, null=True) - art20hab21 = models.FloatField(blank=True, null=True) - art20act21 = models.FloatField(blank=True, null=True) - naf20art21 = models.FloatField(blank=True, null=True) - art19inc20 = models.FloatField(blank=True, null=True) - art19fer20 = models.FloatField(blank=True, null=True) - art19rou20 = models.FloatField(blank=True, null=True) - art19mix20 = models.FloatField(blank=True, null=True) - art19hab20 = models.FloatField(blank=True, null=True) - art19act20 = models.FloatField(blank=True, null=True) - naf19art20 = models.FloatField(blank=True, null=True) - art18inc19 = models.FloatField(blank=True, null=True) - art18fer19 = models.FloatField(blank=True, null=True) - art18rou19 = models.FloatField(blank=True, null=True) - art18mix19 = models.FloatField(blank=True, null=True) - art18hab19 = models.FloatField(blank=True, null=True) - art18act19 = models.FloatField(blank=True, null=True) - naf18art19 = models.FloatField(blank=True, null=True) - art17inc18 = models.FloatField(blank=True, null=True) - art17fer18 = models.FloatField(blank=True, null=True) - art17rou18 = models.FloatField(blank=True, null=True) - art17mix18 = models.FloatField(blank=True, null=True) - art17hab18 = models.FloatField(blank=True, null=True) - art17act18 = models.FloatField(blank=True, null=True) - naf17art18 = models.FloatField(blank=True, null=True) - art16inc17 = models.FloatField(blank=True, null=True) - art16fer17 = models.FloatField(blank=True, null=True) - art16rou17 = models.FloatField(blank=True, null=True) - art16mix17 = models.FloatField(blank=True, null=True) - art16hab17 = models.FloatField(blank=True, null=True) - art16act17 = models.FloatField(blank=True, null=True) - naf16art17 = models.FloatField(blank=True, null=True) - art15inc16 = models.FloatField(blank=True, null=True) - art15fer16 = models.FloatField(blank=True, null=True) - art15rou16 = models.FloatField(blank=True, null=True) - art15mix16 = models.FloatField(blank=True, null=True) - art15hab16 = models.FloatField(blank=True, null=True) - art15act16 = models.FloatField(blank=True, null=True) - naf15art16 = models.FloatField(blank=True, null=True) - art14inc15 = models.FloatField(blank=True, null=True) - art14fer15 = models.FloatField(blank=True, null=True) - art14rou15 = models.FloatField(blank=True, null=True) - art14mix15 = models.FloatField(blank=True, null=True) - art14hab15 = models.FloatField(blank=True, null=True) - art14act15 = models.FloatField(blank=True, null=True) - naf14art15 = models.FloatField(blank=True, null=True) - art13inc14 = models.FloatField(blank=True, null=True) - art13fer14 = models.FloatField(blank=True, null=True) - art13rou14 = models.FloatField(blank=True, null=True) - art13mix14 = models.FloatField(blank=True, null=True) - art13hab14 = models.FloatField(blank=True, null=True) - art13act14 = models.FloatField(blank=True, null=True) - naf13art14 = models.FloatField(blank=True, null=True) - art12inc13 = models.FloatField(blank=True, null=True) - art12fer13 = models.FloatField(blank=True, null=True) - art12rou13 = models.FloatField(blank=True, null=True) - art12mix13 = models.FloatField(blank=True, null=True) - art12hab13 = models.FloatField(blank=True, null=True) - art12act13 = models.FloatField(blank=True, null=True) - naf12art13 = models.FloatField(blank=True, null=True) - art11inc12 = models.FloatField(blank=True, null=True) - art11fer12 = models.FloatField(blank=True, null=True) - art11rou12 = models.FloatField(blank=True, null=True) - art11mix12 = models.FloatField(blank=True, null=True) - art11hab12 = models.FloatField(blank=True, null=True) - art11act12 = models.FloatField(blank=True, null=True) - naf11art12 = models.FloatField(blank=True, null=True) - art10inc11 = models.FloatField(blank=True, null=True) - art10fer11 = models.FloatField(blank=True, null=True) - art10rou11 = models.FloatField(blank=True, null=True) - art10mix11 = models.FloatField(blank=True, null=True) - art10hab11 = models.FloatField(blank=True, null=True) - art10act11 = models.FloatField(blank=True, null=True) - naf10art11 = models.FloatField(blank=True, null=True) - art09inc10 = models.FloatField(blank=True, null=True) - art09fer10 = models.FloatField(blank=True, null=True) - art09rou10 = models.FloatField(blank=True, null=True) - art09mix10 = models.FloatField(blank=True, null=True) - art09hab10 = models.FloatField(blank=True, null=True) - art09act10 = models.FloatField(blank=True, null=True) - naf09art10 = models.FloatField(blank=True, null=True) - naf11art21 = models.FloatField(blank=True, null=True) - naf11inc21 = models.FloatField(blank=True, null=True) - naf11fer21 = models.FloatField(blank=True, null=True) - naf11rou21 = models.FloatField(blank=True, null=True) - naf11mix21 = models.FloatField(blank=True, null=True) - naf11act21 = models.FloatField(blank=True, null=True) - naf11hab21 = models.FloatField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_cerema" - - -class PublicDataCommune(models.Model): - insee = models.CharField(max_length=5, blank=True, null=True) - departement_id = models.CharField(max_length=3, blank=True, null=True) - epci_id = models.CharField(blank=True, null=True) - ept_id = models.CharField(blank=True, null=True) - scot_id = models.CharField(blank=True, null=True) - first_millesime = models.IntegerField(blank=True, null=True) - last_millesime = models.IntegerField(blank=True, null=True) - name = models.CharField(max_length=50, blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - ocsge_available = models.BooleanField(blank=True, null=True) - surface_artif = models.FloatField(blank=True, null=True) - area = models.FloatField(blank=True, null=True) - consommation_correction_status = models.CharField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_commune" - - -class PublicDataCommuneEpci(models.Model): - commune_id = models.CharField(max_length=5, blank=True, null=True) - epci_id = models.CharField(max_length=9, blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_commune_epci" - - -class PublicDataCommunediff(models.Model): - year_old = models.IntegerField(blank=True, null=True) - year_new = models.IntegerField(blank=True, null=True) - new_artif = models.FloatField(blank=True, null=True) - new_natural = models.FloatField(blank=True, null=True) - city_id = models.CharField(max_length=5, blank=True, null=True) - net_artif = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_communediff" - - -class PublicDataCommunepop(models.Model): - id = models.BigAutoField(primary_key=True) - year = models.IntegerField() - pop = models.IntegerField(blank=True, null=True) - city_id = models.BigIntegerField() - household = models.IntegerField(blank=True, null=True) - household_change = models.IntegerField(blank=True, null=True) - pop_change = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_communepop" - - -class PublicDataCommunesol(models.Model): - year = models.IntegerField(blank=True, null=True) - surface = models.FloatField(blank=True, null=True) - city_id = models.CharField(max_length=5, blank=True, null=True) - matrix_id = models.BigIntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_communesol" - - -class PublicDataCouverturesol(models.Model): - id = models.BigAutoField(primary_key=True) - code = models.CharField(unique=True, max_length=8) - label = models.CharField(max_length=250) - parent = models.ForeignKey("self", models.DO_NOTHING, blank=True, null=True) - code_prefix = models.CharField(unique=True, max_length=10) - map_color = models.CharField(max_length=8, blank=True, null=True) - label_short = models.CharField(max_length=50, blank=True, null=True) - is_key = models.BooleanField() - - class Meta: - managed = False - db_table = "public_data_couverturesol" - - -class PublicDataCouvertureusagematrix(models.Model): - id = models.BigAutoField(primary_key=True) - is_artificial = models.BooleanField(blank=True, null=True) - couverture = models.ForeignKey(PublicDataCouverturesol, models.DO_NOTHING, blank=True, null=True) - usage = models.ForeignKey("PublicDataUsagesol", models.DO_NOTHING, blank=True, null=True) - is_natural = models.BooleanField(blank=True, null=True) - label = models.CharField(max_length=20) - is_impermeable = models.BooleanField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_couvertureusagematrix" - unique_together = (("couverture", "usage"),) - - -class PublicDataDepartement(models.Model): - source_id = models.CharField(max_length=3, blank=True, null=True) - name = models.CharField(max_length=30, blank=True, null=True) - region_id = models.CharField(max_length=2, blank=True, null=True) - ocsge_millesimes = models.TextField(blank=True, null=True) # This field type is a guess. - srid_source = models.IntegerField(blank=True, null=True) - is_artif_ready = models.BooleanField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_departement" - - -class PublicDataEpci(models.Model): - source_id = models.CharField(max_length=9, blank=True, null=True) - name = models.CharField(max_length=230, blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_epci" - - -class PublicDataEpciDepartements(models.Model): - epci_id = models.CharField(max_length=9, blank=True, null=True) - departement_id = models.CharField(max_length=3, blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_epci_departements" - - -class PublicDataLandconso(models.Model): - land_id = models.CharField(blank=True, null=True) - land_type = models.CharField(blank=True, null=True) - surface = models.FloatField(blank=True, null=True) - year = models.IntegerField(blank=True, null=True) - total = models.FloatField(blank=True, null=True) - activite = models.FloatField(blank=True, null=True) - habitat = models.FloatField(blank=True, null=True) - mixte = models.FloatField(blank=True, null=True) - route = models.FloatField(blank=True, null=True) - ferroviaire = models.FloatField(blank=True, null=True) - inconnu = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_landconso" - - -class PublicDataLandconsocomparison(models.Model): - relevance_level = models.CharField(blank=True, null=True) - land_type = models.CharField(blank=True, null=True) - land_id = models.CharField(blank=True, null=True) - from_year = models.IntegerField(blank=True, null=True) - to_year = models.IntegerField(blank=True, null=True) - total_median = models.FloatField(blank=True, null=True) - total_median_percent = models.FloatField(blank=True, null=True) - total_avg = models.FloatField(blank=True, null=True) - total_percent = models.FloatField(blank=True, null=True) - activite_median = models.FloatField(blank=True, null=True) - activite_median_percent = models.FloatField(blank=True, null=True) - activite_avg = models.FloatField(blank=True, null=True) - activite_percent = models.FloatField(blank=True, null=True) - habitat_median = models.FloatField(blank=True, null=True) - habitat_median_percent = models.FloatField(blank=True, null=True) - habitat_avg = models.FloatField(blank=True, null=True) - habitat_percent = models.FloatField(blank=True, null=True) - mixte_median = models.FloatField(blank=True, null=True) - mixte_median_percent = models.FloatField(blank=True, null=True) - mixte_avg = models.FloatField(blank=True, null=True) - mixte_percent = models.FloatField(blank=True, null=True) - route_median = models.FloatField(blank=True, null=True) - route_median_percent = models.FloatField(blank=True, null=True) - route_avg = models.FloatField(blank=True, null=True) - route_percent = models.FloatField(blank=True, null=True) - ferroviaire_median = models.FloatField(blank=True, null=True) - ferroviaire_median_percent = models.FloatField(blank=True, null=True) - ferroviaire_avg = models.FloatField(blank=True, null=True) - ferroviaire_percent = models.FloatField(blank=True, null=True) - inconnu_median = models.FloatField(blank=True, null=True) - inconnu_median_percent = models.FloatField(blank=True, null=True) - inconnu_avg = models.FloatField(blank=True, null=True) - inconnu_percent = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_landconsocomparison" - - -class PublicDataLandconsostats(models.Model): - land_id = models.CharField(blank=True, null=True) - land_type = models.CharField(blank=True, null=True) - comparison_level = models.CharField(blank=True, null=True) - comparison_id = models.CharField(blank=True, null=True) - from_year = models.IntegerField(blank=True, null=True) - to_year = models.IntegerField(blank=True, null=True) - total = models.FloatField(blank=True, null=True) - total_percent = models.FloatField(blank=True, null=True) - activite = models.FloatField(blank=True, null=True) - activite_percent = models.FloatField(blank=True, null=True) - habitat = models.FloatField(blank=True, null=True) - habitat_percent = models.FloatField(blank=True, null=True) - mixte = models.FloatField(blank=True, null=True) - mixte_percent = models.FloatField(blank=True, null=True) - route = models.FloatField(blank=True, null=True) - route_percent = models.FloatField(blank=True, null=True) - ferroviaire = models.FloatField(blank=True, null=True) - ferroviaire_percent = models.FloatField(blank=True, null=True) - inconnu = models.FloatField(blank=True, null=True) - inconnu_percent = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_landconsostats" - - -class PublicDataLandpop(models.Model): - land_id = models.CharField(blank=True, null=True) - land_type = models.CharField(blank=True, null=True) - year = models.IntegerField(blank=True, null=True) - evolution = models.FloatField(blank=True, null=True) - population = models.FloatField(blank=True, null=True) - source = models.CharField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_landpop" - - -class PublicDataLandpopcomparison(models.Model): - relevance_level = models.CharField(blank=True, null=True) - land_type = models.CharField(blank=True, null=True) - land_id = models.CharField(blank=True, null=True) - from_year = models.IntegerField(blank=True, null=True) - to_year = models.IntegerField(blank=True, null=True) - evolution_median = models.FloatField(blank=True, null=True) - evolution_median_percent = models.FloatField(blank=True, null=True) - evolution_avg = models.FloatField(blank=True, null=True) - evolution_percent = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_landpopcomparison" - - -class PublicDataLandpopstats(models.Model): - land_id = models.CharField(blank=True, null=True) - land_type = models.CharField(blank=True, null=True) - comparison_level = models.CharField(blank=True, null=True) - comparison_id = models.CharField(blank=True, null=True) - from_year = models.IntegerField(blank=True, null=True) - to_year = models.IntegerField(blank=True, null=True) - evolution = models.FloatField(blank=True, null=True) - evolution_percent = models.FloatField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_landpopstats" - - -class PublicDataOcsge(models.Model): - uuid = models.CharField(blank=True, null=True) - couverture = models.CharField(blank=True, null=True) - usage = models.CharField(blank=True, null=True) - year = models.IntegerField(blank=True, null=True) - id_source = models.CharField(blank=True, null=True) - is_artificial = models.BooleanField(blank=True, null=True) - surface = models.FloatField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - departement = models.CharField(blank=True, null=True) - is_impermeable = models.BooleanField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_ocsge" - - -class PublicDataOcsgediff(models.Model): - uuid = models.CharField(blank=True, null=True) - year_old = models.IntegerField(blank=True, null=True) - year_new = models.IntegerField(blank=True, null=True) - cs_new = models.CharField(blank=True, null=True) - cs_old = models.CharField(blank=True, null=True) - us_new = models.CharField(blank=True, null=True) - us_old = models.CharField(blank=True, null=True) - surface = models.FloatField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - departement = models.CharField(blank=True, null=True) - is_new_artif = models.BooleanField(blank=True, null=True) - is_new_natural = models.BooleanField(blank=True, null=True) - is_new_impermeable = models.BooleanField(blank=True, null=True) - is_new_not_impermeable = models.BooleanField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_ocsgediff" - - -class PublicDataRegion(models.Model): - source_id = models.CharField(max_length=2, blank=True, null=True) - name = models.CharField(max_length=35, blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_region" - - -class PublicDataScot(models.Model): - siren = models.CharField(blank=True, null=True) - source_id = models.CharField(blank=True, null=True) - name = models.CharField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_scot" - - -class PublicDataScotDepartements(models.Model): - scot_id = models.CharField(blank=True, null=True) - departement_id = models.CharField(max_length=3, blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_scot_departements" - - -class PublicDataScotRegions(models.Model): - scot_id = models.CharField(blank=True, null=True) - region_id = models.CharField(max_length=2, blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_scot_regions" - - -class PublicDataSudocuh(models.Model): - code_insee = models.CharField(primary_key=True, max_length=200) - code_departement = models.CharField(max_length=200) - nom_region = models.CharField(max_length=200) - nom_commune = models.CharField(max_length=200) - collectivite_porteuse = models.CharField(max_length=200) - siren_epci = models.CharField(max_length=200, blank=True, null=True) - code_etat = models.CharField(max_length=2) - du_opposable = models.CharField(max_length=200) - du_en_cours = models.CharField(max_length=200, blank=True, null=True) - code_bcsi = models.CharField(max_length=200) - etat_commune = models.CharField(max_length=200) - etat_detaille = models.CharField(max_length=200) - prescription_du_en_vigueur = models.DateField(blank=True, null=True) - approbation_du_en_vigueur = models.DateField(blank=True, null=True) - executoire_du_en_vigueur = models.DateField(blank=True, null=True) - prescription_proc_en_cours = models.DateField(blank=True, null=True) - population_municipale = models.IntegerField() - population_totale = models.IntegerField() - superficie = models.FloatField() - - class Meta: - managed = False - db_table = "public_data_sudocuh" - - -class PublicDataSudocuhepci(models.Model): - id = models.BigAutoField(primary_key=True) - nom_region = models.CharField(max_length=200) - code_departement = models.CharField(max_length=200) - nom_departement = models.CharField(max_length=200) - siren = models.CharField(max_length=200) - type_epci = models.CharField(max_length=200) - nom_epci = models.CharField(max_length=200) - date_creation_epci = models.DateField(blank=True, null=True) - epci_interdepartemental = models.BooleanField() - competence_plan = models.BooleanField() - competence_scot = models.BooleanField() - competence_plh = models.BooleanField() - obligation_plh = models.BooleanField() - nb_communes = models.IntegerField() - insee_pop_tot_2021 = models.IntegerField() - insee_pop_municipale = models.IntegerField() - insee_superficie = models.FloatField() - - class Meta: - managed = False - db_table = "public_data_sudocuhepci" - - -class PublicDataUsagesol(models.Model): - id = models.BigAutoField(primary_key=True) - code = models.CharField(unique=True, max_length=8) - label = models.CharField(max_length=250) - parent = models.ForeignKey("self", models.DO_NOTHING, blank=True, null=True) - code_prefix = models.CharField(unique=True, max_length=10) - map_color = models.CharField(max_length=8, blank=True, null=True) - label_short = models.CharField(max_length=50, blank=True, null=True) - is_key = models.BooleanField() - - class Meta: - managed = False - db_table = "public_data_usagesol" - - -class PublicDataZoneconstruite(models.Model): - uuid = models.CharField(blank=True, null=True) - id_source = models.CharField(blank=True, null=True) - millesime = models.IntegerField(blank=True, null=True) - year = models.IntegerField(blank=True, null=True) - surface = models.FloatField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - departement = models.CharField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_zoneconstruite" - - -class PublicDataZoneurba(models.Model): - checksum = models.CharField(blank=True, null=True) - libelle = models.CharField(blank=True, null=True) - libelong = models.CharField(blank=True, null=True) - idurba = models.CharField(blank=True, null=True) - typezone = models.CharField(blank=True, null=True) - partition = models.CharField(blank=True, null=True) - datappro = models.CharField(blank=True, null=True) - datvalid = models.CharField(blank=True, null=True) - area = models.FloatField(blank=True, null=True) - srid_source = models.IntegerField(blank=True, null=True) - mpoly = models.GeometryField(blank=True, null=True) - - class Meta: - managed = False - db_table = "public_data_zoneurba" - - -class UsersUser(models.Model): - id = models.BigAutoField(primary_key=True) - password = models.CharField(max_length=128) - last_login = models.DateTimeField(blank=True, null=True) - is_superuser = models.BooleanField() - first_name = models.CharField(max_length=150) - last_name = models.CharField(max_length=150) - email = models.CharField(unique=True, max_length=254) - is_staff = models.BooleanField() - is_active = models.BooleanField() - date_joined = models.DateTimeField() - email_checked = models.DateTimeField(blank=True, null=True) - function = models.CharField(max_length=250) - organism = models.CharField(max_length=250) - organism_group = models.CharField(max_length=250, blank=True, null=True) - created_at = models.DateTimeField(blank=True, null=True) - updated_at = models.DateTimeField(blank=True, null=True) - - class Meta: - managed = False - db_table = "users_user" - - -class UsersUserGroups(models.Model): - id = models.BigAutoField(primary_key=True) - user = models.ForeignKey(UsersUser, models.DO_NOTHING) - group = models.ForeignKey(AuthGroup, models.DO_NOTHING) - - class Meta: - managed = False - db_table = "users_user_groups" - unique_together = (("user", "group"),) - - -class UsersUserUserPermissions(models.Model): - id = models.BigAutoField(primary_key=True) - user = models.ForeignKey(UsersUser, models.DO_NOTHING) - permission = models.ForeignKey(AuthPermission, models.DO_NOTHING) - - class Meta: - managed = False - db_table = "users_user_user_permissions" - unique_together = (("user", "permission"),) From e083bca781d14ea7de1a8bfdc53f3ab1287dc386 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 23:06:42 +0100 Subject: [PATCH 09/14] feat(conso): simplify ConsommationProgressionService( --- .../progression/ConsommationProgression.py | 1 - .../ConsommationProgressionService.py | 99 +++++-------------- public_data/domain/containers.py | 1 - .../ConsommationProgressionService.py | 0 4 files changed, 23 insertions(+), 78 deletions(-) create mode 100644 public_data/infra/consommation/progression/ConsommationProgressionService.py diff --git a/public_data/domain/consommation/progression/ConsommationProgression.py b/public_data/domain/consommation/progression/ConsommationProgression.py index 03a0c21a5..83d47150e 100644 --- a/public_data/domain/consommation/progression/ConsommationProgression.py +++ b/public_data/domain/consommation/progression/ConsommationProgression.py @@ -20,7 +20,6 @@ class AnnualConsommation: class ConsommationProgressionAggregation: start_date: int end_date: int - communes_code_insee: list[str] consommation: list[AnnualConsommation] diff --git a/public_data/domain/consommation/progression/ConsommationProgressionService.py b/public_data/domain/consommation/progression/ConsommationProgressionService.py index 7a90d74df..9e0780387 100644 --- a/public_data/domain/consommation/progression/ConsommationProgressionService.py +++ b/public_data/domain/consommation/progression/ConsommationProgressionService.py @@ -1,96 +1,43 @@ -from django.db.models import Sum -from django.db.models.query import QuerySet - -from public_data.domain.ClassCacher import ClassCacher -from public_data.models import Cerema, Commune, Land - -from .ConsommationProgression import ( +from public_data.domain.consommation.progression.ConsommationProgression import ( AnnualConsommation, ConsommationProgressionAggregation, ConsommationProgressionLand, ) +from public_data.models import Land, LandConso class ConsommationProgressionService: - def __init__(self, class_cacher: ClassCacher): - self.class_cacher = class_cacher - - def get_by_communes_aggregation( + def get_by_land( self, - communes: QuerySet[Commune], + land: Land, start_date: int, end_date: int, ) -> ConsommationProgressionAggregation: - communes_key = "-".join(communes.values_list("insee", flat=True)) - key = f"{start_date}-{end_date}-{communes_key}" - - if self.class_cacher.exists(key): - return self.class_cacher.get(key) - - artif_fields = Cerema.get_art_field( - start=str(start_date), - end=str(end_date), - ) - determinants = [ - "hab", - "act", - "mix", - "rou", - "fer", - "inc", - ] - years = [str(i) for i in range(start_date, end_date + 1)] - determinants_fields = [ - f"art{year[-2:]}{det}{str(int(year) + 1)[-2:]}" for year in years for det in determinants - ] - - fields = artif_fields + determinants_fields - - qs = Cerema.objects.filter(city_insee__in=communes.values("insee")) - args = {field: Sum(field) for field in fields} - qs = qs.aggregate(**args) - - results = {} - - surface = float(sum([commune.area for commune in communes])) - - for key, val in qs.items(): - year = int(f"20{key[3:5]}") - - if year not in results: - results[year] = {} - - if key in artif_fields: - results[year]["total"] = val - - if key in determinants_fields: - det = key[5:8] - results[year][det] = val - - progression = ConsommationProgressionAggregation( + conso = LandConso.objects.filter( + land_id=land.id, + land_type=land.land_type, + year__gte=start_date, + year__lte=end_date, + ).order_by("year") + return ConsommationProgressionAggregation( start_date=start_date, end_date=end_date, - communes_code_insee=[commune.insee for commune in communes], consommation=[ AnnualConsommation( - year=year, - total=results[year]["total"] / 10000, - habitat=results[year]["hab"] / 10000, - activite=results[year]["act"] / 10000, - mixte=results[year]["mix"] / 10000, - route=results[year]["rou"] / 10000, - ferre=results[year]["fer"] / 10000, - non_reseigne=results[year]["inc"] / 10000, - per_mille_of_area=results[year]["total"] / 10000 / surface * 1000, + year=c.year, + habitat=c.habitat, + activite=c.activite, + mixte=c.mixte, + route=c.route, + ferre=c.ferroviaire, + non_reseigne=c.inconnu, + total=c.total, + per_mille_of_area=c.total / land.area * 1000, ) - for year in results + for c in conso ], ) - self.class_cacher.set(key, value=progression) - - return progression - def get_by_lands( self, lands: list[Land], @@ -108,8 +55,8 @@ def get_by_lands( land=land, start_date=start_date, end_date=end_date, - consommation=self.get_by_communes_aggregation( - communes=land.get_cities(), + consommation=self.get_by_land( + land=land, start_date=start_date, end_date=end_date, ).consommation, diff --git a/public_data/domain/containers.py b/public_data/domain/containers.py index 702328d09..c5ef0da73 100644 --- a/public_data/domain/containers.py +++ b/public_data/domain/containers.py @@ -19,5 +19,4 @@ class PublicDataContainer(containers.DeclarativeContainer): consommation_progression_service = providers.Factory( ConsommationProgressionService, - class_cacher=class_cacher, ) diff --git a/public_data/infra/consommation/progression/ConsommationProgressionService.py b/public_data/infra/consommation/progression/ConsommationProgressionService.py new file mode 100644 index 000000000..e69de29bb From 60557cf6357186c0434a84d5d7400e0f488783b2 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 23:09:53 +0100 Subject: [PATCH 10/14] chore(project): remove get_look_a_like_conso_per_year --- project/models/project_base.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/project/models/project_base.py b/project/models/project_base.py index c4c7ba062..a4ac3aa90 100644 --- a/project/models/project_base.py +++ b/project/models/project_base.py @@ -721,17 +721,6 @@ def get_city_conso_per_year(self, group_name=None): """ return self.get_land_conso_per_year("city_name", group_name=group_name) - def get_look_a_like_conso_per_year(self): - """Return same data as get_conso_per_year but for land listed in - look_a_like property""" - return { - land.name: land.get_conso_per_year( - self.analyse_start_date, - self.analyse_end_date, - ) - for land in self.get_look_a_like() - } - def get_look_a_like_pop_change_per_year( self, criteria: Literal["pop", "household"] = "pop", From 4e0fa869d575b649c88bc26ce3eb1fd0d23de647 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 23:25:34 +0100 Subject: [PATCH 11/14] chore(project): simplify area --- project/models/project_base.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/project/models/project_base.py b/project/models/project_base.py index a4ac3aa90..aa25eebc3 100644 --- a/project/models/project_base.py +++ b/project/models/project_base.py @@ -417,27 +417,17 @@ def get_folder_name(self): @property def area(self) -> float: - """ - The area of the combined emprise of the project in hectare. - As this value should not change after the creation of a project, - we cache it for an arbitrary long time. - """ cache_key = f"project/{self.id}/area" if cache.has_key(cache_key): return cache.get(cache_key) - total_area = 0 - - for emprise in self.emprise_set.all(): - total_area += emprise.mpoly.transform(emprise.srid_source, clone=True).area - - total_area /= 10000 + area = self.land.area ONE_MONTH = 60 * 60 * 24 * 30 - cache.set(key=cache_key, value=total_area, timeout=ONE_MONTH) + cache.set(key=cache_key, value=area, timeout=ONE_MONTH) - return total_area + return area @cached_property def __related_departements(self): @@ -1040,7 +1030,7 @@ def get_base_sol_artif(self, sol: Literal["couverture", "usage"] = "couverture") .annotate(surface=Sum("surface")) ) - @property + @cached_property def land(self) -> Commune | Departement | Epci | Region | Scot: return Land(self.get_public_key()).land From aad2444251624262f02703841b7eccff71f50c0c Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 23:27:07 +0100 Subject: [PATCH 12/14] chore(project): simplify project area --- project/models/project_base.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/project/models/project_base.py b/project/models/project_base.py index aa25eebc3..c793e89cf 100644 --- a/project/models/project_base.py +++ b/project/models/project_base.py @@ -417,17 +417,7 @@ def get_folder_name(self): @property def area(self) -> float: - cache_key = f"project/{self.id}/area" - - if cache.has_key(cache_key): - return cache.get(cache_key) - - area = self.land.area - - ONE_MONTH = 60 * 60 * 24 * 30 - cache.set(key=cache_key, value=area, timeout=ONE_MONTH) - - return area + return self.land.area @cached_property def __related_departements(self): From e8dc95eea78494a0e9b20e41a74728a0b1f77791 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Mon, 18 Nov 2024 23:27:41 +0100 Subject: [PATCH 13/14] chore(project): optimise combined_emprise --- project/models/project_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/models/project_base.py b/project/models/project_base.py index c793e89cf..379eb93f1 100644 --- a/project/models/project_base.py +++ b/project/models/project_base.py @@ -75,7 +75,7 @@ class Status(models.TextChoices): ) name = models.CharField("Nom", max_length=100, validators=[is_alpha_validator]) - @property + @cached_property def combined_emprise(self) -> MultiPolygon: return self.land.mpoly From 453d7416c367221b199707ce2c62492d31bd7fa9 Mon Sep 17 00:00:00 2001 From: "Alexis A." Date: Tue, 19 Nov 2024 00:01:14 +0100 Subject: [PATCH 14/14] feat(population): add dbt step to airflow dag( --- airflow/dags/ingest_population.py | 10 +++++++++- airflow/include/sql/sparte/models/majic/schema.yml | 4 ---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/airflow/dags/ingest_population.py b/airflow/dags/ingest_population.py index 32bcfc7c7..895bff2e5 100644 --- a/airflow/dags/ingest_population.py +++ b/airflow/dags/ingest_population.py @@ -4,6 +4,8 @@ import pandas as pd from airflow.decorators import dag, task from include.container import Container +from include.pools import DBT_POOL +from include.utils import get_dbt_command_from_directory from pendulum import datetime URL = "https://www.insee.fr/fr/statistiques/fichier/3698339/base-pop-historiques-1876-2021.xlsx" @@ -42,8 +44,14 @@ def ingest(path_on_bucket) -> int | None: os.remove(tmp_localpath) return row_count + @task.bash(pool=DBT_POOL) + def dbt_build() -> str: + return get_dbt_command_from_directory(cmd="dbt build -s +insee+") + path_on_bucket = download() - ingest(path_on_bucket) + ingest_task = ingest(path_on_bucket) + + path_on_bucket >> ingest_task >> dbt_build() ingest_population() diff --git a/airflow/include/sql/sparte/models/majic/schema.yml b/airflow/include/sql/sparte/models/majic/schema.yml index ce8b60065..0e1f8fccb 100644 --- a/airflow/include/sql/sparte/models/majic/schema.yml +++ b/airflow/include/sql/sparte/models/majic/schema.yml @@ -34,7 +34,6 @@ models: - name: departement data_tests: - not_null - - unique - has_all_departements - relationships: to: ref('departement') @@ -44,7 +43,6 @@ models: - name: epci data_tests: - not_null - - unique - has_all_epcis - relationships: to: ref('epci') @@ -54,7 +52,6 @@ models: - name: region data_tests: - not_null - - unique - has_all_regions - relationships: to: ref('region') @@ -64,7 +61,6 @@ models: - name: scot data_tests: - not_null - - unique - has_all_scots - relationships: to: ref('scot')