From e35f65151805c786593c41c2cd8f1142c1f6f449 Mon Sep 17 00:00:00 2001 From: cpschau <124347782+cpschau@users.noreply.github.com> Date: Thu, 12 Sep 2024 19:20:07 +0200 Subject: [PATCH] Update NUTS3 shapes (#1286) * increase nuts3 shape resolution * retrieve and use shapes with EPSG:4326 * co-retrieve nuts2 shapes; add retries * added release note * changed doc/data_sources --- doc/data_sources.rst | 14 +++++++------- doc/release_notes.rst | 2 ++ rules/build_electricity.smk | 2 +- rules/build_sector.smk | 2 +- rules/retrieve.smk | 29 +++++++++++++++++++++++++++-- scripts/build_biomass_potentials.py | 4 ++-- scripts/build_shapes.py | 1 - 7 files changed, 40 insertions(+), 14 deletions(-) mode change 100644 => 100755 rules/build_electricity.smk mode change 100644 => 100755 rules/retrieve.smk mode change 100644 => 100755 scripts/build_shapes.py diff --git a/doc/data_sources.rst b/doc/data_sources.rst index ecd72458d..ac4eb379c 100644 --- a/doc/data_sources.rst +++ b/doc/data_sources.rst @@ -61,13 +61,6 @@ scope to reduce file size, or are not provided through stable URLs elsewhere. - **License:** CC-BY 4.0 (`reference `__) - **Description:** Total GHG emissions and removals in the EU. -``data/bundle/nuts`` - -- **Source:** GISCO -- **Link:** https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ -- **License:** `custom `__ -- **Description:** Europe's NUTS administrative regions. - ``data/bundle/emobility`` - **Source:** German Federal Highway Research Institute (BASt) @@ -127,6 +120,13 @@ Specific retrieval rules Data in this section is retrieved and extracted in rules specified in ``rules/retrieve.smk``. +``data/nuts`` + +- **Source:** GISCO +- **Link:** https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ +- **License:** `custom `__ +- **Description:** Europe's NUTS administrative regions. + ``data/ENSPRESO_BIOMASS.xlsx`` - **Source:** European Commission Joint Research Centre (JRC) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index db1393805..ce3c2cdb1 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -18,6 +18,8 @@ Release Notes * Updated district heating supply temperatures based on `Euroheat's DHC Market Outlook 2024`__ and `AGFW-Hauptbericht 2022 `__. `min_forward_temperature` and `return_temperature` (not given by Euroheat) are extrapolated based on German values. +* Increased the resolution of NUTS3 and NUTS2 shapes from 1:60M to 1:3M, with data now directly retrieved from GISCO + * Made the overdimensioning factor for heating systems specific for central/decentral heating, defaults to no overdimensionining for central heating and no changes to decentral heating compared to previous version. * bugfix: The carrier of stores was silently overwritten by their bus_carrier as a side effect when building the co2 constraints diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk old mode 100644 new mode 100755 index 8c3ce32d6..3df4422e1 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -98,7 +98,7 @@ rule build_shapes: input: naturalearth=ancient("data/naturalearth/ne_10m_admin_0_countries_deu.shp"), eez=ancient("data/eez/World_EEZ_v12_20231025_LR/eez_v12_lowres.gpkg"), - nuts3=ancient("data/bundle/NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp"), + nuts3=ancient("data/nuts/NUTS_RG_03M_2013_4326_LEVL_3.geojson"), nuts3pop=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), nuts3gdp=ancient("data/bundle/nama_10r_3gdp.tsv.gz"), ch_cantons=ancient("data/ch_cantons.csv"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b3cfc4afa..ccd4243e7 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -405,7 +405,7 @@ rule build_biomass_potentials: input: enspreso_biomass="data/ENSPRESO_BIOMASS.xlsx", eurostat="data/eurostat/Balances-April2023", - nuts2="data/bundle/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", + nuts2="data/nuts/NUTS_RG_03M_2013_4326_LEVL_2.geojson", regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/ch_cantons.csv"), diff --git a/rules/retrieve.smk b/rules/retrieve.smk old mode 100644 new mode 100755 index 844618e03..c842e61fc --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -5,6 +5,7 @@ import requests from datetime import datetime, timedelta from shutil import move, unpack_archive +from zipfile import ZipFile if config["enable"].get("retrieve", "auto") == "auto": config["enable"]["retrieve"] = has_internet_access() @@ -16,12 +17,10 @@ if config["enable"]["retrieve"] is False: if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True): datafiles = [ "je-e-21.03.02.xls", - "NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp", "nama_10r_3popgdp.tsv.gz", "nama_10r_3gdp.tsv.gz", "corine/g250_clc06_V18_5.tif", "eea/UNFCCC_v23.csv", - "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", "emobility/KFZ__count", "emobility/Pkw__count", "h2_salt_caverns_GWh_per_sqkm.geojson", @@ -77,6 +76,32 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", "../scripts/retrieve_eurostat_household_data.py" +if config["enable"]["retrieve"]: + + rule retrieve_nuts_shapes: + input: + shapes=storage( + "https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-03m.geojson.zip" + ), + output: + shapes_level_3="data/nuts/NUTS_RG_03M_2013_4326_LEVL_3.geojson", + shapes_level_2="data/nuts/NUTS_RG_03M_2013_4326_LEVL_2.geojson", + params: + zip_file="data/nuts/ref-nuts-2013-03m.geojson.zip", + run: + os.rename(input.shapes, params.zip_file) + with ZipFile(params.zip_file, "r") as zip_ref: + for level in ["LEVL_3", "LEVL_2"]: + filename = f"NUTS_RG_03M_2013_4326_{level}.geojson" + zip_ref.extract(filename, Path(output.shapes_level_3).parent) + extracted_file = Path(output.shapes_level_3).parent / filename + extracted_file.rename( + getattr(output, f"shapes_level_{level[-1]}") + ) + os.remove(params.zip_file) + + + if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True): rule retrieve_cutout: diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index a3c51292c..bb56ebedf 100755 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -193,7 +193,7 @@ def build_nuts2_shapes(): - consistently name ME, MK """ nuts2 = gpd.GeoDataFrame( - gpd.read_file(snakemake.input.nuts2).set_index("id").geometry + gpd.read_file(snakemake.input.nuts2).set_index("NUTS_ID").geometry ) countries = gpd.read_file(snakemake.input.country_shapes).set_index("name") @@ -345,7 +345,7 @@ def add_unsustainable_potentials(df): snakemake = mock_snakemake( "build_biomass_potentials", simpl="", - clusters="38", + clusters="39", planning_horizons=2050, ) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py old mode 100644 new mode 100755 index 2370f2aef..29eb91478 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -150,7 +150,6 @@ def country_cover(country_shapes, eez_shapes=None): def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): df = gpd.read_file(nuts3) - df = df.loc[df["STAT_LEVL_"] == 3] df["geometry"] = df["geometry"].map(_simplify_polys) df = df.rename(columns={"NUTS_ID": "id"})[["id", "geometry"]].set_index("id")