From 1d0cc46098d48c8caa98d1a05276cb67e0a77e47 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Fri, 10 May 2024 13:19:17 +0200 Subject: [PATCH] Fix validation issues Version bump --- dev/Untitled.ipynb | 41 ++++++++++++----------------------------- premise/__init__.py | 2 +- premise/new_database.py | 20 ++------------------ premise/steel.py | 7 ++++--- premise/validation.py | 20 +++++++++++++------- 5 files changed, 32 insertions(+), 58 deletions(-) diff --git a/dev/Untitled.ipynb b/dev/Untitled.ipynb index 9b100347..d95a77b6 100644 --- a/dev/Untitled.ipynb +++ b/dev/Untitled.ipynb @@ -12,7 +12,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "premise v.(2, 1, 0, 'dev3')\n", + "premise v.(2, 1, 0, 'dev5')\n", "+------------------------------------------------------------------+\n", "| Warning |\n", "+------------------------------------------------------------------+\n", @@ -79,7 +79,6 @@ " source_db=\"ecoinvent 3.9.1 cutoff\",\n", " source_version=\"3.9.1\",\n", " key='tUePmX_S5B8ieZkkM7WUU2CnO8SmShwmAeWK9x2rTFo=',\n", - " use_multiprocessing=False,\n", ")" ] }, @@ -93,7 +92,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Updating: external: 100%|█████████████| 13/13 [00:47<00:00, 3.66s/it]" + "Processing scenarios: 100%|█████████████| 1/1 [01:24<00:00, 84.58s/it]" ] }, { @@ -122,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "211a6853-25fe-4c87-999f-542a1266c678", "metadata": {}, "outputs": [ @@ -130,35 +129,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Write new database(s) to Brightway.\n", + "Write new database(s) to matrix.\n", "Running all checks...\n", - "Minor anomalies found: check the change report.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Writing activities to SQLite3 database:\n", - "0% [##############################] 100% | ETA: 00:00:00\n", - "Total time elapsed: 00:00:29\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Title: Writing activities to SQLite3 database:\n", - " Started: 04/30/2024 18:46:43\n", - " Finished: 04/30/2024 18:47:13\n", - " Total time elapsed: 00:00:29\n", - " CPU %: 93.90\n", - " Memory %: 9.58\n" + "Minor anomalies found: check the change report.\n", + "Matrices saved in /Users/romain/GitHub/premise/dev/export/remind/SSP2-PkBudg500/2050.\n", + "Generate scenario report.\n", + "Report saved under /Users/romain/GitHub/premise/dev/export/scenario_report.\n", + "Generate change report.\n", + "Report saved under /Users/romain/GitHub/premise/dev.\n" ] } ], "source": [ - "ndb.write_db_to_brightway(\"test heat 2\")" + "ndb.write_db_to_matrices()" ] }, { @@ -4415,7 +4398,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.11.8" } }, "nbformat": 4, diff --git a/premise/__init__.py b/premise/__init__.py index d67db1e5..2cbfc78e 100644 --- a/premise/__init__.py +++ b/premise/__init__.py @@ -1,5 +1,5 @@ __all__ = ("NewDatabase", "clear_cache", "get_regions_definition") -__version__ = (2, 1, 0, "dev5") +__version__ = (2, 1, 0, "dev6") from premise.new_database import NewDatabase diff --git a/premise/new_database.py b/premise/new_database.py index 47cd20e9..5afc969a 100644 --- a/premise/new_database.py +++ b/premise/new_database.py @@ -4,14 +4,10 @@ """ -import copy import logging -import multiprocessing import os import pickle from datetime import datetime -from multiprocessing import Pool as ProcessPool -from multiprocessing.pool import ThreadPool as Pool from pathlib import Path from typing import List, Union @@ -490,7 +486,6 @@ def __init__( keep_uncertainty_data=False, gains_scenario="CLE", use_absolute_efficiency=False, - use_multiprocessing=True, ) -> None: self.source = source_db self.version = check_db_version(source_version) @@ -498,7 +493,6 @@ def __init__( self.system_model = check_system_model(system_model) self.system_model_args = system_args self.use_absolute_efficiency = use_absolute_efficiency - self.multiprocessing = use_multiprocessing self.keep_uncertainty_data = keep_uncertainty_data # if version is anything other than 3.8 or 3.9 @@ -583,13 +577,8 @@ def _fetch_iam_data(scenario): self.database.extend(data) print("- Fetching IAM data") - # use multiprocessing to speed up the process - if self.multiprocessing: - with Pool(processes=multiprocessing.cpu_count()) as pool: - pool.map(_fetch_iam_data, self.scenarios) - else: - for scenario in self.scenarios: - _fetch_iam_data(scenario) + for scenario in self.scenarios: + _fetch_iam_data(scenario) print("Done!") @@ -1112,8 +1101,6 @@ def write_db_to_simapro(self, filepath: str = None): print("Write Simapro import file(s).") - # use multiprocessing to speed up the process - for scenario in self.scenarios: scenario = load_database(scenario) _prepare_database( @@ -1146,8 +1133,6 @@ def write_db_to_olca(self, filepath: str = None): print("Write Simapro import file(s) for OpenLCA.") - # use multiprocessing to speed up the process - for scenario in self.scenarios: scenario = load_database(scenario) _prepare_database( @@ -1179,7 +1164,6 @@ def write_datapackage( cache_fp = DIR_CACHED_DB / f"cached_{self.source}_inventories.pickle" raise ValueError(f"No cached inventories found at {cache_fp}.") - # use multiprocessing to speed up the process for scenario in self.scenarios: scenario = load_database(scenario) _prepare_database( diff --git a/premise/steel.py b/premise/steel.py index 33d6c2d0..c72e1c98 100644 --- a/premise/steel.py +++ b/premise/steel.py @@ -542,14 +542,16 @@ def adjust_process_efficiency(self, datasets): energy += sum( exc["amount"] * 26.4 for exc in ws.technosphere(dataset) - if "hard coal" in exc["name"] and exc["unit"] == "kilogram" + if "hard coal" in exc["name"] + and exc["unit"] == "kilogram" ) # add input of natural gas energy += sum( exc["amount"] * 36 for exc in ws.technosphere(dataset) - if "natural gas" in exc["name"] and exc["unit"] == "cubic meter" + if "natural gas" in exc["name"] + and exc["unit"] == "cubic meter" ) scaling_factor = max(9.0 / energy, scaling_factor) @@ -562,7 +564,6 @@ def adjust_process_efficiency(self, datasets): ws.either(*[ws.contains("name", x) for x in list_fuels]) ], biosphere_filters=[ws.contains("name", "Carbon dioxide, fossil")], - remove_uncertainty=False, ) # Update the comments diff --git a/premise/validation.py b/premise/validation.py index 7e1bb9c4..f7580ac0 100644 --- a/premise/validation.py +++ b/premise/validation.py @@ -12,6 +12,7 @@ from .filesystem_constants import DATA_DIR from .geomap import Geomap from .logger import create_logger +from .utils import rescale_exchanges logger = create_logger("validation") @@ -770,9 +771,9 @@ def check_heat_conversion_efficiency(self): message = f"Heat conversion efficiency is {efficiency:.2f}, expected to be less than 3.0. Corrected to 3.0." self.log_issue(ds, "heat conversion efficiency", message) - # scale it back to 3 - for exc in ds["exchanges"]: - exc["amount"] *= efficiency / 3.0 + scaling_factor = efficiency / 3.0 + rescale_exchanges(ds, scaling_factor) + expected_co2 *= scaling_factor co2 = sum( [ @@ -1184,11 +1185,16 @@ def check_geo_linking(self, exc_loc, dataset_loc): def check_electricity_mix(self): # check that the electricity mix in teh market datasets # corresponds to the IAM scenario projection + vars = [ + x for x in self.iam_data.electricity_markets.coords["variables"].values + if x.lower().startswith("hydro") + ] if self.year in self.iam_data.electricity_markets.coords["year"].values: + hydro_share = self.iam_data.electricity_markets.sel( - variables="Hydro", year=self.year - ) / self.iam_data.electricity_markets.sel( + variables=vars, year=self.year + ).sum(dim="variables") / self.iam_data.electricity_markets.sel( variables=[ v for v in self.iam_data.electricity_markets.variables.values @@ -1200,8 +1206,8 @@ def check_electricity_mix(self): ) else: hydro_share = self.iam_data.electricity_markets.sel( - variables="Hydro" - ).interp(year=self.year) / self.iam_data.electricity_markets.sel( + variables=vars + ).interp(year=self.year).sum(dim="variables") / self.iam_data.electricity_markets.sel( variables=[ v for v in self.iam_data.electricity_markets.variables.values