Fix validation issues

Version bump

dev/Untitled.ipynb

@@ -12,7 +12,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "premise v.(2, 1, 0, 'dev3')\n",
+      "premise v.(2, 1, 0, 'dev5')\n",
       "+------------------------------------------------------------------+\n",
       "| Warning                                                          |\n",
       "+------------------------------------------------------------------+\n",
@@ -79,7 +79,6 @@
     "        source_db=\"ecoinvent 3.9.1 cutoff\",\n",
     "        source_version=\"3.9.1\",\n",
     "        key='tUePmX_S5B8ieZkkM7WUU2CnO8SmShwmAeWK9x2rTFo=',\n",
-    "        use_multiprocessing=False,\n",
     ")"
    ]
   },
@@ -93,7 +92,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Updating: external: 100%|█████████████| 13/13 [00:47<00:00,  3.66s/it]"
+      "Processing scenarios: 100%|█████████████| 1/1 [01:24<00:00, 84.58s/it]"
      ]
     },
     {
@@ -122,43 +121,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "211a6853-25fe-4c87-999f-542a1266c678",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Write new database(s) to Brightway.\n",
+      "Write new database(s) to matrix.\n",
       "Running all checks...\n",
-      "Minor anomalies found: check the change report.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Writing activities to SQLite3 database:\n",
-      "0% [##############################] 100% | ETA: 00:00:00\n",
-      "Total time elapsed: 00:00:29\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Title: Writing activities to SQLite3 database:\n",
-      "  Started: 04/30/2024 18:46:43\n",
-      "  Finished: 04/30/2024 18:47:13\n",
-      "  Total time elapsed: 00:00:29\n",
-      "  CPU %: 93.90\n",
-      "  Memory %: 9.58\n"
+      "Minor anomalies found: check the change report.\n",
+      "Matrices saved in /Users/romain/GitHub/premise/dev/export/remind/SSP2-PkBudg500/2050.\n",
+      "Generate scenario report.\n",
+      "Report saved under /Users/romain/GitHub/premise/dev/export/scenario_report.\n",
+      "Generate change report.\n",
+      "Report saved under /Users/romain/GitHub/premise/dev.\n"
      ]
     }
    ],
    "source": [
-    "ndb.write_db_to_brightway(\"test heat 2\")"
+    "ndb.write_db_to_matrices()"
    ]
   },
   {
@@ -4415,7 +4398,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.8"
   }
  },
  "nbformat": 4,

premise/__init__.py

@@ -1,5 +1,5 @@
 __all__ = ("NewDatabase", "clear_cache", "get_regions_definition")
-__version__ = (2, 1, 0, "dev5")
+__version__ = (2, 1, 0, "dev6")
 
 
 from premise.new_database import NewDatabase

premise/new_database.py

@@ -4,14 +4,10 @@
 
 """
 
-import copy
 import logging
-import multiprocessing
 import os
 import pickle
 from datetime import datetime
-from multiprocessing import Pool as ProcessPool
-from multiprocessing.pool import ThreadPool as Pool
 from pathlib import Path
 from typing import List, Union
 
@@ -490,15 +486,13 @@ def __init__(
         keep_uncertainty_data=False,
         gains_scenario="CLE",
         use_absolute_efficiency=False,
-        use_multiprocessing=True,
     ) -> None:
         self.source = source_db
         self.version = check_db_version(source_version)
         self.source_type = source_type
         self.system_model = check_system_model(system_model)
         self.system_model_args = system_args
         self.use_absolute_efficiency = use_absolute_efficiency
-        self.multiprocessing = use_multiprocessing
         self.keep_uncertainty_data = keep_uncertainty_data
 
         # if version is anything other than 3.8 or 3.9
@@ -583,13 +577,8 @@ def _fetch_iam_data(scenario):
             self.database.extend(data)
 
         print("- Fetching IAM data")
-        # use multiprocessing to speed up the process
-        if self.multiprocessing:
-            with Pool(processes=multiprocessing.cpu_count()) as pool:
-                pool.map(_fetch_iam_data, self.scenarios)
-        else:
-            for scenario in self.scenarios:
-                _fetch_iam_data(scenario)
+        for scenario in self.scenarios:
+            _fetch_iam_data(scenario)
 
         print("Done!")
 
@@ -1112,8 +1101,6 @@ def write_db_to_simapro(self, filepath: str = None):
 
         print("Write Simapro import file(s).")
 
-        # use multiprocessing to speed up the process
-
         for scenario in self.scenarios:
             scenario = load_database(scenario)
             _prepare_database(
@@ -1146,8 +1133,6 @@ def write_db_to_olca(self, filepath: str = None):
 
         print("Write Simapro import file(s) for OpenLCA.")
 
-        # use multiprocessing to speed up the process
-
         for scenario in self.scenarios:
             scenario = load_database(scenario)
             _prepare_database(
@@ -1179,7 +1164,6 @@ def write_datapackage(
             cache_fp = DIR_CACHED_DB / f"cached_{self.source}_inventories.pickle"
             raise ValueError(f"No cached inventories found at {cache_fp}.")
 
-        # use multiprocessing to speed up the process
         for scenario in self.scenarios:
             scenario = load_database(scenario)
             _prepare_database(

premise/steel.py

@@ -542,14 +542,16 @@ def adjust_process_efficiency(self, datasets):
                     energy += sum(
                         exc["amount"] * 26.4
                         for exc in ws.technosphere(dataset)
-                        if "hard coal" in exc["name"] and exc["unit"] == "kilogram"
+                        if "hard coal" in exc["name"]
+                        and exc["unit"] == "kilogram"
                     )
 
                     # add input of natural gas
                     energy += sum(
                         exc["amount"] * 36
                         for exc in ws.technosphere(dataset)
-                        if "natural gas" in exc["name"] and exc["unit"] == "cubic meter"
+                        if "natural gas" in exc["name"]
+                        and exc["unit"] == "cubic meter"
                     )
 
                     scaling_factor = max(9.0 / energy, scaling_factor)
@@ -562,7 +564,6 @@ def adjust_process_efficiency(self, datasets):
                         ws.either(*[ws.contains("name", x) for x in list_fuels])
                     ],
                     biosphere_filters=[ws.contains("name", "Carbon dioxide, fossil")],
-                    remove_uncertainty=False,
                 )
 
                 # Update the comments

premise/validation.py

@@ -12,6 +12,7 @@
 from .filesystem_constants import DATA_DIR
 from .geomap import Geomap
 from .logger import create_logger
+from .utils import rescale_exchanges
 
 logger = create_logger("validation")
 
@@ -770,9 +771,9 @@ def check_heat_conversion_efficiency(self):
                     message = f"Heat conversion efficiency is {efficiency:.2f}, expected to be less than 3.0. Corrected to 3.0."
                     self.log_issue(ds, "heat conversion efficiency", message)
 
-                    # scale it back to 3
-                    for exc in ds["exchanges"]:
-                        exc["amount"] *= efficiency / 3.0
+                    scaling_factor = efficiency / 3.0
+                    rescale_exchanges(ds, scaling_factor)
+                    expected_co2 *= scaling_factor
 
                 co2 = sum(
                     [
@@ -1184,11 +1185,16 @@ def check_geo_linking(self, exc_loc, dataset_loc):
     def check_electricity_mix(self):
         # check that the electricity mix in teh market datasets
         # corresponds to the IAM scenario projection
+        vars = [
+            x for x in self.iam_data.electricity_markets.coords["variables"].values
+            if x.lower().startswith("hydro")
+        ]
 
         if self.year in self.iam_data.electricity_markets.coords["year"].values:
+
             hydro_share = self.iam_data.electricity_markets.sel(
-                variables="Hydro", year=self.year
-            ) / self.iam_data.electricity_markets.sel(
+                variables=vars, year=self.year
+            ).sum(dim="variables") / self.iam_data.electricity_markets.sel(
                 variables=[
                     v
                     for v in self.iam_data.electricity_markets.variables.values
@@ -1200,8 +1206,8 @@ def check_electricity_mix(self):
             )
         else:
             hydro_share = self.iam_data.electricity_markets.sel(
-                variables="Hydro"
-            ).interp(year=self.year) / self.iam_data.electricity_markets.sel(
+                variables=vars
+            ).interp(year=self.year).sum(dim="variables") / self.iam_data.electricity_markets.sel(
                 variables=[
                     v
                     for v in self.iam_data.electricity_markets.variables.values