From 9c35e8cea1cd3316820137d0b06db998044a4978 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 3 Dec 2024 16:36:21 +0100 Subject: [PATCH 01/45] TEMPORARY Run 'transport' CI workflow on PR branch --- .github/workflows/transport.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/transport.yaml b/.github/workflows/transport.yaml index 1b3a16a96..21f1fd9e1 100644 --- a/.github/workflows/transport.yaml +++ b/.github/workflows/transport.yaml @@ -18,8 +18,8 @@ env: on: # Uncomment these lines for debugging, but leave them commented on 'main' - # pull_request: - # branches: [ main ] + pull_request: + branches: [ main ] # push: # branches: [ main ] schedule: From 2447e47f1dd19e2c7c16cac96c46e0b7ab54f952 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 3 Dec 2024 16:37:22 +0100 Subject: [PATCH 02/45] TEMPORARY Use branch for iiasa/message_data#582 --- .github/workflows/transport.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/transport.yaml b/.github/workflows/transport.yaml index 21f1fd9e1..a757c68aa 100644 --- a/.github/workflows/transport.yaml +++ b/.github/workflows/transport.yaml @@ -3,7 +3,7 @@ name: MESSAGEix-Transport env: # The repository, ref (branch), and workflow file name to dispatch target-repo: iiasa/message_data - target-ref: dev + target-ref: enh/transport-2024-W47 target-workflow: transport # Starting point of the workflow. From aba02fdd72abefd91ba8fd5ec7b6848c3d3ccf97 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Jan 2025 17:40:09 +0100 Subject: [PATCH 03/45] Include genno KeyLike in .types --- message_ix_models/types.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/message_ix_models/types.py b/message_ix_models/types.py index 963300ec2..6cafc2c5d 100644 --- a/message_ix_models/types.py +++ b/message_ix_models/types.py @@ -5,6 +5,14 @@ import pandas as pd import sdmx.model.common +from genno.core.key import KeyLike # TODO Import from genno.types, when possible + +__all__ = [ + "KeyLike", + "MaintainableArtefactArgs", + "MutableParameterData", + "ParameterData", +] #: Collection of :mod:`message_ix` or :mod:`ixmp` parameter data. Keys should be #: parameter names (:class:`str`), and values should be data frames with the same From 8a8ba971050b09999a49acede42b425647f0be74 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Jan 2025 18:01:44 +0100 Subject: [PATCH 04/45] Satisfy mypy 1.14.1 in .model.transport --- message_ix_models/model/transport/config.py | 5 ++-- message_ix_models/model/transport/operator.py | 23 ++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/message_ix_models/model/transport/config.py b/message_ix_models/model/transport/config.py index 0daeac8ed..945a1caee 100644 --- a/message_ix_models/model/transport/config.py +++ b/message_ix_models/model/transport/config.py @@ -1,7 +1,6 @@ import logging from dataclasses import InitVar, dataclass, field, replace -from enum import Enum -from typing import TYPE_CHECKING, Literal, Optional, Union +from typing import TYPE_CHECKING, Any, Literal, Optional, Union import message_ix from genno import Quantity @@ -226,7 +225,7 @@ class Config(ConfigHelper): #: #: :mod:`.transport.build` and :mod:`.transport.report` code will respond to these #: settings in documented ways. - project: dict[str, Enum] = field( + project: dict[str, Any] = field( default_factory=lambda: dict( futures=FUTURES_SCENARIO.BASE, navigate=NAVIGATE_SCENARIO.REF ) diff --git a/message_ix_models/model/transport/operator.py b/message_ix_models/model/transport/operator.py index 6917456d8..9f036b1f1 100644 --- a/message_ix_models/model/transport/operator.py +++ b/message_ix_models/model/transport/operator.py @@ -6,7 +6,7 @@ from functools import partial, reduce from itertools import pairwise, product from operator import gt, le, lt -from typing import TYPE_CHECKING, Any, Hashable, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Hashable, Optional, Union, cast import genno import numpy as np @@ -238,10 +238,17 @@ def broadcast_wildcard( def broadcast_t_c_l( technologies: list[Code], commodities: list[Union[Code, str]], - kind: Literal["input", "output"], + kind: str, default_level: Optional[str] = None, ) -> "AnyQuantity": - """Return a Quantity for broadcasting dimension (t) to (c, l) for `kind`.""" + """Return a Quantity for broadcasting dimension (t) to (c, l) for `kind`. + + Parameter + --------- + kind : + Either "input" or "output". + """ + assert kind in ("input", "output") # Convert list[Union[Code, str]] into an SDMX Codelist for simpler usage cl_commodity: "Codelist" = Codelist() @@ -285,7 +292,7 @@ def broadcast_t_c_l( def broadcast_y_yv_ya( - y: list[int], y_include: list[int], *, method: Literal["product", "zip"] = "product" + y: list[int], y_include: list[int], *, method: str = "product" ) -> "AnyQuantity": """Return a quantity for broadcasting y to (yv, ya). @@ -293,10 +300,14 @@ def broadcast_y_yv_ya( If :py:`"y::model"` is passed as `y_include`, this is equivalent to :attr:`.ScenarioInfo.yv_ya`. + + Parameters + ---------- + method : + Either "product" or "zip". """ dims = ["y", "yv", "ya"] - - func = product if method == "product" else zip + func = {"product": product, "zip": zip}[method] series = ( pd.DataFrame(func(y, y_include), columns=dims[1:]) .query("ya >= yv") From ee6a3c1ca2f3635a9432f34676593df32cd70338 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 3 Dec 2024 17:31:19 +0100 Subject: [PATCH 05/45] Add .util.sdmx.AnnotationsMixIn --- message_ix_models/util/sdmx.py | 60 ++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index 80ae204f4..d7120c8b4 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -2,6 +2,7 @@ import logging from collections.abc import Mapping +from dataclasses import dataclass, fields from datetime import datetime from enum import Enum, Flag from importlib.metadata import version @@ -12,25 +13,56 @@ import sdmx import sdmx.message from iam_units import registry -from sdmx.model.v21 import AnnotableArtefact, Annotation, Code, InternationalString +from sdmx.model import common from .common import package_data_path if TYPE_CHECKING: from os import PathLike + from typing import TypeVar - import sdmx.model.common from sdmx.message import StructureMessage + # TODO Use "from typing import Self" once Python 3.11 is the minimum supported + Self = TypeVar("Self", bound="AnnotationsMixIn") + log = logging.getLogger(__name__) -CodeLike = Union[str, Code] +CodeLike = Union[str, common.Code] + + +@dataclass +class AnnotationsMixIn: + """Mix-in for dataclasses to allow (de)serializing as SDMX annotations.""" + + # TODO Type with overrides: list → list + def get_annotations(self, _rtype: Union[type[list], type[dict]]): + result = [] + for f in fields(self): + anno_id = f.name.replace("_", "-") + result.append( + common.Annotation(id=anno_id, text=repr(getattr(self, f.name))) + ) + + if _rtype is list: + return result + else: + return dict(annotations=result) + + @classmethod + def from_obj(cls: type["Self"], obj: common.AnnotableArtefact) -> "Self": + args = [] + for f in fields(cls): + anno_id = f.name.replace("_", "-") + args.append(obj.eval_annotation(id=anno_id)) + + return cls(*args) # FIXME Reduce complexity from 13 → ≤11 def as_codes( # noqa: C901 data: Union[list[str], dict[str, CodeLike]], -) -> list[Code]: +) -> list[common.Code]: """Convert `data` to a :class:`list` of :class:`.Code` objects. Various inputs are accepted: @@ -40,7 +72,7 @@ def as_codes( # noqa: C901 further :class:`dict` with keys matching other Code attributes. """ # Assemble results as a dictionary - result: dict[str, Code] = {} + result: dict[str, common.Code] = {} if isinstance(data, list): # FIXME typing ignored temporarily for PR#9 @@ -50,7 +82,7 @@ def as_codes( # noqa: C901 for id, info in data.items(): # Pass through Code; convert other types to dict() - if isinstance(info, Code): + if isinstance(info, common.Code): result[info.id] = info continue elif isinstance(info, str): @@ -61,14 +93,16 @@ def as_codes( # noqa: C901 raise TypeError(info) # Create a Code object - code = Code( + code = common.Code( id=str(id), name=_info.pop("name", str(id).title()), ) # Store the description, if any try: - code.description = InternationalString(value=_info.pop("description")) + code.description = common.InternationalString( + value=_info.pop("description") + ) except KeyError: pass @@ -90,7 +124,9 @@ def as_codes( # noqa: C901 # Convert other dictionary (key, value) pairs to annotations for id, value in _info.items(): code.annotations.append( - Annotation(id=id, text=value if isinstance(value, str) else repr(value)) + common.Annotation( + id=id, text=value if isinstance(value, str) else repr(value) + ) ) result[code.id] = code @@ -98,7 +134,7 @@ def as_codes( # noqa: C901 return list(result.values()) -def eval_anno(obj: AnnotableArtefact, id: str): +def eval_anno(obj: common.AnnotableArtefact, id: str): """Retrieve the annotation `id` from `obj`, run :func:`eval` on its contents. .. deprecated:: 2023.9.12 @@ -220,9 +256,7 @@ def write(obj, base_dir: Optional["PathLike"] = None, basename: Optional[str] = log.info(f"Wrote {path}") -def register_agency( - agency: "sdmx.model.common.Agency", -) -> "sdmx.model.common.AgencyScheme": +def register_agency(agency: "common.Agency") -> "common.AgencyScheme": """Add `agency` to the :class:`.AgencyScheme` "IIASA_ECE:AGENCIES".""" # Read the existing agency scheme as_ = read("IIASA_ECE:AGENCIES") From 240736f3068e5b16279e7eb51a1845048cd01034 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 3 Dec 2024 17:33:15 +0100 Subject: [PATCH 06/45] Add .transport.config.refresh_cl_scenario() - Only write the file if it is different from a refreshed codelist. - Use ScenarioCodeAnnotations class. - Add Config.use_scenario_code() - Update base scenario URLs. --- message_ix_models/model/transport/config.py | 93 ++++++++++++++++----- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/message_ix_models/model/transport/config.py b/message_ix_models/model/transport/config.py index 945a1caee..ba4ebd774 100644 --- a/message_ix_models/model/transport/config.py +++ b/message_ix_models/model/transport/config.py @@ -12,9 +12,10 @@ from message_ix_models.report.util import as_quantity from message_ix_models.util import identify_nodes, package_data_path from message_ix_models.util.config import ConfigHelper +from message_ix_models.util.sdmx import AnnotationsMixIn if TYPE_CHECKING: - from sdmx.model.common import Codelist + from sdmx.model import common log = logging.getLogger(__name__) @@ -419,9 +420,32 @@ def set_navigate_scenario(self, value: Optional[str]) -> None: self.project.update(navigate=s) self.check() + def use_scenario_code(self, code: "common.Code") -> None: + """Update settings given a `code` with :class:`ScenarioCodeAnnotations`.""" + sca = ScenarioCodeAnnotations.from_obj(code) + + # Look up the SSP_2024 Enum + self.ssp = SSP_2024.by_urn(sca.SSP_URN) + + # Store settings on the context + self.project["LED"] = sca.is_LED_scenario + self.project["EDITS"] = {"activity": sca.EDITS_activity_id} + + self.base_scenario_url = sca.base_scenario_URL + + +@dataclass +class ScenarioCodeAnnotations(AnnotationsMixIn): + """Set of annotations appearing on each Code in ``CL_TRANSPORT_SCENARIO``.""" + + SSP_URN: str + is_LED_scenario: bool + EDITS_activity_id: Optional[str] + base_scenario_URL: str + -def get_cl_scenario() -> "Codelist": - """Generate ``Codelist=IIASA_ECE:CL_TRANSPORT_SCENARIO``. +def get_cl_scenario() -> "common.Codelist": + """Retrieve ``Codelist=IIASA_ECE:CL_TRANSPORT_SCENARIO``. This code lists contains unique IDs for scenarios supported by the MESSAGEix-Transport workflow (:mod:`.transport.workflow`), plus the annotations: @@ -434,51 +458,74 @@ def get_cl_scenario() -> "Codelist": """ from sdmx.model import common + from message_ix_models.util.sdmx import read + + IIASA_ECE = read("IIASA_ECE:AGENCIES")["IIASA_ECE"] + + return refresh_cl_scenario( + common.Codelist( + id="CL_TRANSPORT_SCENARIO", maintainer=IIASA_ECE, version="1.0.0" + ) + ) + + +def refresh_cl_scenario(cl: "common.Codelist") -> "common.Codelist": + """Refresh ``Codelist=IIASA_ECE:CL_TRANSPORT_SCENARIO``. + + The code list is entirely regenerated. If it is different from `cl`, the new + version is returned. Otherwise, `cl` is returned unaltered. + """ + from sdmx.model import common + from message_ix_models.util.sdmx import read, write # Other data structures - as_ = read("IIASA_ECE:AGENCIES") + IIASA_ECE = read("IIASA_ECE:AGENCIES")["IIASA_ECE"] cl_ssp_2024 = read("ICONICS:SSP(2024)") - cl: "common.Codelist" = common.Codelist( - id="CL_TRANSPORT_SCENARIO", maintainer=as_["IIASA_ECE"], version="1.0.0" + candidate: "common.Codelist" = common.Codelist( + id="CL_TRANSPORT_SCENARIO", maintainer=IIASA_ECE, version="1.0.0" ) - def _a(*values): + # - The model name is per a Microsoft Teams message on 2024-11-25. + # - The scenario names appear to form a sequence from "baseline_DEFAULT" to + # "baseline_DEFAULT_step_15" and finally "baseline". The one used below is the + # latest in this sequence for which y₀=2020, rather than 2030. + base_url = "ixmp://ixmp-dev/SSP_SSP{}_v1.1/baseline_DEFAULT_step_13" + + def _a(c, led, edits): """Shorthand to generate the annotations.""" - return [ - common.Annotation(id="SSP-URN", text=values[0]), - common.Annotation(id="is-LED-scenario", text=repr(values[1])), - common.Annotation(id="EDITS-activity-id", text=repr(values[2])), - ] + return ScenarioCodeAnnotations( + c.urn, led, edits, base_url.format(c.id) + ).get_annotations(dict) for ssp_code in cl_ssp_2024: - cl.append( - common.Code( - id=f"SSP{ssp_code.id}", annotations=_a(ssp_code.urn, False, None) - ) + candidate.append( + common.Code(id=f"SSP{ssp_code.id}", **_a(ssp_code, False, None)) ) for ssp in ("1", "2"): ssp_code = cl_ssp_2024[ssp] - cl.append( + candidate.append( common.Code( id=f"LED-SSP{ssp_code.id}", name=f"Low Energy Demand/High-with-Low scenario with SSP{ssp_code.id} " "demographics", - annotations=_a(ssp_code.urn, True, None), + **_a(ssp_code, True, None), ) ) for id_, name in (("CA", "Current Ambition"), ("HA", "High Ambition")): - cl.append( + candidate.append( common.Code( id=f"EDITS-{id_}", name=f"EDITS scenario with ITF PASTA {id_!r} activity", - annotations=_a(cl_ssp_2024["2"].urn, False, id_), + **_a(cl_ssp_2024["2"], False, id_), ) ) - write(cl) - - return cl + if not candidate.compare(cl, strict=True): + write(candidate) + return candidate + else: + return cl From e2bc2910be24b54d9769ac31bb14001b38955f5b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 3 Dec 2024 17:33:45 +0100 Subject: [PATCH 07/45] Remove data/transport/base-scenario-url.json Replace with annotations in CL_TRANSPORT_SCENARIO. --- ...IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml | 49 +++++++++++++---- .../data/transport/base-scenario-url.json | 52 ------------------- 2 files changed, 38 insertions(+), 63 deletions(-) delete mode 100644 message_ix_models/data/transport/base-scenario-url.json diff --git a/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml b/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml index 525d53028..1addb8d3b 100644 --- a/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml +++ b/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml @@ -2,8 +2,8 @@ false - 2024-11-18T11:40:00.804051 - Generated by message_ix_models 2024.4.23.dev2473+gd12ad74b9.d20240806 + 2024-12-03T21:18:35.564011 + Generated by message_ix_models 2024.8.7.dev455+gd9d66fa40.d20241203 @@ -11,7 +11,7 @@ - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).1 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).1' False @@ -19,12 +19,15 @@ None + + 'ixmp://ixmp-dev/SSP_SSP1_v1.1/baseline_DEFAULT_step_13' + - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' False @@ -32,12 +35,15 @@ None + + 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).3 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).3' False @@ -45,12 +51,15 @@ None + + 'ixmp://ixmp-dev/SSP_SSP3_v1.1/baseline_DEFAULT_step_13' + - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).4 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).4' False @@ -58,12 +67,15 @@ None + + 'ixmp://ixmp-dev/SSP_SSP4_v1.1/baseline_DEFAULT_step_13' + - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).5 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).5' False @@ -71,13 +83,16 @@ None + + 'ixmp://ixmp-dev/SSP_SSP5_v1.1/baseline_DEFAULT_step_13' + Low Energy Demand/High-with-Low scenario with SSP1 demographics - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).1 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).1' True @@ -85,13 +100,16 @@ None + + 'ixmp://ixmp-dev/SSP_SSP1_v1.1/baseline_DEFAULT_step_13' + Low Energy Demand/High-with-Low scenario with SSP2 demographics - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' True @@ -99,13 +117,16 @@ None + + 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + EDITS scenario with ITF PASTA 'CA' activity - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' False @@ -113,13 +134,16 @@ 'CA' + + 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + EDITS scenario with ITF PASTA 'HA' activity - urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2 + 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' False @@ -127,6 +151,9 @@ 'HA' + + 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + diff --git a/message_ix_models/data/transport/base-scenario-url.json b/message_ix_models/data/transport/base-scenario-url.json deleted file mode 100644 index 03525acde..000000000 --- a/message_ix_models/data/transport/base-scenario-url.json +++ /dev/null @@ -1,52 +0,0 @@ -[ - { - "ssp": "ICONICS:SSP(2024).1", - "policy": false, - "url": "ixmp://ixmp-dev/SSP_dev_SSP1_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).2", - "policy": false, - "url": "ixmp://ixmp-dev/SSP_dev_SSP2_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).3", - "policy": false, - "url": "ixmp://ixmp-dev/SSP_dev_SSP3_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).4", - "policy": false, - "url": "ixmp://ixmp-dev/SSP_dev_SSP4_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).5", - "policy": false, - "url": "ixmp://ixmp-dev/SSP_dev_SSP5_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).1", - "policy": true, - "url": "ixmp://ixmp-dev/SSP_dev_SSP1_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).2", - "policy": true, - "url": "ixmp://ixmp-dev/SSP_dev_SSP2_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).3", - "policy": true, - "url": "ixmp://ixmp-dev/SSP_dev_SSP3_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).4", - "policy": true, - "url": "ixmp://ixmp-dev/SSP_dev_SSP4_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - }, - { - "ssp": "ICONICS:SSP(2024).5", - "policy": true, - "url": "ixmp://ixmp-dev/SSP_dev_SSP5_v0.1_Blv0.18/baseline_prep_lu_bkp_solved_materials" - } -] From aeaf3b7eea138ac497edee4c7adfdbfbc7d586a0 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 3 Dec 2024 17:34:36 +0100 Subject: [PATCH 08/45] Simplify .transport.workflow via .use_scenario_code() --- message_ix_models/model/transport/workflow.py | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/message_ix_models/model/transport/workflow.py b/message_ix_models/model/transport/workflow.py index 3948f09ea..701814e8e 100644 --- a/message_ix_models/model/transport/workflow.py +++ b/message_ix_models/model/transport/workflow.py @@ -1,4 +1,3 @@ -import json import logging from hashlib import blake2s from itertools import product @@ -7,7 +6,6 @@ from genno import KeyExistsError from message_ix_models.model.workflow import Config as WorkflowConfig -from message_ix_models.util import package_data_path if TYPE_CHECKING: from message_ix import Scenario @@ -48,7 +46,7 @@ def base_scenario_url( :py:`method = "auto"` Automatically identify the base scenario URL from the contents of - :file:`base-scenario-url.json`. The settings :attr:`.Config.ssp + ``CL_TRANSPORT_SCENARIO``. The settings :attr:`.Config.ssp <.transport.config.Config.ssp>` and :attr:`.Config.policy` are used to match an entry in the file. :py:`method = "bare"` @@ -62,17 +60,7 @@ def base_scenario_url( config: "Config" = context.transport if method == "auto": - # Load URL info from file - with open(package_data_path("transport", "base-scenario-url.json")) as f: - info = json.load(f) - - # Identify a key that matches the settings on `config` - key = (str(config.ssp), config.policy) - for item in info: - if (item["ssp"], item["policy"]) == key: - return item["url"] - - raise ValueError(f"No base URL for ({key!r})") # pragma: no cover + return config.base_scenario_url elif method == "bare": # Use a 'bare' RES or empty scenario if context.platform_info["name"] in (__name__, "message-ix-models"): @@ -187,7 +175,6 @@ def generate( ) -> "Workflow": from message_ix_models import Workflow from message_ix_models.model.workflow import solve - from message_ix_models.project.ssp import SSP_2024 from message_ix_models.report import report from . import build @@ -219,25 +206,22 @@ def generate( cl_scenario = get_cl_scenario() for scenario_code, policy in product(cl_scenario, (False, True)): - # Retrieve information from annotations on `scenario_code` - ssp_urn = str(scenario_code.get_annotation(id="SSP-URN").text) - is_LED = scenario_code.eval_annotation(id="is-LED-scenario") - EDITS_activity = scenario_code.eval_annotation(id="EDITS-activity-id") + config = context.transport + # Update the .transport.Config from the `scenario_code` + config.use_scenario_code(scenario_code) - # Look up the SSP_2024 code - ssp = SSP_2024.by_urn(ssp_urn) + config.policy = policy - # Store settings on the context - context.transport.ssp = ssp - context.transport.policy = policy - context.transport.project["LED"] = is_LED + # TEMP + is_LED = config.project["LED"] + EDITS_activity = config.project["EDITS"]["activity"] # Construct labels including the SSP code and policy identifier # ‘Short’ label used for workflow steps label = f"{scenario_code.id}{' policy' if policy else ''}" # ‘Full’ label used in the scenario name if not is_LED and EDITS_activity is None: - label_full = f"SSP_2024.{ssp.name}" + label_full = f"SSP_2024.{config.ssp.name}" else: label_full = label @@ -266,7 +250,12 @@ def generate( # Build MESSAGEix-Transport on the scenario name = wf.add_step( - f"{label} built", base, build.main, target=target_url, clone=True, ssp=ssp + f"{label} built", + base, + build.main, + target=target_url, + clone=True, + ssp=config.ssp, ) # This block copied from message_data.projects.navigate.workflow @@ -276,7 +265,7 @@ def generate( # 'Simulate' build and produce debug outputs debug.append(f"{label} debug build") - wf.add_step(debug[-1], base, build.main, ssp=ssp, dry_run=True) + wf.add_step(debug[-1], base, build.main, ssp=config.ssp, dry_run=True) # Solve wf.add_step(f"{label} solved", name, solve, config=SOLVE_CONFIG) From 35ba82a686f070f5416f7cfc43d66fb1d81d9791 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 11:05:48 +0100 Subject: [PATCH 09/45] Limit t=conm_ar in transport/R12/act-non_ldv.csv This commodity is not used in IEA EWEB 2024 data for 2019, so we constrain its activity to a low, non-zero level. --- .../data/transport/R12/act-non_ldv.csv | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/message_ix_models/data/transport/R12/act-non_ldv.csv b/message_ix_models/data/transport/R12/act-non_ldv.csv index de6731a09..880a4bdf4 100644 --- a/message_ix_models/data/transport/R12/act-non_ldv.csv +++ b/message_ix_models/data/transport/R12/act-non_ldv.csv @@ -2,9 +2,22 @@ # # Units: dimensionless # -node, technology, year, value -R12_AFR, FCg_bus, 2020, 0.01 -R12_AFR, ICG_bus, 2020, 0.01 -R12_AFR, FR_ICG, 2020, 0.0001 -R12_AFR, FR_FCg, 2020, 0.00001 -R12_NAM, FR_FCg, 2020, 0.001 +node, technology, year, value +R12_AFR, FCg_bus, 2020, 0.01 +R12_AFR, ICG_bus, 2020, 0.01 +R12_AFR, FR_ICG, 2020, 0.0001 +R12_AFR, FR_FCg, 2020, 0.00001 +R12_NAM, FR_FCg, 2020, 0.001 +# PNK: Added 2024-12-04 to address high methanol use in 2020 +R12_AFR, conm_ar, 2020, 0.01 +R12_CHN, conm_ar, 2020, 0.01 +R12_EEU, conm_ar, 2020, 0.01 +R12_FSU, conm_ar, 2020, 0.01 +R12_LAM, conm_ar, 2020, 0.01 +R12_MEA, conm_ar, 2020, 0.01 +R12_NAM, conm_ar, 2020, 0.01 +R12_PAO, conm_ar, 2020, 0.01 +R12_PAS, conm_ar, 2020, 0.01 +R12_RCPA, conm_ar, 2020, 0.01 +R12_SAS, conm_ar, 2020, 0.01 +R12_WEU, conm_ar, 2020, 0.01 From e724c7c38b2d8d7478640ea1ad06468c0797c697 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 11:06:27 +0100 Subject: [PATCH 10/45] Include computed demand for transport F RAIL --- message_ix_models/model/transport/demand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/model/transport/demand.py b/message_ix_models/model/transport/demand.py index c328dddc4..e2db4d41f 100644 --- a/message_ix_models/model/transport/demand.py +++ b/message_ix_models/model/transport/demand.py @@ -182,7 +182,7 @@ def dummy( # Apply the adjustment factor (fv + "1", "mul", fv + "0", "fv factor:n-t-y"), # Select only the ROAD data. NB Do not drop so 't' labels can be used for 'c', next. - ((fv + "2", "select", fv + "1"), dict(indexers=dict(t=["ROAD"]))), + ((fv + "2", "select", fv + "1"), dict(indexers=dict(t=["RAIL", "ROAD"]))), # Relabel ((fv_cny, "relabel2", fv + "2"), dict(new_dims={"c": "transport F {t}"})), # Convert to ixmp format From d751e1ad2413c489b277d04d9b6b54cd933cc89d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 14:38:39 +0100 Subject: [PATCH 11/45] Generate output to "transport F RAIL vehicle" Mirror calculation flow in .ldv for same parameter. --- message_ix_models/model/transport/freight.py | 39 ++++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/message_ix_models/model/transport/freight.py b/message_ix_models/model/transport/freight.py index 0eeef98bd..3ca6cf93e 100644 --- a/message_ix_models/model/transport/freight.py +++ b/message_ix_models/model/transport/freight.py @@ -53,18 +53,43 @@ def prepare_computer(c: genno.Computer): # Extract the 'input' data frame c.add(k[3], lambda d: d["input"], to_add[-1]) + # Create base quantity for "output" parameter + # TODO Combine in a loop with "input", above—similar to .ldv + k_output = genno.KeySeq("F output") + nty = tuple("nty") + c.add(k_output[0] * nty, wildcard(1.0, "dimensionless", nty)) + for i, coords in enumerate(["n::ex world", "t::F", "y::model"]): + c.add( + k_output[i + 1] * nty, + "broadcast_wildcard", + k_output[i] * nty, + coords, + dim=coords[0], + ) + + for par_name, base, ks, i in (("output", k_output[3] * nty, k_output, 3),): + # Produce the full quantity for input/output efficiency + prev = c.add( + ks[i + 1], + "mul", + ks[i], + f"broadcast:t-c-l:transport+{par_name}", + "broadcast:y-yv-ya:all", + ) + + # Convert to ixmp/MESSAGEix-structured pd.DataFrame + # NB quote() is necessary with dask 2024.11.0, not with earlier versions + c.add(ks[i + 2], "as_message_df", prev, name=par_name, dims=DIMS, common=COMMON) + + # Convert to target units + to_add.append("output::transport F+ixmp") + c.add(to_add[-1], convert_units, ks[i + 2], "transport info") + # Produce corresponding output, capacity_factor, technical_lifetime - # FIXME Use "… F RAIL …" as appropriate c.add( k[4], partial( make_matched_dfs, - output=dict( - value=registry("1.0 gigatonne km"), - commodity="transport F ROAD vehicle", - level="useful", - time_dest=COMMON["time_dest"], - ), capacity_factor=registry.Quantity("1"), technical_lifetime=registry("10 year"), ), From 31be31380526d29d6d6c7a878ccd2e3c9f222093 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 14:39:40 +0100 Subject: [PATCH 12/45] Add "F" as a parent of freight modes; units --- .../data/transport/technology.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/message_ix_models/data/transport/technology.yaml b/message_ix_models/data/transport/technology.yaml index 7a6fb9158..e6ffaa933 100644 --- a/message_ix_models/data/transport/technology.yaml +++ b/message_ix_models/data/transport/technology.yaml @@ -302,6 +302,7 @@ F RAIL: - f rail electr - f rail lightoil units: Gv km + output: {commodity: transport F RAIL vehicle} iea-eweb-flow: [RAIL] crail_pub: @@ -454,8 +455,28 @@ F ROAD: - FR_ICG - FR_ICH units: Gv km + output: {commodity: transport F ROAD vehicle} iea-eweb-flow: [ROAD] +F: # Freight modes + # TODO Prepare this hierarchically from the following + # child: + # - F RAIL + # - F ROAD + child: + - f rail electr + - f rail lightoil + - f road electr + - FR_FCg + - FR_FCH + - FR_FCm + - FR_ICAe + - FR_ICE_H + - FR_ICE_L + - FR_ICE_M + - FR_ICG + - FR_ICH + AIR: name: Aviation report: "Domestic Aviation" From 81726644314ced389fcf3e95b672b9cfdef0644a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 14:40:12 +0100 Subject: [PATCH 13/45] Copy "output" anno from F {RAIL,ROAD} to children --- message_ix_models/model/transport/structure.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/message_ix_models/model/transport/structure.py b/message_ix_models/model/transport/structure.py index 7df40fdbc..f2b30e43f 100644 --- a/message_ix_models/model/transport/structure.py +++ b/message_ix_models/model/transport/structure.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from copy import deepcopy from typing import Any, Union from sdmx.model.common import Annotation, Code @@ -109,6 +110,12 @@ def make_spec(regions: str) -> Spec: output = dict(commodity=f"transport vehicle {t.id}", level="useful") t.annotations.append(Annotation(id="output", text=repr(output))) + # Associate other techs with their output commodities + for mode in "F RAIL", "F ROAD": + parent = techs[techs.index(mode)] + for t in parent.child: + t.annotations.append(deepcopy(parent.get_annotation(id="output"))) + # Generate a spec for the generalized disutility formulation for LDVs s2 = disutility.get_spec( groups=s.add.set["consumer_group"], technologies=LDV_techs, template=TEMPLATE From f4a4a10bd320a48329c0a5875e675f126747e4cd Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 14:41:19 +0100 Subject: [PATCH 14/45] Silence warnings re: units in .transport.ldv "output" for most technologies is dimensionless because ACT and the output commodities have the same units. --- message_ix_models/model/transport/ldv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/model/transport/ldv.py b/message_ix_models/model/transport/ldv.py index ded7e6f86..491e03169 100644 --- a/message_ix_models/model/transport/ldv.py +++ b/message_ix_models/model/transport/ldv.py @@ -299,7 +299,7 @@ def prepare_tech_econ( # Create base quantity for "output" parameter nty = tuple("nty") - c.add(k.output[0] * nty, wildcard(1.0, "gigavehicle km", nty)) + c.add(k.output[0] * nty, wildcard(1.0, "", nty)) for i, coords in enumerate(["n::ex world", "t::LDV", "y::model"]): c.add( k.output[i + 1] * nty, From 279fb4d36e919fec6b508c2c6646172c3ea12146 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 14:47:13 +0100 Subject: [PATCH 15/45] Add .util.genno.insert --- message_ix_models/util/genno.py | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 message_ix_models/util/genno.py diff --git a/message_ix_models/util/genno.py b/message_ix_models/util/genno.py new file mode 100644 index 000000000..30cdbb5fe --- /dev/null +++ b/message_ix_models/util/genno.py @@ -0,0 +1,53 @@ +"""Utilities for working with :mod:`.genno`. + +Most code appearing here **should** be migrated upstream, to genno itself. +""" + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from genno import Computer + + from message_ix_models.types import KeyLike + + +log = logging.getLogger(__name__) + + +def insert(c: "Computer", key: "KeyLike", operation, tag: str = "pre") -> "KeyLike": + """Insert a task that performs `operation` on `key`. + + 1. The existing task at `key` is moved to a new key, ``{key}+{tag}``. + 2. A new task is inserted at `key` that performs `operation` on the output of the + original task. + + One way to use :func:`insert` is with a ‘pass-through’ `operation` that, for + instance, performs logging, assertions, or other steps, then returns its input + unchanged. In this way, all other tasks in the graph referring to `key` receive + exactly the same input as they would have previously, prior to the :func:`insert` + call. + + It is also possible to insert `operation` that mutates its input in certain ways. + + .. todo:: Migrate to :py:`genno.Computer.insert()`. + + Returns + ------- + KeyLike + same as the `key` parameter. + """ + import genno + + # Determine a key for the task that to be shifted + k_pre = genno.Key(key) + tag + assert k_pre not in c + + # Move the existing task at `key` to `k_pre` + c.graph[k_pre] = c.graph.pop(key) + log.info(f"Move {key!r} to {k_pre!r}") + + # Add `operation` at `key`, operating on the output of the original task + c.graph[key] = (operation, k_pre) + + return key From fc8aef5d6dffd4e13e45a46666c19ef24077d122 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 21:38:54 +0100 Subject: [PATCH 16/45] Improve type hints in {,.transport}.testing --- message_ix_models/model/transport/testing.py | 4 ++-- message_ix_models/testing/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/message_ix_models/model/transport/testing.py b/message_ix_models/model/transport/testing.py index fc24ba9d0..a5f6203e3 100644 --- a/message_ix_models/model/transport/testing.py +++ b/message_ix_models/model/transport/testing.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Optional, Union import pytest -from genno import Computer from message_ix import ModelError, Reporter, Scenario import message_ix_models.report @@ -23,6 +22,7 @@ if TYPE_CHECKING: import pandas import pint + from genno import Computer log = logging.getLogger(__name__) @@ -85,7 +85,7 @@ def configure_build( years: str, tmp_path: Optional[Path] = None, options=None, -) -> tuple[Computer, ScenarioInfo]: +) -> tuple["Computer", ScenarioInfo]: test_context.update(regions=regions, years=years, output_path=tmp_path) # By default, omit plots while testing diff --git a/message_ix_models/testing/__init__.py b/message_ix_models/testing/__init__.py index 54fc17425..c405d9b85 100644 --- a/message_ix_models/testing/__init__.py +++ b/message_ix_models/testing/__init__.py @@ -312,7 +312,7 @@ def export_test_data(context: Context): # Retrieve the type mapping first, to be modified as sheets are discarded ix_type_mapping = reader.parse("ix_type_mapping").set_index("item") - for name in reader.sheet_names: + for name in map(str, reader.sheet_names): # Check if this sheet is to be included if name == "ix_type_mapping": # Already handled From 4ed12ac0a69aa1f7e7eaeef2f194d034fa627ffc Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 21:41:57 +0100 Subject: [PATCH 17/45] Add .testing.check --- message_ix_models/testing/check.py | 398 +++++++++++++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 message_ix_models/testing/check.py diff --git a/message_ix_models/testing/check.py b/message_ix_models/testing/check.py new file mode 100644 index 000000000..48ebcb6e8 --- /dev/null +++ b/message_ix_models/testing/check.py @@ -0,0 +1,398 @@ +"""In-line checks for :mod:`genno` graphs.""" + +import logging +from abc import ABC, abstractmethod +from collections.abc import Callable, Collection, Mapping +from dataclasses import dataclass +from functools import partial +from itertools import count +from typing import TYPE_CHECKING, ClassVar, Optional, TypeVar, Union + +import genno +import pandas as pd + +from message_ix_models.util.genno import insert + +if TYPE_CHECKING: + import pathlib + + from message_ix_models.types import KeyLike + + T = TypeVar("T") + +log = logging.getLogger(__name__) + + +@dataclass +class Check(ABC): + """Class representing a single check.""" + + types: ClassVar[tuple[type, ...]] = () + + def __post_init__(self) -> None: + if not self.types: + log.error(f"{type(self).__name__}.types is empty → check will never run") + + def __call__(self, obj) -> tuple[bool, str]: + if not isinstance(obj, self.types): + # return True, f"{type(obj)} not handled by {type(self)}; skip" # DEBUG + return True, "" + else: + result = self.run(obj) + if isinstance(result, tuple): + return result + else: + return result, "" + + @property + def _description(self) -> str: + """Description derived from the class docstring.""" + assert self.__doc__ + return self.__doc__.splitlines()[0].rstrip(".") + + def recurse_parameter_data(self, obj) -> tuple[bool, str]: + """:func:`run` recursively on :any:`.ParameterData`.""" + _pass, fail = [], {} + + for k, v in obj.items(): + ret = self.run(v) + result, message = ret if isinstance(ret, tuple) else (ret, "") + if result: + _pass.append(k) + else: + fail[k] = message + + lines = [f"{self._description} in {len(_pass)}/{len(obj)} parameters"] + if fail: + lines.extend(["", f"FAIL: {len(fail)} parameters"]) + lines.extend(f"{k!r}:\n{v}" for k, v in fail.items()) + + return not bool(fail), "\n".join(lines) + + @abstractmethod + def run(self, obj) -> Union[bool, tuple[bool, str]]: + """Run the check on `obj` and return either :any:`True` or :any:`False`.""" + + +class CheckResult: + """Accumulator for the results of multiple checks. + + This class' :meth:`__call__` method can be used as the :py:`result_cb` argument to + :func:`.apply_checks`. After doing so, :py:`bool(check_result)` will give the + overall passage or failure of the check suite. + """ + + def __init__(self): + self._pass = True + + def __bool__(self) -> bool: + return self._pass + + def __call__(self, value: bool, message: str) -> None: + self._pass &= value + if message: + log.log(logging.INFO if value else logging.ERROR, message) + + +@dataclass +class ContainsDataForParameters(Check): + #: Collection of parameter names that should be present in the object. + parameter_names: set[str] + + types = (dict,) + + def run(self, obj): + if self.parameter_names: + if self.parameter_names != set(obj): + return False, f"Parameters {set(obj)} != {self.parameter_names}" + else: + N = len(self.parameter_names) + return True, f"{N}/{N} expected parameters present" + return True + + +@dataclass +class Dump(Check): + """Dump to a temporary path for inspection. + + This always returns :any:`True`. + """ + + base_path: "pathlib.Path" + + types = (dict, pd.DataFrame, genno.Quantity) + + def recurse_parameter_data(self, obj) -> tuple[bool, str]: + for k, v in obj.items(): + self.run(v, name=k) + + return True, "" + + def run(self, obj, *, name: Optional[str] = None): + if isinstance(obj, dict): + return self.recurse_parameter_data(obj) + + # Construct a path that does not yet exist + name_stem = name or "debug" + name_seq = map(lambda i: f"{name_stem}-{i}", count()) + while True: + path = self.base_path.joinpath(next(name_seq)).with_suffix(".csv") + if not path.exists(): + break + + # Get a pandas object + pd_obj = ( + obj.to_series().reset_index() if isinstance(obj, genno.Quantity) else obj + ) + + log.info(f"Dump data to {path}") + pd_obj.to_csv(path, index=False) + + return True, "" + + +@dataclass +class HasCoords(Check): + """Object has/lacks certain coordinates.""" + + coords: dict[str, Collection[str]] + inverse: bool = False + types = (dict, pd.DataFrame, genno.Quantity) + + def run(self, obj): + if isinstance(obj, dict): + return self.recurse_parameter_data(obj) + + # Prepare a coords mapping for the object + if isinstance(obj, pd.DataFrame): + coords = {dim: obj[dim].unique() for dim in obj.columns} + else: + coords = obj.coords + + result = True + message = [] + for dim, v in self.coords.items(): + if dim not in coords: + continue + exp, obs = set(v), set(coords[dim]) + + if not self.inverse and not exp <= obs: + result = False + message.append(f"\nDimension {dim!r}: missing coords {exp - obs}") + elif self.inverse and not exp.isdisjoint(obs): + result = False + message.append(f"\nDimension {dim!r}: coords {exp ^ obs} present") + return result, "\n".join(message) + + +@dataclass +class HasUnits(Check): + """Quantity has the expected units.""" + + units: Optional[Union[str, dict]] + types = (genno.Quantity,) + + def run(self, obj): + from genno.testing import assert_units as a_u_genno + + from message_ix_models.model.transport.testing import assert_units as a_u_local + + if self.units is None: + return True + + if isinstance(self.units, dict): + func = a_u_local + if isinstance(obj, genno.Quantity): + obj = obj.to_series().reset_index() + else: + func = a_u_genno + + try: + func(obj, self.units) + except AssertionError as e: + return False, repr(e) + else: + return True, f"Units are {self.units!r}" + + +@dataclass +class NoneMissing(Check): + """No missing values.""" + + setting: None = None + types = (pd.DataFrame, dict) + + def run(self, obj): + if isinstance(obj, dict): + return self.recurse_parameter_data(obj) + + missing = obj.isna() + if missing.any(axis=None): + return False, "NaN values in data frame" + return True, self._description + + +@dataclass +class NonNegative(Check): + """No negative values. + + .. todo:: Add args so the check can be above or below any threshold value. + """ + + types = (pd.DataFrame, dict) + + def run(self, obj): + if isinstance(obj, dict): + return self.recurse_parameter_data(obj) + + result = obj["value"] < 0 + if result.any(axis=None): + return False, f"Negative values for {result.sum()} observations" + return True, self._description + + +@dataclass +class Log(Check): + """Log contents. + + This always returns :any:`True`. + """ + + rows: Optional[int] = 7 + types = (dict, pd.DataFrame, genno.Quantity) + + def recurse_parameter_data(self, obj) -> tuple[bool, str]: + for k, v in obj.items(): + sep = f"{k!r} -----" + log.debug(sep) + self.run(v) + + return True, "" + + def run(self, obj): + if isinstance(obj, dict): + return self.recurse_parameter_data(obj) + + # Get a pandas object that has a .to_string() method + pd_obj = obj.to_series() if isinstance(obj, genno.Quantity) else obj + + lines = [ + f"{len(pd_obj)} observations", + pd_obj.to_string(max_rows=self.rows, min_rows=self.rows), + ] + log.debug("\n".join(lines)) + + return True, "" + + +@dataclass +class Size(Check): + """Quantity has expected size on certain dimensions.""" + + setting: dict[str, int] + types = (genno.Quantity,) + + def run(self, obj): + result = True + message = [] + for dim, N in self.setting.items(): + if dim not in obj.dims: + continue + if N != len(obj.coords[dim]): + message.append( + f"Dimension {dim!r} has length {len(obj.coords[dim])} != {N}" + ) + result = False + else: + message.append(f"Dimension {dim!r} has length {N}") + return result, "\n".join(message) + + +def apply_checks( + value: "T", + checks: Collection[Check], + *, + key: "KeyLike", + result_cb: Callable[[bool, str], None], +) -> "T": + """Apply some `checks` to `value`. + + Parameters + ---------- + value + Anything. + checks + 0 or more :class:`.Check` instances. Each is called on `value`. + key + Used to log information about the checks performed. + result_cb + Callback function that is passed the result of each :class:`.Check` call. + + Returns + ------- + Any + `value` exactly as passed. + """ + separator = f"=== {key!r}: {len(checks)} checks ===" + log.info(separator) + + # Invoke each of the checks, accumulating the result via `result_cb` + for check in checks: + result_cb(*check(value)) + + log.info("=" * len(separator)) + + # Pass through the original value + return value + + +def insert_checks( + computer: "genno.Computer", + target: "KeyLike", + check_map: Mapping["KeyLike", Collection["Check"]], + check_common: Collection["Check"], +) -> CheckResult: + """Insert some :class:`Checks <.Check>` into `computer`. + + Parameters + ---------- + computer + target + A new key added to trigger all the tasks and checks. + check_map + A mapping from existing keys (that must appear in `computer`) to collections of + :class:`.Check` instances to be applied to those keys. + check_common + A collection of common :class:`.Check` instances, to be applied to every key in + `check_map`. + + Returns + ------- + CheckResult + after the checks are triggered (for instance, with :py:`computer.get(target)`), + this object will contain the overall check pass/fail result. + """ + # Create a CheckResult instance to absorb the outputs of each apply_checks() call + # and sub-call + result = CheckResult() + + # Iterate over keys mentioned in `check_map` + for key, checks in check_map.items(): + # Insert a task with apply_checks() as the callable + insert( + computer, + key, + partial( + apply_checks, + key=key, + # A collection of Check instances, including those specific to `key` and + # those from `check_common` + checks=tuple(checks) + tuple(check_common), + result_cb=result, + ), + ) + + # Add a task at `target` that collects the outputs of every inserted call + computer.add(target, list(check_map)) + + return result From 57055ff86ad8e4e8e7d1732fffffbcfadaca7a9d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 21:44:30 +0100 Subject: [PATCH 18/45] Improve .transport.test_build.test_debug - Use .testing.check features to simplify list of assertions/debug output. - Add the same set of cases as .test_demand.test_exo. --- .../tests/model/transport/test_build.py | 171 ++++++++++-------- 1 file changed, 100 insertions(+), 71 deletions(-) diff --git a/message_ix_models/tests/model/transport/test_build.py b/message_ix_models/tests/model/transport/test_build.py index 1e64fa785..602826495 100644 --- a/message_ix_models/tests/model/transport/test_build.py +++ b/message_ix_models/tests/model/transport/test_build.py @@ -1,17 +1,32 @@ import logging +from collections.abc import Collection from copy import copy +from typing import TYPE_CHECKING, Literal import genno import ixmp import pytest -from genno import Quantity -from genno.testing import assert_units from pytest import mark, param from message_ix_models.model.structure import get_codes from message_ix_models.model.transport import build, report, structure +from message_ix_models.model.transport.ldv import TARGET from message_ix_models.model.transport.testing import MARK, configure_build, make_mark from message_ix_models.testing import bare_res +from message_ix_models.testing.check import ( + ContainsDataForParameters, + Dump, + HasCoords, + HasUnits, + Log, + NoneMissing, + Size, + insert_checks, +) + +if TYPE_CHECKING: + from message_ix_models.testing.check import Check + from message_ix_models.types import KeyLike log = logging.getLogger(__name__) @@ -164,76 +179,90 @@ def test_build_existing(tmp_path, test_context, url, solve=False): del mp +CHECKS: dict["KeyLike", Collection["Check"]] = { + "broadcast:t-c-l:transport+input": (HasUnits("dimensionless"),), + "broadcast:t-c-l:transport+output": ( + HasUnits("dimensionless"), + HasCoords({"commodity": ["transport F RAIL vehicle"]}), + ), + "output::transport F+ixmp": ( + HasCoords( + {"commodity": ["transport F RAIL vehicle", "transport F ROAD vehicle"]} + ), + ), + "transport F::ixmp": ( + ContainsDataForParameters( + {"capacity_factor", "input", "output", "technical_lifetime"} + ), + ), + # The following partly replicates .test_ldv.test_get_ldv_data() + TARGET: ( + ContainsDataForParameters( + { + "bound_new_capacity_lo", + "bound_new_capacity_up", + "capacity_factor", + "emission_factor", + "fix_cost", + "growth_activity_lo", + "growth_activity_up", + "historical_new_capacity", + "initial_activity_up", + "input", + "inv_cost", + "output", + "relation_activity", + "technical_lifetime", + "var_cost", + } + ), + ), +} + + @build.get_computer.minimum_version @pytest.mark.parametrize( - "regions, years, N_node, options", - [ - ("R12", "B", 12, dict()), - ], + "build_kw", + ( + dict(regions="R11", years="A", options=dict()), + dict(regions="R11", years="B", options=dict()), + dict(regions="R11", years="B", options=dict(futures_scenario="A---")), + dict(regions="R11", years="B", options=dict(futures_scenario="debug")), + dict(regions="R12", years="B", options=dict()), + dict(regions="R12", years="B", options=dict(navigate_scenario="act+ele+tec")), + dict(regions="R14", years="B", options=dict()), + param(dict(regions="ISR", years="A", options=dict()), marks=MARK[3]), + ), ) -def test_debug(test_context, tmp_path, regions, years, N_node, options): +def test_debug( + test_context, + tmp_path, + build_kw, + N_node, + verbosity: Literal[0, 1, 2, 3] = 2, # NB Increase this to show more verbose output +): """Debug particular calculations in the transport build process.""" - # Import certain keys - # from message_ix_models.model.transport.key import pdt_ny - - c, info = configure_build( - test_context, tmp_path=tmp_path, regions=regions, years=years, options=options - ) - - fail = False # Sentinel value for deferred failure assertion - - # Check that some keys (a) can be computed without error and (b) have correct units - # commented: these are slow because they repeat some calculations many times. - # Uncommented as needed for debugging - for key, unit in ( - # Uncomment and modify these line(s) to check certain values - # ("transport nonldv::ixmp", None), - ): - print(f"\n\n-- {key} --\n\n") - print(c.describe(key)) - - # Quantity can be computed - result = c.get(key) - - # # Display the entire `result` object - # print(f"{result = }") - - if isinstance(result, Quantity): - print(result.to_series().to_string()) - - # Quantity has the expected units - assert_units(result, unit) - - # Quantity has the expected size on the n/node dimension - assert N_node == len(result.coords["n"]), result.coords["n"].data - - # commented: dump to a temporary path for inspection - # fn = f"{key.replace(' ', '-')}-{hash(tuple(options.items()))}" - # dump = tmp_path.joinpath(fn).with_suffix(".csv") - # print(f"Dumped to {dump}") - # qty.to_series().to_csv(dump) - elif isinstance(result, dict): - for k, v in sorted(result.items()): - print( - f"=== {k} ({len(v)} obs) ===", - v.head().to_string(), # Initial rows - "...", - v.tail().to_string(), # Final rows - # v.to_string(), # Entire value - f"=== {k} ({len(v)} obs) ===", - sep="\n", - ) - # print(v.tail().to_string()) - - # Write to file - # if k == "output": - # v.to_csv("debug-output.csv", index=False) - - missing = v.isna() - if missing.any(axis=None): - print("… missing values") - fail = True # Fail later - - assert not fail # Any failure in the above loop - - assert not fail # Any failure in the above loop + # Get a Computer prepared to build the model with the given options + c, info = configure_build(test_context, tmp_path=tmp_path, **build_kw) + + # Construct a list of common checks + verbose: dict[int, list["Check"]] = { + 0: [], + 1: [Log(7)], + 2: [Log(None)], + 3: [Dump(tmp_path)], + } + common = [Size({"n": N_node}), NoneMissing()] + verbose[verbosity] + + # Insert key-specific and common checks + k = "test_debug" + result = insert_checks(c, k, CHECKS, common) + + # Show what will be computed + if verbosity == 2: + print(c.describe(k)) + + # Compute the test key + c.get(k) + + assert result, "1 or more checks failed" From b95acd6088233d5004c8801a265a3ae477c01e28 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 22:03:15 +0100 Subject: [PATCH 19/45] Add "N_node" test fixture --- .../tests/model/transport/test_build.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/message_ix_models/tests/model/transport/test_build.py b/message_ix_models/tests/model/transport/test_build.py index 602826495..ea48e928e 100644 --- a/message_ix_models/tests/model/transport/test_build.py +++ b/message_ix_models/tests/model/transport/test_build.py @@ -31,6 +31,19 @@ log = logging.getLogger(__name__) +@pytest.fixture +def N_node(request) -> int: + """Expected number of nodes, by introspection of other parameter values.""" + if "build_kw" in request.fixturenames: + build_kw = request.getfixturevalue("build_kw") + + # NB This could also be done by len(.model.structure.get_codelist(…)), but hard- + # coding is probably a little faster + return {"ISR": 1, "R11": 11, "R12": 12, "R14": 14}[build_kw["regions"]] + else: + raise NotImplementedError + + @pytest.mark.parametrize("years", [None, "A", "B"]) @pytest.mark.parametrize( "regions_arg, regions_exp", From ccac4609b7df2deb98ec6cd717b8fcebf374983b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 22:50:40 +0100 Subject: [PATCH 20/45] Drop .test_demand.test_exo Include all assertions in .test_build.CHECKS --- .../tests/model/transport/test_build.py | 39 ++++++++- .../tests/model/transport/test_demand.py | 87 ------------------- 2 files changed, 38 insertions(+), 88 deletions(-) diff --git a/message_ix_models/tests/model/transport/test_build.py b/message_ix_models/tests/model/transport/test_build.py index ea48e928e..8805124e2 100644 --- a/message_ix_models/tests/model/transport/test_build.py +++ b/message_ix_models/tests/model/transport/test_build.py @@ -9,7 +9,7 @@ from pytest import mark, param from message_ix_models.model.structure import get_codes -from message_ix_models.model.transport import build, report, structure +from message_ix_models.model.transport import build, demand, report, structure from message_ix_models.model.transport.ldv import TARGET from message_ix_models.model.transport.testing import MARK, configure_build, make_mark from message_ix_models.testing import bare_res @@ -20,6 +20,7 @@ HasUnits, Log, NoneMissing, + NonNegative, Size, insert_checks, ) @@ -208,6 +209,42 @@ def test_build_existing(tmp_path, test_context, url, solve=False): {"capacity_factor", "input", "output", "technical_lifetime"} ), ), + # + # The following are intermediate checks formerly in .test_demand.test_exo + "mode share:n-t-y:base": (HasUnits(""),), + "mode share:n-t-y": (HasUnits(""),), + "population:n-y": (HasUnits("Mpassenger"),), + "cg share:n-y-cg": (HasUnits(""),), + "GDP:n-y:PPP+capita": (HasUnits("kUSD / passenger / year"),), + "GDP:n-y:PPP+capita+index": (HasUnits(""),), + "votm:n-y": (HasUnits(""),), + "PRICE_COMMODITY:n-c-y:transport+smooth": (HasUnits("USD / km"),), + "cost:n-y-c-t": (HasUnits("USD / km"),), + # These units are implied by the test of "transport pdt:*": + # "transport pdt:n-y:total" [=] Mm / year + demand.pdt_nyt + "1": (HasUnits("passenger km / year"),), + demand.ldv_ny + "total": (HasUnits("Gp km / a"),), + # FIXME Handle dimensionality instead of exact units + # demand.ldv_nycg: (HasUnits({"[length]": 1, "[passenger]": 1, "[time]": -1}),), + "pdt factor:n-y-t": (HasUnits(""),), + # "fv factor:n-y": (HasUnits(""),), # Fails: this key no longer exists + # "fv:n:advance": (HasUnits(""),), # Fails: only fuzzed data in message-ix-models + demand.fv_cny: (HasUnits("Gt km"),), + # + # Exogenous demand calculation succeeds + "transport demand::ixmp": ( + # Data is returned for the demand parameter only + ContainsDataForParameters({"demand"}), + HasCoords({"level": ["useful"]}), + # Certain labels are specifically excluded/dropped in the calculation + HasCoords( + {"commodity": ["transport pax ldv", "transport F WATER"]}, inverse=True + ), + # No negative values + NonNegative(), + # …plus default NoneMissing + ), + # # The following partly replicates .test_ldv.test_get_ldv_data() TARGET: ( ContainsDataForParameters( diff --git a/message_ix_models/tests/model/transport/test_demand.py b/message_ix_models/tests/model/transport/test_demand.py index 06614013b..6fd475067 100644 --- a/message_ix_models/tests/model/transport/test_demand.py +++ b/message_ix_models/tests/model/transport/test_demand.py @@ -45,93 +45,6 @@ def test_demand_dummy(test_context, regions, years): assert any(data["demand"]["commodity"] == "transport pax URLMM") -@build.get_computer.minimum_version -@pytest.mark.parametrize( - "regions, years, N_node, options", - [ - param("R11", "A", 11, dict(), marks=MARK[1]), - param("R11", "B", 11, dict(), marks=MARK[1]), - param("R11", "B", 11, dict(futures_scenario="debug"), marks=MARK[1]), - param("R11", "B", 11, dict(futures_scenario="A---"), marks=MARK[1]), - ("R12", "B", 12, dict()), - ("R12", "B", 12, dict(navigate_scenario="act+ele+tec")), - param("R14", "B", 14, dict(), marks=make_mark[2](genno.ComputationError)), - param("ISR", "A", 1, dict(), marks=MARK[3]), - ], -) -def test_exo(test_context, tmp_path, regions, years, N_node, options): - """Exogenous demand calculation succeeds.""" - c, info = testing.configure_build( - test_context, tmp_path=tmp_path, regions=regions, years=years, options=options - ) - - # Check that some keys (a) can be computed without error and (b) have correct units - # commented: these are slow because they repeat some calculations many times. - # Uncommented as needed for debugging - for key, unit in ( - # ("mode share:n-t-y:base", ""), - # ("mode share:n-t-y", ""), - # ("population:n-y", "Mpassenger"), - # ("cg share:n-y-cg", ""), - # ("GDP:n-y:PPP+capita", "kUSD / passenger / year"), - # ("GDP:n-y:PPP+capita+index", ""), - # ("votm:n-y", ""), - # ("PRICE_COMMODITY:n-c-y:transport+smooth", "USD / km"), - # ("cost:n-y-c-t", "USD / km"), - # # These units are implied by the test of "transport pdt:*": - # # "transport pdt:n-y:total" [=] Mm / year - # (demand.pdt_nyt + "1", "passenger km / year"), - # (demand.ldv_ny + "total", "Gp km / a"), - # (demand.ldv_nycg, {"[length]": 1, "[passenger]": 1, "[time]": -1}), - # ("pdt factor:n-y-t", ""), - # ("fv factor:n-y", ""), - # ("fv:n:advance", ""), - # (demand.fv_cny, "Gt km"), - ): - try: - # Quantity can be computed - qty = c.get(key) - - # Quantity has the expected units - assert_units(qty, unit) - - # Quantity has the expected size on the n/node dimension - assert N_node == len(qty.coords["n"]), qty.coords["n"].data - - # commented: dump to a temporary path for inspection - # fn = f"{key.replace(' ', '-')}-{hash(tuple(options.items()))}" - # dump = tmp_path.joinpath(fn).with_suffix(".csv") - # print(f"Dumped to {dump}") - # qty.to_series().to_csv(dump) - except Exception: - # Something else - print(f"\n\n-- {key} --\n\n") - print(c.describe(key)) - print(qty.to_series().to_string(), qty.attrs, qty.dims, qty.coords) - raise - - # Demand can be computed - data = c.get("transport demand::ixmp") - - # Data is returned for the demand parameter only - assert {"demand"} == set(data.keys()) - - # Certain labels are specifically excluded/dropped in the calculation - assert not {"transport pax ldv", "transport F WATER"} & set( - data["demand"]["commodity"].unique() - ) - assert {"useful"} == set(data["demand"]["level"].unique()) - - # No missing data - assert not data["demand"].isna().any().any() - - # No negative values - check = data["demand"]["value"] < 0 - if check.any(): # pragma: no cover - print(data["demand"][check].to_string()) - assert False, "Negative values in demand" - - @build.get_computer.minimum_version @pytest.mark.parametrize( "ssp", From a3b0ee6a7eb8d661ccff9f2f8e24caff8d2e0445 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 4 Dec 2024 23:13:39 +0100 Subject: [PATCH 21/45] Align keys in .transport.{freight,non_ldv} with .ldv --- message_ix_models/model/transport/demand.py | 16 +++++++-------- message_ix_models/model/transport/freight.py | 13 +++++++----- message_ix_models/model/transport/non_ldv.py | 20 +++++++++++-------- .../tests/model/transport/test_build.py | 2 +- .../tests/model/transport/test_demand.py | 2 +- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/message_ix_models/model/transport/demand.py b/message_ix_models/model/transport/demand.py index e2db4d41f..b029f417c 100644 --- a/message_ix_models/model/transport/demand.py +++ b/message_ix_models/model/transport/demand.py @@ -186,7 +186,7 @@ def dummy( # Relabel ((fv_cny, "relabel2", fv + "2"), dict(new_dims={"c": "transport F {t}"})), # Convert to ixmp format - (("t demand freight::ixmp", "as_message_df", fv_cny), _DEMAND_KW), + (("demand::F+ixmp", "as_message_df", fv_cny), _DEMAND_KW), # Select only non-LDV PDT ((pdt_nyt + "1", "select", pdt_nyt), dict(indexers=dict(t=["LDV"]), inverse=True)), # Relabel PDT @@ -196,21 +196,21 @@ def dummy( ), (pdt_cny, "convert_units", pdt_cny + "0", "Gp km / a"), # Convert to ixmp format - (("t demand pax non-ldv::ixmp", "as_message_df", pdt_cny), _DEMAND_KW), + (("demand::P+ixmp", "as_message_df", pdt_cny), _DEMAND_KW), # Relabel ldv pdt:n-y-cg ((ldv_cny + "0", "relabel2", ldv_nycg), dict(new_dims={"c": "transport pax {cg}"})), (ldv_cny, "convert_units", ldv_cny + "0", "Gp km / a"), - (("t demand pax ldv::ixmp", "as_message_df", ldv_cny), _DEMAND_KW), + (("demand::LDV+ixmp", "as_message_df", ldv_cny), _DEMAND_KW), # Dummy demands, if these are configured - ("t demand dummy::ixmp", dummy, "c::transport", "nodes::ex world", y, "config"), + ("demand::dummy+ixmp", dummy, "c::transport", "nodes::ex world", y, "config"), # Merge all data together ( "transport demand::ixmp", "merge_data", - "t demand pax ldv::ixmp", - "t demand pax non-ldv::ixmp", - "t demand freight::ixmp", - "t demand dummy::ixmp", + "demand::LDV+ixmp", + "demand::P+ixmp", + "demand::F+ixmp", + "demand::dummy+ixmp", ), ] diff --git a/message_ix_models/model/transport/freight.py b/message_ix_models/model/transport/freight.py index 3ca6cf93e..34b8a353d 100644 --- a/message_ix_models/model/transport/freight.py +++ b/message_ix_models/model/transport/freight.py @@ -26,6 +26,9 @@ level="l", ) +#: Shorthand for tags on keys +Fi = "::F+ixmp" + def prepare_computer(c: genno.Computer): from genno.core.attrseries import AttrSeries @@ -47,7 +50,7 @@ def prepare_computer(c: genno.Computer): c.add(k[2], "as_message_df", prev, name="input", dims=DIMS, common=COMMON) # Convert units - to_add.append("input::transport F+ixmp") + to_add.append(f"input{Fi}") c.add(to_add[-1], convert_units, k[2], "transport info") # Extract the 'input' data frame @@ -82,7 +85,7 @@ def prepare_computer(c: genno.Computer): c.add(ks[i + 2], "as_message_df", prev, name=par_name, dims=DIMS, common=COMMON) # Convert to target units - to_add.append("output::transport F+ixmp") + to_add.append(f"output{Fi}") c.add(to_add[-1], convert_units, ks[i + 2], "transport info") # Produce corresponding output, capacity_factor, technical_lifetime @@ -100,7 +103,7 @@ def prepare_computer(c: genno.Computer): c.add(k[5], convert_units, k[4], "transport info") # Fill values - to_add.append("other::transport F+ixmp") + to_add.append(f"other{Fi}") c.add(to_add[-1], same_node, k[5]) # Base values for conversion technologies @@ -116,7 +119,7 @@ def prepare_computer(c: genno.Computer): # Convert output to MESSAGE data structure c.add(k[10], "as_message_df", prev, name="output", dims=DIMS, common=COMMON) - to_add.append("usage output::transport F+ixmp") + to_add.append(f"usage output{Fi}") c.add(to_add[-1], lambda v: same_time(same_node(v)), k[10]) # Create corresponding input values in Gv km @@ -133,7 +136,7 @@ def prepare_computer(c: genno.Computer): prev = c.add( k[i + 3], "as_message_df", prev, name="input", dims=DIMS, common=COMMON ) - to_add.append("usage input::transport F+ixmp") + to_add.append(f"usage input{Fi}") c.add(to_add[-1], prev) # Merge data to one collection diff --git a/message_ix_models/model/transport/non_ldv.py b/message_ix_models/model/transport/non_ldv.py index 72b941219..141d459a8 100644 --- a/message_ix_models/model/transport/non_ldv.py +++ b/message_ix_models/model/transport/non_ldv.py @@ -57,6 +57,10 @@ Units: TJ """ +#: Shorthand for tags on keys +Oi = "::O+ixmp" +Pi = "::P+ixmp" + def prepare_computer(c: Computer): from . import files as exo @@ -85,7 +89,7 @@ def prepare_computer(c: Computer): keys.append(k + "emi") # Data for usage technologies - k_usage = "transport nonldv usage::ixmp" + k_usage = f"transport usage{Pi}" keys.append(k_usage) c.add(k_usage, usage_data, exo.load_factor_nonldv, t_modes, n, y) @@ -123,7 +127,7 @@ def prepare_computer(c: Computer): # Add minimum activity for transport technologies keys.extend(iter_keys(c.apply(bound_activity_lo))) - k_constraint = "constraints::ixmp+transport+non-ldv" + k_constraint = f"constraints{Pi}" keys.append(k_constraint) c.add(k_constraint, constraint_data, "t::transport", t_modes, n, y, "config") @@ -131,7 +135,7 @@ def prepare_computer(c: Computer): keys.extend(bound_activity(c)) # Add to the scenario - k_all = "transport nonldv::ixmp" + k_all = f"transport{Pi}" c.add(k_all, "merge_data", *keys) c.add("transport_data", __name__, key=k_all) @@ -189,7 +193,7 @@ def bound_activity(c: "Computer") -> list[Key]: dims=dict(node_loc="n", technology="t", year_act="y"), common=dict(mode="all", time="year"), ) - k_bau = Key("bound_activity_up::non_ldv+ixmp") + k_bau = Key(f"bound_activity_up{Pi}") c.add(k_bau, "as_message_df", base, name=k_bau.name, **kw) return [k_bau] @@ -352,9 +356,9 @@ def broadcast_other_transport(technologies) -> Quantity: dims=dict(node_loc="n", technology="t", year_act="y"), common=dict(mode="all", time="year"), ) - k_bal = Key("bound_activity_lo::transport other+ixmp") + k_bal = Key(f"bound_activity_lo{Oi}") c.add(k_bal, "as_message_df", k_cnty.prev, name=k_bal.name, **kw) - k_bau = Key("bound_activity_up::transport other+ixmp") + k_bau = Key(f"bound_activity_up{Oi}") c.add(k_bau, "as_message_df", k_cnty.prev, name=k_bau.name, **kw) # Divide by self to ensure values = 1.0 but same dimensionality @@ -365,10 +369,10 @@ def broadcast_other_transport(technologies) -> Quantity: # Produce MESSAGE parameter input:nl-t-yv-ya-m-no-c-l-h-ho kw["dims"].update(commodity="c", node_origin="n", year_vtg="y") kw["common"].update(level="final", time_origin="year") - k_input = Key("input::transport other+ixmp") + k_input = Key(f"input{Oi}") c.add(k_input, "as_message_df", k_cnty.prev, name=k_input.name, **kw) - result = Key("transport other::ixmp") + result = Key(f"transport{Oi}") c.add(result, "merge_data", k_bal, k_bau, k_input) return [result] diff --git a/message_ix_models/tests/model/transport/test_build.py b/message_ix_models/tests/model/transport/test_build.py index 8805124e2..332a4cae3 100644 --- a/message_ix_models/tests/model/transport/test_build.py +++ b/message_ix_models/tests/model/transport/test_build.py @@ -199,7 +199,7 @@ def test_build_existing(tmp_path, test_context, url, solve=False): HasUnits("dimensionless"), HasCoords({"commodity": ["transport F RAIL vehicle"]}), ), - "output::transport F+ixmp": ( + "output::F+ixmp": ( HasCoords( {"commodity": ["transport F RAIL vehicle", "transport F ROAD vehicle"]} ), diff --git a/message_ix_models/tests/model/transport/test_demand.py b/message_ix_models/tests/model/transport/test_demand.py index 6fd475067..137609542 100644 --- a/message_ix_models/tests/model/transport/test_demand.py +++ b/message_ix_models/tests/model/transport/test_demand.py @@ -65,7 +65,7 @@ def test_exo_pdt(test_context, ssp, regions="R12", years="B"): ) data = c.get("transport demand::ixmp") - # data = c.get("t demand pax non-ldv::ixmp") + # data = c.get("demand::P+ixmp") # Returns a dict with a single key/data frame assert {"demand"} == set(data.keys()) From a626290131856dc9e67e87bf812a08ce9983bcf2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 00:56:56 +0100 Subject: [PATCH 22/45] Allow no graphviz in .transport.build.get_computer() --- message_ix_models/model/transport/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/message_ix_models/model/transport/build.py b/message_ix_models/model/transport/build.py index f269aa13f..3610d140d 100644 --- a/message_ix_models/model/transport/build.py +++ b/message_ix_models/model/transport/build.py @@ -13,6 +13,7 @@ from message_ix_models.model import bare, build from message_ix_models.util import minimum_version from message_ix_models.util._logging import mark_time +from message_ix_models.util.graphviz import HAS_GRAPHVIZ from . import Config from .structure import get_technology_groups @@ -460,7 +461,7 @@ def get_computer( # Add tasks for debugging the build add_debug(c) - if visualize: + if visualize and HAS_GRAPHVIZ: path = context.get_local_path("transport", "build.svg") path.parent.mkdir(exist_ok=True) c.visualize(filename=path, key="add transport data") From 46fc9e05743d6969b948272d39fbc2970280a143 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 01:33:47 +0100 Subject: [PATCH 23/45] Derive freight cap. factor/lifetime from "output" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …instead of "input" parameter values. --- message_ix_models/model/transport/freight.py | 8 ++++---- message_ix_models/tests/model/transport/test_build.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/message_ix_models/model/transport/freight.py b/message_ix_models/model/transport/freight.py index 34b8a353d..26d08d845 100644 --- a/message_ix_models/model/transport/freight.py +++ b/message_ix_models/model/transport/freight.py @@ -53,9 +53,6 @@ def prepare_computer(c: genno.Computer): to_add.append(f"input{Fi}") c.add(to_add[-1], convert_units, k[2], "transport info") - # Extract the 'input' data frame - c.add(k[3], lambda d: d["input"], to_add[-1]) - # Create base quantity for "output" parameter # TODO Combine in a loop with "input", above—similar to .ldv k_output = genno.KeySeq("F output") @@ -88,7 +85,10 @@ def prepare_computer(c: genno.Computer): to_add.append(f"output{Fi}") c.add(to_add[-1], convert_units, ks[i + 2], "transport info") - # Produce corresponding output, capacity_factor, technical_lifetime + # Extract the 'output' data frame + c.add(k[3], lambda d: d["output"], to_add[-1]) + + # Produce corresponding capacity_factor and technical_lifetime c.add( k[4], partial( diff --git a/message_ix_models/tests/model/transport/test_build.py b/message_ix_models/tests/model/transport/test_build.py index 332a4cae3..f61c5b91f 100644 --- a/message_ix_models/tests/model/transport/test_build.py +++ b/message_ix_models/tests/model/transport/test_build.py @@ -204,10 +204,12 @@ def test_build_existing(tmp_path, test_context, url, solve=False): {"commodity": ["transport F RAIL vehicle", "transport F ROAD vehicle"]} ), ), + "other::F+ixmp": (HasCoords({"technology": ["f rail electr"]}),), "transport F::ixmp": ( ContainsDataForParameters( {"capacity_factor", "input", "output", "technical_lifetime"} ), + # HasCoords({"technology": ["f rail electr"]}), ), # # The following are intermediate checks formerly in .test_demand.test_exo From 1b8c10e3fd99c08acf8e04a7a3fdcedaf110a2a9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 02:52:15 +0100 Subject: [PATCH 24/45] Add CPLEX time limit to .transport.workflow --- message_ix_models/model/transport/workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/message_ix_models/model/transport/workflow.py b/message_ix_models/model/transport/workflow.py index 701814e8e..7e633c18a 100644 --- a/message_ix_models/model/transport/workflow.py +++ b/message_ix_models/model/transport/workflow.py @@ -28,6 +28,7 @@ iis=1, lpmethod=4, scaind=1, + tilim=30 * 60, ), ), ) From b77089220ad214412ccbd18a86b02bac2f392068 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 11:24:11 +0100 Subject: [PATCH 25/45] Reference .transport.files key in .non_ldv --- message_ix_models/model/transport/files.py | 6 ++++++ message_ix_models/model/transport/non_ldv.py | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/message_ix_models/model/transport/files.py b/message_ix_models/model/transport/files.py index dfafd851f..a647e51ef 100644 --- a/message_ix_models/model/transport/files.py +++ b/message_ix_models/model/transport/files.py @@ -229,6 +229,12 @@ def add(*, replace: bool = False, **kwargs): exception. kwargs : Passed on to :class:`ExogenousDataFile`. + + Returns + ------- + Key + The :attr:`ExogenousDataFile.key` at which the loaded and transformed data will + be available. """ edf = ExogenousDataFile(**kwargs) diff --git a/message_ix_models/model/transport/non_ldv.py b/message_ix_models/model/transport/non_ldv.py index 141d459a8..cee731e5a 100644 --- a/message_ix_models/model/transport/non_ldv.py +++ b/message_ix_models/model/transport/non_ldv.py @@ -119,8 +119,7 @@ def prepare_computer(c: Computer): # Handle data from the file energy-other.csv try: - k = Key("energy:c-nl:transport other") - keys.extend(iter_keys(c.apply(other, k))) + keys.extend(iter_keys(c.apply(other, exo.energy_other))) except MissingKeyError: log.warning(f"No key {k!r}; unable to add data for 'transport other *' techs") From b3718f5e18bd2be9eee1f1064adf734f784a3de7 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 11:27:02 +0100 Subject: [PATCH 26/45] Adjust parametrization of test_report_base_solved Use (regions=R12, years=B) to match 'main' model version. --- message_ix_models/tests/model/transport/test_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/tests/model/transport/test_report.py b/message_ix_models/tests/model/transport/test_report.py index 4dd1a53cf..10ceee4b2 100644 --- a/message_ix_models/tests/model/transport/test_report.py +++ b/message_ix_models/tests/model/transport/test_report.py @@ -60,7 +60,7 @@ def test_configure_legacy(): "regions, years", ( param("R11", "A", marks=make_mark[2](ValueError)), - param("R12", "A", marks=MARK[8]), + param("R12", "B", marks=MARK[8]), param("R14", "A", marks=make_mark[2](genno.ComputationError)), param("ISR", "A", marks=MARK[3]), ), From 3b6cf4a770d45870e68ea946d055ea1de75514a4 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 14:39:25 +0100 Subject: [PATCH 27/45] Bump initial_{activity,new_capacity}_up values Resolve infeasibilities with updated base model. --- message_ix_models/model/transport/config.py | 6 ++++-- message_ix_models/model/transport/non_ldv.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/message_ix_models/model/transport/config.py b/message_ix_models/model/transport/config.py index ba4ebd774..6970585bf 100644 --- a/message_ix_models/model/transport/config.py +++ b/message_ix_models/model/transport/config.py @@ -70,8 +70,10 @@ class Config(ConfigHelper): "non-LDV growth_activity_lo": -0.0192 * 1.0, "non-LDV growth_activity_up": 0.0192 * 2.0, "non-LDV growth_new_capacity_up": 0.0192 * 1.0, - "non-LDV initial_activity_up": 1.0, - "non-LDV initial_new_capacity_up": 1.0, + # NB If these values are not large enough, they can cause infeasibilities in + # the base period for technologies that do not have historical_activity + "non-LDV initial_activity_up": 2.0, + "non-LDV initial_new_capacity_up": 2.0, } ) diff --git a/message_ix_models/model/transport/non_ldv.py b/message_ix_models/model/transport/non_ldv.py index cee731e5a..5c6f224a3 100644 --- a/message_ix_models/model/transport/non_ldv.py +++ b/message_ix_models/model/transport/non_ldv.py @@ -199,7 +199,10 @@ def bound_activity(c: "Computer") -> list[Key]: def bound_activity_lo(c: Computer) -> list[Key]: - """Set minimum activity for certain technologies to ensure |y0| energy use.""" + """Set minimum activity for certain technologies to ensure |y0| energy use. + + Responds to values in :attr:`.Config.minimum_activity`. + """ @lru_cache def techs_for(mode: Code, commodity: str) -> list[Code]: From 2628708992fab2a94d7f0591ba91153c851c8b8d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 5 Dec 2024 15:44:14 +0100 Subject: [PATCH 28/45] Set all minimum_activity values to 0.01 --- .../data/transport/R12/config.yaml | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/message_ix_models/data/transport/R12/config.yaml b/message_ix_models/data/transport/R12/config.yaml index ca36cf7a3..8bce0e031 100644 --- a/message_ix_models/data/transport/R12/config.yaml +++ b/message_ix_models/data/transport/R12/config.yaml @@ -37,33 +37,33 @@ minimum activity: # IEA EWEB. These values are computed to make those scale factors come out near # 1.0. # TODO reduce or remove these once technology activities are calibrated directly - - [R12_AFR, ROAD, ethanol, 0.0780] - - [R12_CHN, RAIL, electr, 2.33] - - [R12_CHN, ROAD, ethanol, 5.36] - - [R12_CHN, ROAD, gas, 23.1] - - [R12_EEU, RAIL, electr, 0.219] - - [R12_EEU, ROAD, gas, 0.260] - - [R12_FSU, RAIL, electr, 1.42] - - [R12_FSU, RAIL, lightoil, 1.833] - - [R12_FSU, ROAD, electr, 0.121] - - [R12_FSU, ROAD, gas, 3.68] - - [R12_LAM, RAIL, electr, 0.102] - - [R12_LAM, RAIL, lightoil, 0.615] - - [R12_LAM, ROAD, gas, 3.94] - - [R12_MEA, RAIL, electr, 0.0566] - - [R12_MEA, RAIL, lightoil, 0.0928] - - [R12_MEA, ROAD, gas, 6.40] - - [R12_NAM, RAIL, electr, 0.111] - - [R12_PAO, RAIL, electr, 0.493] - - [R12_PAO, ROAD, gas, 0.0861] - - [R12_PAS, RAIL, electr, 0.184] - - [R12_PAS, ROAD, gas, 2.21] - - [R12_RCPA, RAIL, lightoil, 0.00849] - - [R12_RCPA, ROAD, gas, 0.0198] - - [R12_SAS, RAIL, electr, 0.442] - - [R12_SAS, ROAD, gas, 3.643] - - [R12_WEU, RAIL, electr, 1.03] - - [R12_WEU, ROAD, gas, 1.58] + - [R12_AFR, ROAD, ethanol, 0.01] + - [R12_CHN, RAIL, electr, 0.01] + - [R12_CHN, ROAD, ethanol, 0.01] + - [R12_CHN, ROAD, gas, 0.01] + - [R12_EEU, RAIL, electr, 0.01] + - [R12_EEU, ROAD, gas, 0.01] + - [R12_FSU, RAIL, electr, 0.01] + - [R12_FSU, RAIL, lightoil, 0.01] + - [R12_FSU, ROAD, electr, 0.01] + - [R12_FSU, ROAD, gas, 0.01] + - [R12_LAM, RAIL, electr, 0.01] + - [R12_LAM, RAIL, lightoil, 0.01] + - [R12_LAM, ROAD, gas, 0.01] + - [R12_MEA, RAIL, electr, 0.01] + - [R12_MEA, RAIL, lightoil, 0.01] + - [R12_MEA, ROAD, gas, 0.01] + - [R12_NAM, RAIL, electr, 0.01] + - [R12_PAO, RAIL, electr, 0.01] + - [R12_PAO, ROAD, gas, 0.01] + - [R12_PAS, RAIL, electr, 0.01] + - [R12_PAS, ROAD, gas, 0.01] + - [R12_RCPA, RAIL, lightoil, 0.01] + - [R12_RCPA, ROAD, gas, 0.01] + - [R12_SAS, RAIL, electr, 0.01] + - [R12_SAS, ROAD, gas, 0.01] + - [R12_WEU, RAIL, electr, 0.01] + - [R12_WEU, ROAD, gas, 0.01] share weight convergence: # Settings from MESSAGE (V)-Transport From 3c7c0dd31f0f4ef2a8d4f8cce4b3c84b35932dbc Mon Sep 17 00:00:00 2001 From: r-aneeque <114144149+r-aneeque@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:05:49 +0100 Subject: [PATCH 29/45] Reduce R12_WEU --- message_ix_models/data/transport/R12/freight-activity.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/data/transport/R12/freight-activity.csv b/message_ix_models/data/transport/R12/freight-activity.csv index d5e29be23..25fa1281f 100644 --- a/message_ix_models/data/transport/R12/freight-activity.csv +++ b/message_ix_models/data/transport/R12/freight-activity.csv @@ -21,4 +21,4 @@ R12_PAO, 763.1 R12_PAS, 2800.3 R12_RCPA, 700.1 R12_SAS, 1221.9 -R12_WEU, 3497.0 +R12_WEU, 3000 From 42d6a665eaaf3b3e20c19f06047578337b4613b6 Mon Sep 17 00:00:00 2001 From: r-aneeque <114144149+r-aneeque@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:09:46 +0100 Subject: [PATCH 30/45] adjust R12_WEU modes shares to match 2019 based estimates --- message_ix_models/data/transport/R12/mode-share/default.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/message_ix_models/data/transport/R12/mode-share/default.csv b/message_ix_models/data/transport/R12/mode-share/default.csv index f9ae57571..9ac510e91 100644 --- a/message_ix_models/data/transport/R12/mode-share/default.csv +++ b/message_ix_models/data/transport/R12/mode-share/default.csv @@ -69,7 +69,7 @@ R12_SAS, BUS, 0.33 R12_SAS, LDV, 0.20 R12_SAS, RAIL, 0.16 R12_WEU, 2W, 0.02 -R12_WEU, AIR, 0.11 +R12_WEU, AIR, 0.09 R12_WEU, BUS, 0.07 -R12_WEU, LDV, 0.70 -R12_WEU, RAIL, 0.10 +R12_WEU, LDV, 0.71 +R12_WEU, RAIL, 0.11 From e5ba059fe46c0340bbb8dd117c0a3cbac8e074ab Mon Sep 17 00:00:00 2001 From: r-aneeque <114144149+r-aneeque@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:16:59 +0100 Subject: [PATCH 31/45] Reduce R12_MEA purely based on very high 2025 demand numbers that are unrealistic --- message_ix_models/data/transport/R12/freight-activity.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/data/transport/R12/freight-activity.csv b/message_ix_models/data/transport/R12/freight-activity.csv index 25fa1281f..a6cc45967 100644 --- a/message_ix_models/data/transport/R12/freight-activity.csv +++ b/message_ix_models/data/transport/R12/freight-activity.csv @@ -15,7 +15,7 @@ R12_CHN, 8753.3 R12_EEU, 388.6 R12_FSU, 810.4 R12_LAM, 3100.6 -R12_MEA, 1210.1 +R12_MEA, 1110.1 R12_NAM, 5148.9 R12_PAO, 763.1 R12_PAS, 2800.3 From 97e2ad107a2c2b0f52bf024ad6a39d893dcd676f Mon Sep 17 00:00:00 2001 From: r-aneeque <114144149+r-aneeque@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:19:17 +0100 Subject: [PATCH 32/45] Update R12_MEA mode share purely based on very high 2025 demand numbers that are unrealistic. Aviation adjustment based on scale-1 --- message_ix_models/data/transport/R12/mode-share/default.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/message_ix_models/data/transport/R12/mode-share/default.csv b/message_ix_models/data/transport/R12/mode-share/default.csv index 9ac510e91..55d97f0b7 100644 --- a/message_ix_models/data/transport/R12/mode-share/default.csv +++ b/message_ix_models/data/transport/R12/mode-share/default.csv @@ -39,10 +39,10 @@ R12_LAM, BUS, 0.36 R12_LAM, LDV, 0.50 R12_LAM, RAIL, 0.01 R12_MEA, 2W, 0.05 -R12_MEA, AIR, 0.13 +R12_MEA, AIR, 0.11 R12_MEA, BUS, 0.38 -R12_MEA, LDV, 0.43 -R12_MEA, RAIL, 0.01 +R12_MEA, LDV, 0.445 +R12_MEA, RAIL, 0.015 R12_NAM, 2W, 0.005 R12_NAM, AIR, 0.12 R12_NAM, BUS, 0.06 From b337a6998031fb29a5bcf01f96d9accd961743ec Mon Sep 17 00:00:00 2001 From: r-aneeque <114144149+r-aneeque@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:21:42 +0100 Subject: [PATCH 33/45] R12_MEA freight-mode-share adjustment based on scale-1 and high 2025 values --- .../data/transport/R12/freight-mode-share-ref.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/message_ix_models/data/transport/R12/freight-mode-share-ref.csv b/message_ix_models/data/transport/R12/freight-mode-share-ref.csv index f4766a9eb..7d767cde8 100644 --- a/message_ix_models/data/transport/R12/freight-mode-share-ref.csv +++ b/message_ix_models/data/transport/R12/freight-mode-share-ref.csv @@ -27,9 +27,9 @@ R12_LAM, RAIL, 0.007 R12_LAM, WATER, 0.648 R12_LAM, AIR, 0.001 R12_MEA, ROAD, 0.965 -R12_MEA, RAIL, 0.019 +R12_MEA, RAIL, 0.025 R12_MEA, WATER, 0.0 -R12_MEA, AIR, 0.016 +R12_MEA, AIR, 0.01 R12_NAM, ROAD, 0.590 R12_NAM, RAIL, 0.350 R12_NAM, WATER, 0.0 From eb3a9412de802faeceda867a2628fe32437ec38c Mon Sep 17 00:00:00 2001 From: r-aneeque <114144149+r-aneeque@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:27:33 +0100 Subject: [PATCH 34/45] R12_SAS mode share change based on scale 1 and 2025 values (AIR is too high and demand is too high as well) --- message_ix_models/data/transport/R12/mode-share/default.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/message_ix_models/data/transport/R12/mode-share/default.csv b/message_ix_models/data/transport/R12/mode-share/default.csv index 55d97f0b7..3e985df67 100644 --- a/message_ix_models/data/transport/R12/mode-share/default.csv +++ b/message_ix_models/data/transport/R12/mode-share/default.csv @@ -64,9 +64,9 @@ R12_RCPA, BUS, 0.30 R12_RCPA, LDV, 0.16 R12_RCPA, RAIL, 0.30 R12_SAS, 2W, 0.24 -R12_SAS, AIR, 0.07 -R12_SAS, BUS, 0.33 -R12_SAS, LDV, 0.20 +R12_SAS, AIR, 0.04 +R12_SAS, BUS, 0.34 +R12_SAS, LDV, 0.22 R12_SAS, RAIL, 0.16 R12_WEU, 2W, 0.02 R12_WEU, AIR, 0.09 From 05eb9c1bd46c463d42554939389c89f63d4cce4b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 6 Dec 2024 17:23:49 +0100 Subject: [PATCH 35/45] Recompute minimum_activity values for R12 Remove values for: - R12_AFR, ROAD, ethanol - R12_FSU, ROAD, electr - R12_LAM, ROAD, gas - R12_MEA, RAIL, lightoil - R12_PAO, ROAD, gas - R12_RCPA, RAIL, lightoil - R12_RCPA, ROAD, gas - Add values for: - R12_AFR, RAIL, lightoil - R12_CHN, ROAD, electr - R12_NAM, RAIL, lightoil - R12_PAO, ROAD, ethanol - R12_RCPA, ROAD, ethanol - R12_SAS, ROAD, ethanol - Adjust all others --- .../data/transport/R12/config.yaml | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/message_ix_models/data/transport/R12/config.yaml b/message_ix_models/data/transport/R12/config.yaml index 8bce0e031..e9e3d6fd5 100644 --- a/message_ix_models/data/transport/R12/config.yaml +++ b/message_ix_models/data/transport/R12/config.yaml @@ -31,39 +31,38 @@ node to census_division: minimum activity: # Source: manually calculated from scale-1.csv for - # ixmp://ixmp-dev/MESSAGEix-GLOBIOM 1.1-T-R12 ci nightly/SSP_2024.2 baseline#73 - # and #76. In those scenarios, all the minimum-activity values were set to 0.01. - # The resulting scale factors indicate the mismatch vs. expected energy input per + # ixmp://ixmp-dev/MESSAGEix-GLOBIOM 1.1-T-R12 ci nightly/SSP_2024.2 baseline#547 + # In those scenarios, all the minimum-activity values were set to 0.01. The + # resulting scale factors indicate the mismatch vs. expected energy input per # IEA EWEB. These values are computed to make those scale factors come out near # 1.0. # TODO reduce or remove these once technology activities are calibrated directly - - [R12_AFR, ROAD, ethanol, 0.01] - - [R12_CHN, RAIL, electr, 0.01] - - [R12_CHN, ROAD, ethanol, 0.01] - - [R12_CHN, ROAD, gas, 0.01] - - [R12_EEU, RAIL, electr, 0.01] - - [R12_EEU, ROAD, gas, 0.01] - - [R12_FSU, RAIL, electr, 0.01] - - [R12_FSU, RAIL, lightoil, 0.01] - - [R12_FSU, ROAD, electr, 0.01] - - [R12_FSU, ROAD, gas, 0.01] - - [R12_LAM, RAIL, electr, 0.01] - - [R12_LAM, RAIL, lightoil, 0.01] - - [R12_LAM, ROAD, gas, 0.01] - - [R12_MEA, RAIL, electr, 0.01] - - [R12_MEA, RAIL, lightoil, 0.01] - - [R12_MEA, ROAD, gas, 0.01] - - [R12_NAM, RAIL, electr, 0.01] - - [R12_PAO, RAIL, electr, 0.01] - - [R12_PAO, ROAD, gas, 0.01] - - [R12_PAS, RAIL, electr, 0.01] - - [R12_PAS, ROAD, gas, 0.01] - - [R12_RCPA, RAIL, lightoil, 0.01] - - [R12_RCPA, ROAD, gas, 0.01] - - [R12_SAS, RAIL, electr, 0.01] - - [R12_SAS, ROAD, gas, 0.01] - - [R12_WEU, RAIL, electr, 0.01] - - [R12_WEU, ROAD, gas, 0.01] + - [R12_AFR, RAIL, lightoil, 0.028] + - [R12_CHN, RAIL, electr, 2.329] + - [R12_CHN, ROAD, electr, 0.036] + - [R12_CHN, ROAD, ethanol, 0.073] + - [R12_CHN, ROAD, gas, 0.292] + - [R12_EEU, RAIL, electr, 0.210] + - [R12_EEU, ROAD, gas, 0.020] + - [R12_FSU, RAIL, electr, 1.553] + - [R12_FSU, RAIL, lightoil, 0.046] + - [R12_FSU, ROAD, gas, 0.406] + - [R12_LAM, RAIL, electr, 0.100] + - [R12_LAM, RAIL, lightoil, 0.135] + - [R12_MEA, RAIL, electr, 0.055] + - [R12_MEA, ROAD, gas, 0.240] + - [R12_NAM, RAIL, electr, 0.175] + - [R12_NAM, RAIL, lightoil, 0.500] + - [R12_PAO, RAIL, electr, 0.492] + - [R12_PAO, ROAD, ethanol, 0.059] + - [R12_PAS, RAIL, electr, 0.197] + - [R12_PAS, ROAD, gas, 0.178] + - [R12_RCPA, ROAD, ethanol, 0.130] + - [R12_SAS, RAIL, electr, 0.457] + - [R12_SAS, ROAD, ethanol, 0.070] + - [R12_SAS, ROAD, gas, 0.057] + - [R12_WEU, RAIL, electr, 1.168] + - [R12_WEU, ROAD, gas, 0.049] share weight convergence: # Settings from MESSAGE (V)-Transport From 07c7915dbcf12649cc8a72dfb21eca0f1ec73576 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2024 17:16:52 +0100 Subject: [PATCH 36/45] Add transform="B" option for IEA EWEB data (#230) --- message_ix_models/tools/iea/web.py | 162 +++++++++++++++++++++++++---- 1 file changed, 143 insertions(+), 19 deletions(-) diff --git a/message_ix_models/tools/iea/web.py b/message_ix_models/tools/iea/web.py index a227fda4c..77cd47e1d 100644 --- a/message_ix_models/tools/iea/web.py +++ b/message_ix_models/tools/iea/web.py @@ -5,13 +5,16 @@ from collections.abc import Iterable from copy import copy from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Literal, Optional +import genno import pandas as pd -from genno import Key, Quantity +from genno import Key from genno.core.key import single_key +from genno.operator import concat from platformdirs import user_cache_path +from message_ix_models.model.structure import get_codelist from message_ix_models.tools.exo_data import ExoDataSource, register_source from message_ix_models.util import cached, package_data_path, path_fallback from message_ix_models.util._logging import silence_log @@ -20,13 +23,15 @@ import os import genno + from genno.types import AnyQuantity + from message_ix_models.types import KeyLike from message_ix_models.util.common import MappingAdapter log = logging.getLogger(__name__) -#: ISO 3166-1 alpha-3 codes for “COUNTRY” codes appearing in the 2024 edition. This -#: mapping only includes values that are not matched by :func:`pycountry.lookup`. See +#: ISO 3166-1 alpha-3 codes for 'COUNTRY' codes appearing in the 2024 edition. This +#: mapping includes only values that are not matched by :func:`pycountry.lookup`. See #: :func:`.iso_3166_alpha_3`. COUNTRY_NAME = { "AUSTRALI": "AUS", @@ -94,15 +99,17 @@ class IEA_EWEB(ExoDataSource): """Provider of exogenous data from the IEA Extended World Energy Balances. To use data from this source, call :func:`.exo_data.prepare_computer` with the - :py:`source_kw`: - - - "provider": Either "IEA" or "OECD". See :data:`.FILES`. - - "edition": one of "2021", "2022", or "2023". See :data:`.FILES`. - - "product": :class:`str` or :class:`list` of :class:`str`. - - "flow": :class:`str` or :class:`list` of :class:`str`. - - The returned data have the extra dimensions "product" and "flow", and are not - aggregated by year. + following `source_kw`: + + - :py:`provider`: Either 'IEA' or 'OECD'. See :data:`.FILES`. + - :py:`edition`: one of '2021', '2022', or '2023'. See :data:`.FILES`. + - :py:`product` (optional): :class:`str` or :class:`list` of :class:`str`. Select + only these labels from the 'PRODUCT' dimension. + - :py:`flow` (optional): :class:`str` or :class:`list` of :class:`str`. Select only + these labels from the 'FLOW' dimension. + - :py:`transform` (optional): either "A" (default) or "B". See :meth:`.transform`. + - :py:`regions`: **must** also be given with the value :py:`"R12"` if giving + :py:`transform="B"`. Example ------- @@ -141,6 +148,22 @@ def __init__(self, source, source_kw): if flow := _kw.pop("flow", None): self.indexers.update(flow=flow) + # Handle the 'transform' keyword + self.transform_method = _kw.pop("transform", "A") + regions = _kw.pop("regions", None) + if self.transform_method not in "AB": + raise ValueError(f"transform={self.transform_method!r}") + elif self.transform_method == "B": + if (p, e) != ("IEA", "2024"): + raise ValueError( + f"transform='B' only supported for (provider='IEA', " + f"edition='2024'); got {(p, e)!r}" + ) + elif regions != "R12": + raise ValueError( + f"transform='B' only supported for regions='R12'; got {regions!r}" + ) + if len(_kw): raise ValueError(_kw) @@ -155,7 +178,7 @@ def __call__(self): # - Map dimensions. # - Apply `indexers` to select. return ( - Quantity( + genno.Quantity( load_data( provider=self.provider, edition=self.edition, path=self.path ).set_index(DIMS)["Value"], @@ -166,13 +189,42 @@ def __call__(self): ) def transform(self, c: "genno.Computer", base_key: "genno.Key") -> "genno.Key": - """Aggregate only; do not interpolate on "y".""" + """Prepare `c` to transform raw data from `base_key`. + + 1. Map IEA ``COUNTRY`` codes to ISO 3166-1 alpha-3 codes, where such mapping + exists. See :func:`get_mapping` and :data:`COUNTRY_NAME`. + + The next steps depend on whether :py:`transform="A"` or :py:`transform="B"` was + given with the `source_kw`. + + :py:`transform="A"` (default) + 2. Aggregate using "n::groups"—the same as :meth:`.ExoDataSource.transform`. + This operates on the |n| labels transformed to alpha-3 codes by step (1) + above. + + :py:`transform="B"` + 2. Compute intermediate quantities using :func:`.transform_B`. + 3. Aggregate using the groups returned by :func:`get_node_groups_B`. + + This method does *not* prepare interpolation or aggregation on |y|. + """ # Map values like RUSSIA appearing in the (IEA, 2024) edition to e.g. RUS adapter = get_mapping(self.provider, self.edition) k = c.add(base_key + "adapted", adapter, base_key) - return single_key( - c.add(base_key + "agg", "aggregate", k, "n::groups", keep=False) - ) + + if self.transform_method == "A": + # Key for aggregation groups: hierarchy from the standard code lists, + # already added by .exo_data.prepare_computer() + k_n_agg: "KeyLike" = "n::groups" + elif self.transform_method == "B": + # Derive intermediate values "_IIASA_{AFR,PAS,SAS}" + k = c.add(base_key + "adapted" + "2", transform_B, k) + + # Add groups for aggregation, including these intermediate values + k_n_agg = single_key(c.add(f"n::groups+{self.id}", get_node_groups_B)) + + # Aggregate on 'n' dimension using the `k_n_agg` + return single_key(c.add(base_key + "agg", "aggregate", k, k_n_agg, keep=False)) def fwf_to_csv(path: Path, progress: bool = False) -> Path: # pragma: no cover @@ -354,7 +406,7 @@ def dir_fallback(*parts, **kwargs) -> Path: def get_mapping(provider: str, edition: str) -> "MappingAdapter": """Return a Mapping Adapter from codes appearing in IEA EWEB data. - For each code in the ``COUNTRY`` code list for (`provider`, `edition`) that is a + For each code in the IEA 'COUNTRY' code list for (`provider`, `edition`) that is a country name, the adapter maps the name to a corresponding ISO 3166-1 alpha-3 code. :data:`COUNTRY_NAME` is used for values particular to IEA EWEB. @@ -377,3 +429,75 @@ def get_mapping(provider: str, edition: str) -> "MappingAdapter": maps[dim].append((code.id, new_id)) return MappingAdapter(maps) + + +def get_node_groups_B() -> dict[Literal["n"], dict[str, list[str]]]: + """Return groups for aggregating on |n| as part of the :py:`transform='B'` method. + + These are of three kinds: + + 1. For the nodes 'R12_FSU', 'R12_PAO', 'R12_RCPA', 'R12_WEU', the common :ref:`R12` + is used. + 2. For the nodes 'R12_CHN', 'R12_MEA', 'R12_LAM', 'R12_NAM', the labels in the + ``material-region`` annotation are used. These may reference certain labels + specific to IEA EWEB; omit certain alpha-3 codes present in (1); or both. + 3. For the nodes 'R12_AFR', 'R12_PAS', 'R12_SAS', a mix of the codes generated by + :func:`transform_B` and alpha-3 codes are used. + + .. note:: This function mirrors the behaviour of code that is not present in + :mod:`message_ix_models` using a file :file:`R12_SSP_V1.yaml` that is also not + present. See + `iiasa/message-ix-models#201 `_ + for a detailed discussion. + + See also + -------- + .IEA_EWEB.transform + """ + result = dict( + R12_AFR=["_IIASA_AFR"], + R12_PAS="KOR IDN MYS MMR PHL SGP THA BRN TWN _IIASA_PAS".split(), + R12_SAS="BGD IND NPL PAK LKA _IIASA_SAS".split(), + ) + + cl = get_codelist("node/R12") + for n in "R12_FSU", "R12_PAO", "R12_RCPA", "R12_WEU": + result[n] = [c.id for c in cl[n].child] + for n in "R12_CHN", "R12_MEA", "R12_EEU", "R12_LAM", "R12_NAM": + result[n] = cl[n].eval_annotation(id="material-region") + + return dict(n=result) + + +def transform_B(qty: "AnyQuantity") -> "AnyQuantity": + """Compute some derived intermediate labels along the |n| dimension of `qty`. + + These are used via :meth:`.IEA_EWEB.transform` in the aggregations specified by + :func:`get_node_groups_B`. + + 1. ``_IIASA_AFR = AFRICA - DZA - EGY - LBY - MAR - SDN - SSD - TUN``. Note that + 'AFRICA' is 'AFRICATOT' in the reference notebook, but no such label appears in + the data. + 2. ``_IIASA_PAS = UNOCEANIA - AUS - NZL``. Note that 'UNOCEANIA' is 'OCEANIA' in the + reference notebook, but no such label appears in the data. + 3. ``_IIASA_SAS = OTHERASIA - _IIASA_PAS``. + + .. note:: This function mirrors the behaviour of code in a file + :file:`Step2_REGIONS.ipynb` that is not present in :mod:`message_ix_models`. + + Returns + ------- + genno.Quantity + the original `qty` with 3 appended |n| labels as above. + """ + n_afr = ["DZA", "EGY", "LBY", "MAR", "SDN", "SSD", "TUN"] + q_afr = qty.sel(n="AFRICA", drop=True) - qty.sel(n=n_afr).sum(dim="n") + q_pas = qty.sel(n="UNOCEANIA", drop=True) - qty.sel(n=["AUS", "NZL"]).sum(dim="n") + q_sas = qty.sel(n="OTHERASIA", drop=True) - q_pas + + return concat( + qty, + q_afr.expand_dims({"n": ["_IIASA_AFR"]}), + q_pas.expand_dims({"n": ["_IIASA_PAS"]}), + q_sas.expand_dims({"n": ["_IIASA_SAS"]}), + ) From ba0d35ce13e7f511fc371a9ce2667a396af4c30f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2024 17:18:19 +0100 Subject: [PATCH 37/45] Test transform="B" option for .iea.web - Store context.core.local_data on pytestconfig for reference. - Add user_local_data fixture for development. --- message_ix_models/testing/__init__.py | 5 ++- message_ix_models/tests/tools/iea/test_web.py | 35 +++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/message_ix_models/testing/__init__.py b/message_ix_models/testing/__init__.py index c405d9b85..8b08f3f42 100644 --- a/message_ix_models/testing/__init__.py +++ b/message_ix_models/testing/__init__.py @@ -88,8 +88,11 @@ def session_context(pytestconfig, tmp_env): else Path(pytestconfig.cache.makedir("cache")) ) + # Store current .util.config.Config.local_data setting from the user's configuration + pytestconfig.user_local_data = ctx.core.local_data + # Other local data in the temporary directory for this session only - ctx.local_data = session_tmp_dir + ctx.core.local_data = session_tmp_dir # Also set the "message local data" key in the ixmp config ixmp_config.set("message local data", session_tmp_dir) diff --git a/message_ix_models/tests/tools/iea/test_web.py b/message_ix_models/tests/tools/iea/test_web.py index 0a34ec57f..fe0198633 100644 --- a/message_ix_models/tests/tools/iea/test_web.py +++ b/message_ix_models/tests/tools/iea/test_web.py @@ -1,6 +1,7 @@ """Tests of :mod:`.tools`.""" from importlib.metadata import version +from typing import TYPE_CHECKING import pandas as pd import pytest @@ -17,6 +18,9 @@ ) from message_ix_models.util import HAS_MESSAGE_DATA +if TYPE_CHECKING: + from collections.abc import Generator + # Dask < 2024.4.1 is incompatible with Python >= 3.11.9, but we pin dask in this range # for tests of message_ix < 3.7.0. Skip these tests: MARK_DASK_PYTHON = pytest.mark.skipif( @@ -25,8 +29,27 @@ ) +@pytest.fixture +def user_local_data(pytestconfig, request) -> "Generator": # pragma: no cover + """Symlink :path:`…/iea/` in the test local data directory to the user's.""" + if "test_context" not in request.fixturenames: + return + test_local_data = request.getfixturevalue("test_context").core.local_data + user_local_data = pytestconfig.user_local_data + + source = test_local_data.joinpath("iea") + source.symlink_to(user_local_data.joinpath("iea")) + + try: + yield + finally: + source.unlink() + + class TestIEA_EWEB: @MARK_DASK_PYTHON + # Uncomment the following line to use the full data files from a local copy + # @pytest.mark.usefixtures("user_local_data") @pytest.mark.parametrize("source", ("IEA_EWEB",)) @pytest.mark.parametrize( "source_kw", @@ -55,6 +78,13 @@ class TestIEA_EWEB: marks=pytest.mark.xfail(raises=ValueError), ), dict(provider="IEA", edition="2024", flow=["AVBUNK"]), + pytest.param( + dict(provider="IEA", edition="2024", transform="B"), + marks=pytest.mark.xfail( + raises=ValueError, reason="Missing regions= kwarg" + ), + ), + dict(provider="IEA", edition="2024", transform="B", regions="R12"), ), ) def test_prepare_computer(self, test_context, source, source_kw): @@ -75,8 +105,9 @@ def test_prepare_computer(self, test_context, source, source_kw): # Data contain expected coordinates # NB cannot test total counts here because the fuzzed test data does not - # necessarily include ≥1 data point from each COUNTRY and TIME - assert {"R14_AFR", "R14_WEU"} < set(result.coords["n"].data) + # necessarily include ≥1 data point from each (n, y) + n = source_kw.get("regions", "R14") + assert {f"{n}_AFR", f"{n}_WEU"} < set(result.coords["n"].data) assert {1980, 2018} < set(result.coords["y"].data) From 77332eec8d562763013534952b702c045a522f43 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2024 17:21:54 +0100 Subject: [PATCH 38/45] Edit IEA EWEB documentation --- doc/api/data-sources.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/api/data-sources.rst b/doc/api/data-sources.rst index c5f96d315..003fce35c 100644 --- a/doc/api/data-sources.rst +++ b/doc/api/data-sources.rst @@ -63,7 +63,7 @@ The data: The approach to handling proprietary data is the same as in :mod:`.project.advance` and :mod:`.project.ssp`: -- Copies of the data are stored in the (private) :mod:`message_data` repository using Git LFS. +- Copies of the data are stored in the (private) `message-static-data` repository using Git LFS. This respository is accessible only to users who have a license for the data. - :mod:`message_ix_models` contains only a ‘fuzzed’ version of the data (same structure, random values) for testing purposes. - Non-IIASA users must obtain their own license to access and use the data; obtain the data themselves; and place it on the system where they use :mod:`message_ix_models`. @@ -71,8 +71,8 @@ The approach to handling proprietary data is the same as in :mod:`.project.advan The module :mod:`message_ix_models.tools.iea.web` attempts to detect and support both the providers/formats described below. The code supports using data from any of the above locations and formats, in multiple ways: -- Use :func:`.tools.iea.web.load_data` to load data as :class:`pandas.DataFrame` and apply further pandas processing. -- Use :class:`.IEA_EWEB` via :func:`.tools.exo_data.prepare_computer` to use the data in :mod:`genno` structured calculations. +- Use :class:`.IEA_EWEB` via :func:`.exo_data.prepare_computer` to use the data in :mod:`genno` structured calculations. +- Use :func:`.iea.web.load_data` to load data as :class:`pandas.DataFrame` and apply further processing using pandas. The **documentation** for the `2023 edition `__ of the IEA source/format is publicly available. @@ -82,8 +82,8 @@ Structure The data have the following conceptual dimensions, each enumerated by a different list of codes: - ``FLOW``, ``PRODUCT``: for both of these, the lists of codes appearing in the data are the same from 2021 and 2023 inclusive. -- ``COUNTRY``: The data provided by IEA directly contain codes that are all caps, abbreviated country names, for instance "DOMINICANR". - The data provided by the OECD contain ISO 3166-1 alpha-3 codes, for instance "DOM". +- ``COUNTRY``: The data provided by IEA directly contain codes that are all caps, abbreviated country names, for instance 'DOMINICANR'. + The data provided by the OECD contain ISO 3166-1 alpha-3 codes, for instance 'DOM'. In both cases, there are additional labels denoting country groupings; these are defined in the documentation linked above. Changes visible in these lists include: @@ -98,8 +98,9 @@ The data have the following conceptual dimensions, each enumerated by a differen - New codes: GNQ, MDG, MKD, RWA, SWZ, UGA. - Removed: EQGUINEA, GREENLAND, MALI, MBURKINAFA, MCHAD, MMADAGASCA, MMAURITANI, MPALESTINE, MRWANDA, MUGANDA, NORTHMACED. -- TIME: always a year. -- MEASURE: unit of measurement, either "TJ" or "ktoe". + See the :py:`transform=...` source keyword argument and :meth:`.IEA_EWEB.transform` for different methods of handling this dimension. +- ``TIME``: always a year. +- ``UNIT_MEASURE`` (not labeled): unit of measurement, either 'TJ' or 'ktoe'. :mod:`message_ix_models` is packaged with SDMX structure data (stored in :file:`message_ix_models/data/sdmx/`) comprising code lists extracted from the raw data for the COUNTRY, FLOW, and PRODUCT dimensions. These can be used with other package utilities, for instance: From eabd4c531c416b354ac7c467db94ee78f2d74036 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2024 17:55:42 +0100 Subject: [PATCH 39/45] Organize .project.ssp.transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sort variables and functions. - Split main() to prepare_method_A(). - Add main(…, method=…) argument. - Add stub prepare_method_B(). --- message_ix_models/project/ssp/transport.py | 187 ++++++++++++--------- 1 file changed, 104 insertions(+), 83 deletions(-) diff --git a/message_ix_models/project/ssp/transport.py b/message_ix_models/project/ssp/transport.py index 8bd89b294..cd092138d 100644 --- a/message_ix_models/project/ssp/transport.py +++ b/message_ix_models/project/ssp/transport.py @@ -14,6 +14,9 @@ from genno.types import AnyQuantity +#: Expression for IAMC ‘variable’ names used in :func:`main`. +EXPR = r"^Emissions\|(?P[^\|]+)\|Energy\|Demand\|Transportation(?:\|(?P.*))?$" + def aviation_share(ref: "AnyQuantity") -> "AnyQuantity": """Return (dummy) data for the share of aviation in emissions. @@ -39,57 +42,6 @@ def aviation_share(ref: "AnyQuantity") -> "AnyQuantity": ) -def finalize( - q_all: "AnyQuantity", - q_update: "AnyQuantity", - model_name: str, - scenario_name: str, - path_out: "pathlib.Path", -) -> None: - """Finalize output. - - 1. Reattach "Model" and "Scenario" labels. - 2. Reassemble the "Variable" dimension/coords of `q_update`; drop "e" and "t". - 3. Convert both `q_all` and `q_update` to :class:`pandas.Series`; update the former - with the contents of the latter. - 4. Adjust to IAMC ‘wide’ structure and write to `path_out`. - - Parameters - ---------- - q_all : - All data. - q_update : - Revised data to overwrite corresponding values in `q_all`. - """ - - def _expand(qty): - return qty.expand_dims( - {"Model": [model_name], "Scenario": [scenario_name]} - ).rename({"n": "Region", "UNIT": "Unit", "VARIABLE": "Variable"}) - - s_all = q_all.pipe(_expand).to_series() - - s_all.update( - q_update.pipe(_expand) - .to_frame() - .reset_index() - .assign( - Variable=lambda df: ( - "Emissions|" + df["e"] + "|Energy|Demand|Transportation|" + df["t"] - ).str.replace("|_T", "") - ) - .drop(["e", "t"], axis=1) - .set_index(s_all.index.names)[0] - ) - - ( - s_all.unstack("y") - .reorder_levels(["Model", "Scenario", "Region", "Variable", "Unit"]) - .reset_index() - .to_csv(path_out, index=False) - ) - - def extract_dims( qty: "AnyQuantity", dim_expr: dict, *, drop: bool = True, fillna: str = "_T" ) -> "AnyQuantity": @@ -145,37 +97,65 @@ def extract_dims1(qty: "AnyQuantity", dim: dict) -> "AnyQuantity": # pragma: no return result -def select_re(qty: "AnyQuantity", indexers: dict) -> "AnyQuantity": - """Select using regular expressions for each dimension.""" - new_indexers = dict() - for dim, expr in indexers.items(): - new_indexers[dim] = list( - map(str, filter(re.compile(expr).match, qty.coords[dim].data.astype(str))) - ) - return qty.sel(new_indexers) +def finalize( + q_all: "AnyQuantity", + q_update: "AnyQuantity", + model_name: str, + scenario_name: str, + path_out: "pathlib.Path", +) -> None: + """Finalize output. + 1. Reattach "Model" and "Scenario" labels. + 2. Reassemble the "Variable" dimension/coords of `q_update`; drop "e" and "t". + 3. Convert both `q_all` and `q_update` to :class:`pandas.Series`; update the former + with the contents of the latter. + 4. Adjust to IAMC ‘wide’ structure and write to `path_out`. -#: Expression for IAMC ‘variable’ names used in :func:`main`. -EXPR = r"^Emissions\|(?P[^\|]+)\|Energy\|Demand\|Transportation(?:\|(?P.*))?$" + Parameters + ---------- + q_all : + All data. + q_update : + Revised data to overwrite corresponding values in `q_all`. + """ + + def _expand(qty): + return qty.expand_dims( + {"Model": [model_name], "Scenario": [scenario_name]} + ).rename({"n": "Region", "UNIT": "Unit", "VARIABLE": "Variable"}) + + s_all = q_all.pipe(_expand).to_series() + + s_all.update( + q_update.pipe(_expand) + .to_frame() + .reset_index() + .assign( + Variable=lambda df: ( + "Emissions|" + df["e"] + "|Energy|Demand|Transportation|" + df["t"] + ).str.replace("|_T", "") + ) + .drop(["e", "t"], axis=1) + .set_index(s_all.index.names)[0] + ) + + ( + s_all.unstack("y") + .reorder_levels(["Model", "Scenario", "Region", "Variable", "Unit"]) + .reset_index() + .to_csv(path_out, index=False) + ) @minimum_version("genno 1.25") -def main(path_in: "pathlib.Path", path_out: "pathlib.Path"): +def main(path_in: "pathlib.Path", path_out: "pathlib.Path", method: str) -> None: """Postprocess aviation emissions for SSP 2024. 1. Read input data from `path_in`. - 2. Select data with variable names matching :data:`EXPR`. - 3. Calculate (identical) values for: - - - ``Emissions|*|Energy|Demand|Transportation|Aviation`` - - ``Emissions|*|Energy|Demand|Transportation|Aviation|International`` - - These are currently calculated as the product of :func:`aviation_share` and - ``Emissions|*|Energy|Demand|Transportation``. - 4. Subtract (3) from: - ``Emissions|*|Energy|Demand|Transportation|Road Rail and Domestic Shipping`` - 5. Recombine with all other, unmodified data. - 6. Write to `path_out`. + 2. Call either :func:`prepare_method_A` or :func:`prepare_method_B` according to + the value of `method`. + 3. Write to `path_out`. Parameters ---------- @@ -183,18 +163,15 @@ def main(path_in: "pathlib.Path", path_out: "pathlib.Path"): Input data path. path_out : Output data path. + method : + either 'A' or 'B'. """ import pandas as pd - # Shorthand - e_t = ("e", "t") - t = "t" - k_input = genno.Key("input", ("n", "y", "VARIABLE", "UNIT")) - k = genno.KeySeq("result", ("n", "y", "UNIT") + e_t) - c = genno.Computer() # Read the data from `path` + k_input = genno.Key("input", ("n", "y", "VARIABLE", "UNIT")) c.add( k_input, iamc_like_data_for_query, @@ -207,6 +184,38 @@ def main(path_in: "pathlib.Path", path_out: "pathlib.Path"): df = pd.read_csv(path_in, nrows=1) c.add("model name", genno.quote(df["Model"].iloc[0])) c.add("scenario name", genno.quote(df["Scenario"].iloc[0])) + c.add("path out", path_out) + + # Call a function to prepare the remaining calculations + prepare_func = { + "A": prepare_method_A, + "B": prepare_method_B, + }[method] + prepare_func(c, k_input) + + # Execute + c.get("target") + + +def prepare_method_A(c: "genno.Computer", k_input: "genno.Key") -> None: + """Prepare calculations using method 'A'. + + 1. Select data with variable names matching :data:`EXPR`. + 2. Calculate (identical) values for: + + - ``Emissions|*|Energy|Demand|Transportation|Aviation`` + - ``Emissions|*|Energy|Demand|Transportation|Aviation|International`` + + These are currently calculated as the product of :func:`aviation_share` and + ``Emissions|*|Energy|Demand|Transportation``. + 3. Subtract (2) from: + ``Emissions|*|Energy|Demand|Transportation|Road Rail and Domestic Shipping`` + 4. Recombine with all other, unmodified data. + """ + # Shorthand + e_t = ("e", "t") + t = "t" + k = genno.KeySeq("result", ("n", "y", "UNIT") + e_t) # Filter on "VARIABLE" c.add(k[0] / e_t, select_re, k_input, indexers={"VARIABLE": EXPR}) @@ -256,8 +265,20 @@ def main(path_in: "pathlib.Path", path_out: "pathlib.Path"): k[5], "model name", "scenario name", - path_out=path_out, + "path out", ) - # Execute - c.get("target") + +def prepare_method_B(c, k_input: "genno.Key"): + """Prepare calculations using method 'B'.""" + raise NotImplementedError + + +def select_re(qty: "AnyQuantity", indexers: dict) -> "AnyQuantity": + """Select from `qty` using regular expressions for each dimension.""" + new_indexers = dict() + for dim, expr in indexers.items(): + new_indexers[dim] = list( + map(str, filter(re.compile(expr).match, qty.coords[dim].data.astype(str))) + ) + return qty.sel(new_indexers) From a6cdf4a662e91f3cf5c6e71ec8ccf0d35486ddf0 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2024 17:56:31 +0100 Subject: [PATCH 40/45] Add "mix-models ssp transport --method=B" option --- message_ix_models/project/ssp/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/message_ix_models/project/ssp/cli.py b/message_ix_models/project/ssp/cli.py index 2e3e5ee5c..4ac89147a 100644 --- a/message_ix_models/project/ssp/cli.py +++ b/message_ix_models/project/ssp/cli.py @@ -25,6 +25,7 @@ def gen_structures(context, **kwargs): @cli.command("transport") +@click.option("--method", type=click.Choice(["A", "B"]), required=True) @click.argument("path_in", type=click.Path(exists=True, dir_okay=False, path_type=Path)) @click.argument( "path_out", @@ -32,7 +33,7 @@ def gen_structures(context, **kwargs): required=False, ) @click.pass_obj -def transport_cmd(context: "Context", path_in: Path, path_out: Optional[Path]): +def transport_cmd(context: "Context", method, path_in: Path, path_out: Optional[Path]): """Postprocess aviation emissions. Data are read from PATH_IN, in .xlsx or .csv format. If .xlsx, the data are first @@ -64,7 +65,7 @@ def transport_cmd(context: "Context", path_in: Path, path_out: Optional[Path]): else: path_out_user = path_out - main(path_in, path_out) + main(path_in, path_out, method) if path_out_user != path_out: print(f"Convert CSV output to {path_out_user}") From c2f75995d68d38cdcc82193494947152685fa1cd Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Jan 2025 10:13:29 +0100 Subject: [PATCH 41/45] Add R12/emi-intensity.csv --- .../data/transport/emi-intensity.csv | 65 +++++++++++++++++++ message_ix_models/model/transport/files.py | 13 +++- 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 message_ix_models/data/transport/emi-intensity.csv diff --git a/message_ix_models/data/transport/emi-intensity.csv b/message_ix_models/data/transport/emi-intensity.csv new file mode 100644 index 000000000..3293ceabf --- /dev/null +++ b/message_ix_models/data/transport/emi-intensity.csv @@ -0,0 +1,65 @@ +# Emissions intensity of energy use +# +# Sources +# - Group 1: e-mail text from Shaohui Zhang 2024-10-09 +# - Group 2: e-mail attachment from Shaohui Zhang 2024-10-09 +# "EMEP_EEA Emission factor.xlsx". These all have the following +# dimensions/metadata in other columns: +# - NFR: 1.A.3.a.ii.(i) +# - Sector: Civil aviation (domestic, LTO) +# → technology=AIR +# - Table: Table_3-3 +# - Type: Tier 1 Emission Factor +# - Technology: NA +# - Fuel: Jet Gasoline and Aviation Gasoline +# - Abatement: (empty) +# - Region: NA +# - Pollutant → emission +# - CI_lower, CI_upper, Reference → preserved as comments +# - Unit: kg/tonne fuel, which is equivalent to g / kg fuel +# - Group 3: e-mail from Lena Höglund-Isaksson 2024-10-06. +# The units for these are kt/petajoule gasoline. +# +# Units: g / MJ +# +technology, commodity, emission, value +# +# Group 1 +# - TODO Units are actually g / kg fuel. +# Convert to g / MJ. +# +AIR, lightoil, SO2, 1.2 +# Lower end of confidence interval +AIR, lightoil, NOx, 14.12 +# Upper end of confidence interval +# AIR, lightoil, NOx, 15.14 +# +# Group 2 +# - TODO Units are actually g / kg fuel. +# Convert to g / MJ. +# +# “Calculated using Tier 2 method” +# AIR, lightoil, NOx, 2 +# Commented because it duplicates a line above +# AIR, lightoil, NOx, 4 +# AIR, lightoil, NOx, 8 +# “Calculated using Tier 2 method” +# AIR, lightoil, CO, 600 +AIR, lightoil, CO, 1200 +# AIR, lightoil, CO, 2400 +# “Calculated using Tier 2 method” +# These data were provided with the code 'NMVOC', but we use the label 'VOC' to +# align with MESSAGEix-GLOBIOM, even though these are not strictly the same. +# AIR, lightoil, VOC, 9.5 +AIR, lightoil, VOC, 19 +# AIR, lightoil, VOC, 38 +# “Assuming 0.05% S by mass” +# AIR, lightoil, SOx, 0.5 +AIR, lightoil, SOx, 1 +# AIR, lightoil, SOx, 2 +# +# Group 3 +# - Originally supplied as kt / PJ [=] g / MJ. +# +AIR, lightoil, CH4, 0.0005 +AIR, lightoil, N2O, 0.0031 diff --git a/message_ix_models/model/transport/files.py b/message_ix_models/model/transport/files.py index a647e51ef..13805766d 100644 --- a/message_ix_models/model/transport/files.py +++ b/message_ix_models/model/transport/files.py @@ -347,6 +347,15 @@ def read_structures() -> "sdmx.message.StructureMessage": units="dimensionless", ) +# NB This differs from fuel_emi_intensity in including (a) a 't[echnology]' dimension +# and (b) more and non-GHG species. +emi_intensity = add( + key="emissions intensity:t-c-e:transport", + path="emi-intensity", + name="Emissions intensity of fuel use", + units="g / EJ", +) + energy_other = add( key="energy:c-n:transport other", path="energy-other", @@ -355,9 +364,11 @@ def read_structures() -> "sdmx.message.StructureMessage": required=False, ) +# NB This differs from emi_intensity in (a) having no 't[echnology]' dimension and (b) +# including only CO₂. fuel_emi_intensity = add( key="fuel-emi-intensity:c-e", - name="Carbon emissions intensity of fuel use", + name="GHG emissions intensity of fuel use", description="""Values are in GWP-equivalent mass of carbon, not in mass of the emissions species.""", units="tonne / kWa", From a8b78f7f97e0d146a363edf0a1be09c99ff04810 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Jan 2025 10:15:52 +0100 Subject: [PATCH 42/45] Add unit annotation to "N2O" emission code --- message_ix_models/data/emission.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/message_ix_models/data/emission.yaml b/message_ix_models/data/emission.yaml index 481da3c21..4318f130d 100644 --- a/message_ix_models/data/emission.yaml +++ b/message_ix_models/data/emission.yaml @@ -13,6 +13,7 @@ CO2: N2O: name: Nitrous oxide + units: kt NH3: name: Ammonia From 3bc12f4fdf8b82bad6cdcd973462485b4626dfbe Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Jan 2025 10:18:05 +0100 Subject: [PATCH 43/45] Reformat CL_TRANSPORT_SCENARIO --- .../IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml b/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml index 1addb8d3b..2bfbd1458 100644 --- a/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml +++ b/message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml @@ -1,9 +1,11 @@ + none false - 2024-12-03T21:18:35.564011 - Generated by message_ix_models 2024.8.7.dev455+gd9d66fa40.d20241203 + 2025-01-15T10:37:47.926181 + + Generated by message_ix_models 2025.1.11.dev1+gabce19674.d20250113 @@ -89,7 +91,6 @@ - Low Energy Demand/High-with-Low scenario with SSP1 demographics 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).1' @@ -104,9 +105,9 @@ 'ixmp://ixmp-dev/SSP_SSP1_v1.1/baseline_DEFAULT_step_13' + Low Energy Demand/High-with-Low scenario with SSP1 demographics - Low Energy Demand/High-with-Low scenario with SSP2 demographics 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' @@ -121,9 +122,9 @@ 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + Low Energy Demand/High-with-Low scenario with SSP2 demographics - EDITS scenario with ITF PASTA 'CA' activity 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' @@ -138,9 +139,9 @@ 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + EDITS scenario with ITF PASTA 'CA' activity - EDITS scenario with ITF PASTA 'HA' activity 'urn:sdmx:org.sdmx.infomodel.codelist.Code=ICONICS:SSP(2024).2' @@ -155,6 +156,7 @@ 'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13' + EDITS scenario with ITF PASTA 'HA' activity From 2fafb7e18c97acf8967662aae4161b458c126e5d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Jan 2025 10:22:53 +0100 Subject: [PATCH 44/45] Implement .ssp.transport.prepare_method_B() --- message_ix_models/project/ssp/transport.py | 160 ++++++++++++++++++--- 1 file changed, 141 insertions(+), 19 deletions(-) diff --git a/message_ix_models/project/ssp/transport.py b/message_ix_models/project/ssp/transport.py index cd092138d..ee93e5efe 100644 --- a/message_ix_models/project/ssp/transport.py +++ b/message_ix_models/project/ssp/transport.py @@ -4,8 +4,12 @@ from typing import TYPE_CHECKING, Hashable import genno +import pandas as pd import xarray as xr +from genno import KeySeq +from message_ix_models import Context +from message_ix_models.model.structure import get_codelist from message_ix_models.tools.iamc import iamc_like_data_for_query from message_ix_models.util import minimum_version @@ -115,9 +119,10 @@ def finalize( Parameters ---------- q_all : - All data. + All data. Quantity with dimensions (n, UNIT, VARIABLE). q_update : Revised data to overwrite corresponding values in `q_all`. + Quantity with dimensions (n, UNIT, VARIABLE, e, t). """ def _expand(qty): @@ -127,19 +132,22 @@ def _expand(qty): s_all = q_all.pipe(_expand).to_series() - s_all.update( + s_update = ( q_update.pipe(_expand) .to_frame() .reset_index() .assign( Variable=lambda df: ( "Emissions|" + df["e"] + "|Energy|Demand|Transportation|" + df["t"] - ).str.replace("|_T", "") + ).str.replace("|_T", ""), ) .drop(["e", "t"], axis=1) .set_index(s_all.index.names)[0] + .rename("value") ) + s_all.update(s_update) + ( s_all.unstack("y") .reorder_levels(["Model", "Scenario", "Region", "Variable", "Unit"]) @@ -176,6 +184,7 @@ def main(path_in: "pathlib.Path", path_out: "pathlib.Path", method: str) -> None k_input, iamc_like_data_for_query, path=path_in, + non_iso_3166="keep", query="Model != ''", unique="MODEL SCENARIO", ) @@ -187,17 +196,24 @@ def main(path_in: "pathlib.Path", path_out: "pathlib.Path", method: str) -> None c.add("path out", path_out) # Call a function to prepare the remaining calculations + # This returns a key like prepare_func = { "A": prepare_method_A, "B": prepare_method_B, }[method] - prepare_func(c, k_input) + + k = prepare_func(c, k_input) + + # - Collapse to IAMC "VARIABLE" dimension name + # - Recombine with other data + # - Write back to the file + c.add("target", finalize, k_input, k, "model name", "scenario name", "path out") # Execute c.get("target") -def prepare_method_A(c: "genno.Computer", k_input: "genno.Key") -> None: +def prepare_method_A(c: "genno.Computer", k_input: "genno.Key") -> "genno.Key": """Prepare calculations using method 'A'. 1. Select data with variable names matching :data:`EXPR`. @@ -253,25 +269,131 @@ def prepare_method_A(c: "genno.Computer", k_input: "genno.Key") -> None: c.add(k[4], "mul", k[3] / t, "broadcast:t:AIR emissions") # Add to the input data - c.add(k[5], "add", k[1], k[4]) + return c.add(k[5], "add", k[1], k[4]) - # - Collapse to IAMC "VARIABLE" dimension name - # - Recombine with other data - # - Write back to the file + +def prepare_method_B(c, k_input: "genno.Key") -> None: + """Prepare calculations using method 'B'.""" + from types import SimpleNamespace + + from message_ix_models.model.transport import build + from message_ix_models.model.transport import files as exo + from message_ix_models.tools.exo_data import prepare_computer + + # Fetch a Context instance + # NB It is assumed this is aligned with the contents of the input data file + context = Context.get_instance() + + # TODO Check if this is redundant, i.e. already in build.get_computer + cl_emission = get_codelist("emission") + + # Add the same structure information, notably , used in + # the build and report workflow steps for MESSAGEix-Transport + build.get_computer(context, c) + + ### Prepare data from IEA EWEB: the share of aviation in transport consumption of + ### each 'c[ommodity]' + + # Fetch data from IEA EWEB + flows = ["AVBUNK", "DOMESAIR", "TOTTRANS"] + kw = dict(provider="IEA", edition="2024", flow=flows, transform="B", regions="R12") + keys = prepare_computer(context, c, "IEA_EWEB", kw, strict=False) + + # Shorthand + k = SimpleNamespace( + iea=genno.KeySeq(keys[0]), + cnt=KeySeq("FOO:c-n-t"), + ) + k.fnp = k.iea / "y" + k.cn = k.cnt / "t" + + # Select data for 2019 only + c.add(k.fnp[0], "select", k.iea.base, indexers=dict(y=2019), drop=True) + + # Only use the aggregation on the 'product' dimension, not on 'flow' c.add( - "target", - finalize, - k_input, - k[5], - "model name", - "scenario name", - "path out", + "groups:p:iea to transport", + lambda d: {"product": d["product"]}, + "groups::iea to transport", ) + # Aggregate IEA 'product' dimension for alignment to MESSAGE 'c[ommodity]' + c.add(k.fnp[1], "aggregate", k.fnp[0], "groups:p:iea to transport", keep=False) + # Rename dimensions + c.add(k.cnt[0], "rename_dims", k.fnp[1], name_dict=dict(flow="t", product="c")) -def prepare_method_B(c, k_input: "genno.Key"): - """Prepare calculations using method 'B'.""" - raise NotImplementedError + # Reverse sign of AVBUNK + q_sign = genno.Quantity([-1.0, 1.0, 1.0], coords={"t": flows}) + c.add(k.cnt[1], "mul", k.cnt[0], q_sign) + + # Compute ratio of ('AVBUNK' + 'DOMESAIR') to 'TOTTRANS' + # TODO Confirm that this or another numerator is appropriate + c.add(k.cnt[2], "select", k.cnt[1], indexers=dict(t=["AVBUNK", "DOMESAIR"])) + c.add(k.cn[0], "sum", k.cnt[2], dimensions=["t"]) + c.add(k.cn[1], "select", k.cnt[1], indexers=dict(t="TOTTRANS"), drop=True) + c.add(k.cn[2], "div", k.cn[0], k.cn[1]) + + ### Prepare data from the input data file: total transport consumption of light oil + k.input = genno.KeySeq("input", ("n", "y", "UNIT", "e")) + + # Filter on "VARIABLE" + expr = r"^Final Energy\|Transportation\|(?PLiquids\|Oil)$" + c.add(k.input[0] / "e", select_re, k_input, indexers={"VARIABLE": expr}) + + # Extract the "e" dimensions from "VARIABLE" + c.add(k.input[1], extract_dims, k.input[0] / "e", dim_expr={"VARIABLE": expr}) + + # Convert "UNIT" dim labels to Quantity.units + c.add(k.input[2] / "UNIT", "unique_units_from_dim", k.input[1], dim="UNIT") + + # Relabel: + # - c[ommodity]: 'Liquids|Oil' (IAMC 'variable' component) to 'lightoil' + # - n[ode]: 'AFR' to 'R12_AFR' etc. + labels = dict( + c={"Liquids|Oil": "lightoil"}, + n={n.id.partition("_")[2]: n.id for n in get_codelist("node/R12")}, + ) + c.add(k.input[3] / "UNIT", "relabel", k.input[2] / "UNIT", labels=labels) + + ### Compute estimate of emissions + # Product of aviation share and FE of total transport → FE of aviation + prev = c.add("aviation fe", "mul", k.input[3] / "UNIT", k.cn[2]) + + # Convert exo.emi_intensity to Mt / GWa + c.add( + exo.emi_intensity + "conv", "convert_units", exo.emi_intensity, units="Mt / GWa" + ) + + # Product of FE of aviation and emission intensity → emissions of aviation + prev = c.add("aviation emi::0", "mul", prev, exo.emi_intensity + "conv") + + # Convert units to megatonne / year + prev = c.add("aviation emi::1", "convert_units", prev, units="Mt / year") + + # In one step + # - Expand dimensions with "UNIT" containing labels to be used. + # - Adjust values for species (N2O) that are reported in kt rather than Mt. + data = [] + for e in cl_emission: + try: + label = str(e.get_annotation(id="report").text) + except KeyError: + label = e.id + try: + unit = str(e.get_annotation(id="units").text) + except KeyError: + unit = "Mt" + data.append(["AIR", e.id, f"{unit} {label}/yr", 1.0 if unit == "Mt" else 1e3]) + + dims = "t e UNIT value".split() + q = genno.Quantity(pd.DataFrame(data, columns=dims).set_index(dims[:-1])[dims[-1]]) + prev = c.add("aviation emi::2", "mul", prev, q) + + # Change labels + # - Restore e.g. "AFR" given "R12_AFR" + labels = dict(n={v: k for k, v in labels["n"].items()}, t={"AIR": "Aviation"}) + k.result = c.add("aviation emi::3", "relabel", prev / "c", labels=labels) + return k.result def select_re(qty: "AnyQuantity", indexers: dict) -> "AnyQuantity": From d384ef5be6974743798f03e6450346574443a26f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Jan 2025 12:49:43 +0100 Subject: [PATCH 45/45] Simplify and document .build.add_structure() - Separate STRUCTURE_STATIC and use Computer.add_queue() for clarity. - Add docstrings listing all added keys/tasks. - Move function to .operator.write_report_debug(). --- doc/conf.py | 2 + message_ix_models/model/transport/build.py | 305 ++++++++++++------ message_ix_models/model/transport/operator.py | 27 ++ 3 files changed, 228 insertions(+), 106 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 3acf23705..b5dc7371e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -74,7 +74,9 @@ .. role:: strike .. role:: underline +.. |c| replace:: :math:`c` .. |n| replace:: :math:`n` +.. |t| replace:: :math:`t` .. |y| replace:: :math:`y` .. |y0| replace:: :math:`y_0` diff --git a/message_ix_models/model/transport/build.py b/message_ix_models/model/transport/build.py index 3610d140d..b28b54b51 100644 --- a/message_ix_models/model/transport/build.py +++ b/message_ix_models/model/transport/build.py @@ -1,16 +1,20 @@ """Build MESSAGEix-Transport on a base model.""" import logging +from functools import partial from importlib import import_module +from operator import itemgetter from pathlib import Path from typing import TYPE_CHECKING, Any, Optional +import genno import pandas as pd -from genno import Computer, KeyExistsError, Quantity, quote +from genno import Computer, KeyExistsError, quote from message_ix import Scenario from message_ix_models import Context, ScenarioInfo from message_ix_models.model import bare, build +from message_ix_models.model.structure import get_codelist from message_ix_models.util import minimum_version from message_ix_models.util._logging import mark_time from message_ix_models.util.graphviz import HAS_GRAPHVIZ @@ -21,34 +25,10 @@ if TYPE_CHECKING: import pathlib - from genno.types import AnyQuantity log = logging.getLogger(__name__) -def write_report(qty: "AnyQuantity", path: Path, kwargs=None) -> None: - """Similar to :func:`.genno.operator.write_report`, but include units. - - .. todo:: Move upstream, to :mod:`genno`. - """ - from genno import operator - - from message_ix_models.util import datetime_now_with_tz - - kwargs = kwargs or dict() - kwargs.setdefault( - "header_comment", - f"""`{qty.name}` data from MESSAGEix-Transport calibration. - -Generated: {datetime_now_with_tz().isoformat()} - -Units: {qty.units:~} -""", - ) - - operator.write_report(qty, path, kwargs) - - def add_debug(c: Computer) -> None: """Add tasks for debugging the build.""" from genno import Key, KeySeq @@ -95,7 +75,12 @@ def add_debug(c: Computer) -> None: ) ): debug_keys.append(f"transport debug {i}") - c.add(debug_keys[-1], write_report, key, output_dir.joinpath(f"{stem}.csv")) + c.add( + debug_keys[-1], + "write_report_debug", + key, + output_dir.joinpath(f"{stem}.csv"), + ) def _(*args) -> "pathlib.Path": """Do nothing with the computed `args`, but return `output_path`.""" @@ -235,7 +220,7 @@ def add_exogenous_data(c: Computer, info: ScenarioInfo) -> None: log.info(repr(e)) # Solved scenario that already has this key # Ensure correct units - c.add("population:n-y", "mul", "pop:n-y", Quantity(1.0, units="passenger")) + c.add("population:n-y", "mul", "pop:n-y", genno.Quantity(1.0, units="passenger")) # Dummy prices try: @@ -265,25 +250,123 @@ def add_exogenous_data(c: Computer, info: ScenarioInfo) -> None: c.add("", f, context=context) -def add_structure(c: Computer): - """Add keys to `c` for structures required by :mod:`.transport.build` computations. - - This uses :attr:`.transport.Config.base_model_info` and - :attr:`.transport.Config.spec` to mock the contents that would be reported from an - already-populated Scenario for sets "node", "year", and "cat_year". It also adds - many other keys. +#: :mod:`genno` tasks for model structure information that are 'static'—that is, do not +#: change based on :class:`~.transport.config.Config` settings. See +#: :func:`add_structure`. +#: +#: These include: +#: +#: - ``info``: :attr:`transport.Config.base_model_info +#: `, an instance of :class:`.ScenarioInfo`. +#: - ``transport info``: the logical union of +#: :attr:`~.transport.config.Config.base_model_info` and the :attr:`.Spec.add` member +#: of :attr:`Config.spec <.transport.config.Config.spec>`. This includes +#: all set elements that will be present in the build model. +#: - ``dry_run``: :attr:`.Config.dry_run`. +#: - ``e::codelist``: :func:`.get_codelist` for :ref:`emission-yaml`. +#: - ``groups::iea to transport``, ``groups::transport to iea``, ``indexers::iea to +#: transport``: the 3 outputs of :func:`.groups_iea_eweb`, for use with IEA Extended +#: World Energy Balances data. +#: - ``n::ex world``: |n| as :class:`list` of :class:`str`, excluding "World". See +#: :func:`.nodes_ex_world`. +#: - ``n::ex world+code``: |n| as :class:`list`` of :class:`.Code`, excluding "World". +#: - ``n:n:ex world``: a 1-dimensional :class:`.Quantity` for broadcasting (values all +#: 1). +#: - ``nl::world agg``: :class:`dict` mapping to aggregate "World" from individual |n|. +#: See :func:`.nodes_world_agg`. +STRUCTURE_STATIC = ( + ("info", lambda c: c.transport.base_model_info, "context"), + ( + "transport info", + lambda c: c.transport.base_model_info | c.transport.spec.add, + "context", + ), + ("dry_run", lambda c: c.core.dry_run, "context"), + ("e::codelist", partial(get_codelist, "emission")), + ("groups::iea eweb", "groups_iea_eweb", "t::transport"), + ("groups::iea to transport", itemgetter(0), "groups::iea eweb"), + ("groups::transport to iea", itemgetter(1), "groups::iea eweb"), + ("indexers::iea to transport", itemgetter(2), "groups::iea eweb"), + ("n::ex world", "nodes_ex_world", "n"), + ( + "n:n:ex world", + lambda n: genno.Quantity([1.0] * len(n), coords={"n": n}), + "n::ex world", + ), + ("n::ex world+code", "nodes_ex_world", "nodes"), + ("nl::world agg", "nodes_world_agg", "config"), +) + + +def add_structure(c: Computer) -> None: + """Add tasks to `c` for structures required by :mod:`.transport.build`. + + These include: + + - The following keys *only* if not already present in `c`. If, for example, `c` is + a :class:`.Reporter` prepared from an already-solved :class:`.Scenario`, the + existing tasks referring to the Scenario contents are not changed. + + - ``n``: |n| as :class:`list` of :class:`str`. + - ``y``: |y| in the base model. + - ``cat_year``: simulated data structure for "cat_year" with at least 1 row + :py:`("firstmodelyear", y0)`. + - ``y::model``: |y| within the model horizon as :class:`list` of :class:`int`. + - ``y0``: The first model period, :class:`int`. + + - All tasks from :data:`STRUCTURE_STATIC`. + - ``c::transport``: the |c| set of the :attr:`~.Spec.add` member of + :attr:`Config.spec <.transport.config.Config.spec>`, transport commodities to be + added. + - ``c::transport+base``: all |c| that will be present in the build model + - ``cg``: "consumer group" set elements. + - ``indexers:cg``: ``cg`` as indexers. + - ``nodes``: |n| in the base model. + - ``indexers:scenario``: :class:`dict` mapping "scenario" to the short form of + :attr:`Config.ssp <.transport.config.Config.ssp>` (for instance, "SSP1"), for + indexing. + - ``t::transport``: all transport |t| to be added, :class:`list`. + - ``t::transport agg``: :class:`dict` mapping "t" to the output of + :func:`.get_technology_groups`. For use with operators like 'aggregate', 'select', + etc. + - ``t::transport all``: :class:`dict` mapping "t" to ``t::transport``. + .. todo:: Choose a more informative key. + - ``t::transport modes``: :attr:`Config.demand_modes + <.transport.config.Config.demand_modes>`. + - ``t::transport modes 0``: :class:`dict` mapping "t" to the keys only from + ``t::transport agg``. Use with 'aggregate' to produce the sum across modes, + including "non-LDV". + - ``t::transport modes 1``: same as ``t::transport modes 0`` except excluding + "non-ldv". + - ``t::RAIL`` etc.: transport |t| in the "RAIL" mode/group as :class:`list` of + :class:`str`. See :func:`.get_technology_groups`. + - ``t::transport RAIL`` etc.: :class:`dict` mapping "t" to the elements of + ``t::RAIL``. + - ``broadcast:t-c-l:input``: Quantity for broadcasting (all values 1) from every + transport |t| (same as ``t::transport``) to the :math:`(c, l)` that that + technology receives as input. See :func:`.broadcast_t_c_l`. + - ``broadcast:t-c-l:input``: same as above, but for the :math:`(c, l)` that the + technology produces as output. + - ``broadcast:y-yv-ya:all``: Quantity for broadcasting (all values 1) from every |y| + to every possible combination of :math:`(y^V=y, y^A)`—including historical + periods. See :func:`.broadcast_y_yv_ya`. + - ``broadcast:y-yv-ya``: same as above, but only model periods (``y::model``). + - ``broadcast:y-yv-ya:no vintage``: same as above, but only the cases where + :math:`y^V = y^A`. """ - from operator import itemgetter - from ixmp.report import configure - config: "Config" = c.graph["context"].transport - info = config.base_model_info # Information about the base scenario - spec = config.spec # Specification for MESSAGEix-Transport structure + from .operator import broadcast_t_c_l, broadcast_y_yv_ya + + # Retrieve configuration and other information + config: "Config" = c.graph["context"].transport # .model.transport.Config object + info = config.base_model_info # ScenarioInfo describing the base scenario + spec = config.spec # Specification for MESSAGEix-Transport structure to be built + t_groups = get_technology_groups(spec) # Technology groups/hierarchy # Update RENAME_DIMS with transport-specific concepts/dimensions. This allows to use # genno.operator.load_file(…, dims=RENAME_DIMS) in add_exogenous_data() - # TODO move to a more appropriate location + # TODO Read from a concept scheme or list of dimensions configure( rename_dims={ "area_type": "area_type", @@ -295,67 +378,56 @@ def add_structure(c: Computer): } ) - for key, *comp in ( - # Configuration - ("info", lambda c: c.transport.base_model_info, "context"), + # Tasks only to be added if not already present in `c`. These must be done + # separately because add_queue does not support the strict/pass combination. + for task in ( + ("n", quote(list(map(str, info.set["node"])))), + ("y", quote(info.set["year"])), ( - "transport info", - lambda c: c.transport.base_model_info | c.transport.spec.add, - "context", + "cat_year", + pd.DataFrame([["firstmodelyear", info.y0]], columns=["type_year", "year"]), ), - ("dry_run", lambda c: c.core.dry_run, "context"), - # Structure + ("y::model", "model_periods", "y", "cat_year"), + ("y0", itemgetter(0), "y::model"), + ): + try: + c.add(*task, strict=True) + except KeyExistsError: # Already present + # log.debug(f"Use existing {c.describe(task[0])}") + pass + + # Assemble a queue of tasks + # - `Static` tasks + # - Single 'dynamic' tasks based on config, info, spec, and/or t_groups + # - Multiple static and dynamic tasks generated in loops etc. + tasks = list(STRUCTURE_STATIC) + [ ("c::transport", quote(spec.add.set["commodity"])), ("c::transport+base", quote(spec.add.set["commodity"] + info.set["commodity"])), ("cg", quote(spec.add.set["consumer_group"])), ("indexers:cg", spec.add.set["consumer_group indexers"]), - ("n", quote(list(map(str, info.set["node"])))), ("nodes", quote(info.set["node"])), ("indexers:scenario", quote(dict(scenario=repr(config.ssp).split(":")[1]))), ("t::transport", quote(spec.add.set["technology"])), - # Dictionary form for aggregation - # TODO Choose a more informative key + ("t::transport agg", quote(dict(t=t_groups))), ("t::transport all", quote(dict(t=spec.add.set["technology"]))), ("t::transport modes", quote(config.demand_modes)), - ("y", quote(info.set["year"])), + ("t::transport modes 0", quote(dict(t=list(t_groups.keys())))), ( - "cat_year", - pd.DataFrame([["firstmodelyear", info.y0]], columns=["type_year", "year"]), + "t::transport modes 1", + quote(dict(t=list(filter(lambda k: k != "non-ldv", t_groups.keys())))), ), - ): - try: - c.add(key, *comp, strict=True) # Raise an exception if `key` exists - except KeyExistsError: - continue # Already present; don't overwrite + ] - # Create quantities for broadcasting (t,) to (t, c, l) dimensions - for kind in "input", "output": - c.add( + # Quantities for broadcasting (t,) to (t, c, l) dimensions + tasks += [ + ( f"broadcast:t-c-l:transport+{kind}", - "broadcast_t_c_l", + partial(broadcast_t_c_l, kind=kind, default_level="final"), "t::transport", "c::transport+base", - kind=kind, - default_level="final", ) - - # Retrieve information about the model structure - t_groups = get_technology_groups(spec) - - # List of nodes excluding "World" - # TODO move upstream, to message_ix - c.add("n::ex world", "nodes_ex_world", "n") - c.add( - "n:n:ex world", - lambda n: Quantity([1.0] * len(n), coords={"n": n}), - "n::ex world", - ) - c.add("n::ex world+code", "nodes_ex_world", "nodes") - c.add("nl::world agg", "nodes_world_agg", "config") - - # Model periods only - c.add("y::model", "model_periods", "y", "cat_year") - c.add("y0", itemgetter(0), "y::model") + for kind in ("input", "output") + ] # Quantities for broadcasting y to (yv, ya) for base, tag, method in ( @@ -363,32 +435,28 @@ def add_structure(c: Computer): ("y::model", "", "product"), # Model periods only ("y::model", ":no vintage", "zip"), # Model periods with no vintaging ): - c.add(f"broadcast:y-yv-ya{tag}", "broadcast_y_yv_ya", base, base, method=method) - - # Mappings for use with aggregate, select, etc. - c.add("t::transport agg", quote(dict(t=t_groups))) - # Sum across modes, including "non-ldv" - c.add("t::transport modes 0", quote(dict(t=list(t_groups.keys())))) - # Sum across modes, excluding "non-ldv" - c.add( - "t::transport modes 1", - quote(dict(t=list(filter(lambda k: k != "non-ldv", t_groups.keys())))), - ) + tasks.append( + ( + f"broadcast:y-yv-ya{tag}", + partial(broadcast_y_yv_ya, method=method), + base, + base, + ) + ) # Groups of technologies and indexers + # FIXME Combine or disambiguate these keys for id, techs in t_groups.items(): - # FIXME Combine or disambiguate these keys - # Indexer-form of technology groups - c.add(f"t::transport {id}", quote(dict(t=techs))) - # List form of technology groups - c.add(f"t::{id}", quote(techs)) + tasks += [ + # Indexer-form of technology groups + (f"t::transport {id}", quote(dict(t=techs))), + # List form of technology groups + (f"t::{id}", quote(techs)), + ] - # Mappings for use with IEA Extended World Energy Balances data - c.add("groups::iea eweb", "groups_iea_eweb", "t::transport") - # Unpack - c.add("groups::iea to transport", itemgetter(0), "groups::iea eweb") - c.add("groups::transport to iea", itemgetter(1), "groups::iea eweb") - c.add("indexers::iea to transport", itemgetter(2), "groups::iea eweb") + # - Change each task from single-tuple form to (args, kwargs) with strict=True. + # - Add all to the Computer, making 2 passes. + c.add_queue(map(lambda t: (t, dict(strict=True)), tasks), max_tries=2, fail="raise") @minimum_version("message_ix 3.8") @@ -399,7 +467,29 @@ def get_computer( visualize: bool = True, **kwargs, ) -> Computer: - """Return a :class:`genno.Computer` set up for model-building calculations.""" + """Return a :class:`genno.Computer` set up for model-building calculations. + + The returned computer contains: + + - Everything added by :func:`.add_structure`, :func:`.add_exogenous_data`, and + :func:`.add_debug`. + - For each module in :attr:`.transport.config.Config.modules`, everything added by + the :py:`prepare_computer()` function in that module. + - ``context``: a reference to `context`. + - ``scenario``: a reference to a Scenario, if one appears in `kwargs`. + - ``add transport data``: a list of keys which, when computed, will cause all + transport data to be computed and added to ``scenario``. + + Parameters + ---------- + obj : + If `obj` is an existing :class:`.Computer` (or subclass, such as + :class`.Reporter`), tasks are added the existing tasks in its graph. Otherwise, a + new Computer is created and populated. + visualize : + If :any:`True` (the default), a file :file:`transport/build.svg` is written in + the local data directory with a visualization of the ``add transport data`` key. + """ from . import operator # Configure @@ -408,11 +498,14 @@ def get_computer( # Structure information for the base model scenario = kwargs.get("scenario") if scenario: + # Retrieve structure information from an existing base model/`scenario` config.base_model_info = ScenarioInfo(scenario) config.with_scenario = True config.with_solution = scenario.has_solution() else: + # Generate a Spec/ScenarioInfo for a non-existent base model/`scenario` as + # described by `context` base_spec = bare.get_spec(context) config.base_model_info = base_spec["add"] diff --git a/message_ix_models/model/transport/operator.py b/message_ix_models/model/transport/operator.py index 9f036b1f1..7b1eeb4f9 100644 --- a/message_ix_models/model/transport/operator.py +++ b/message_ix_models/model/transport/operator.py @@ -33,6 +33,8 @@ from .config import Config if TYPE_CHECKING: + from pathlib import Path + from genno.types import AnyQuantity from message_ix import Scenario from xarray.core.types import Dims @@ -1204,3 +1206,28 @@ def votm(gdp_ppp_cap: "AnyQuantity") -> "AnyQuantity": ) assert_units(result, "") return result + + +def write_report_debug(qty: "AnyQuantity", path: "Path", kwargs=None) -> None: + """Similar to :func:`.genno.operator.write_report`, but include units. + + This version is used only in :func:`.add_debug`. + + .. todo:: Move upstream, to :mod:`genno`. + """ + from genno import operator + + from message_ix_models.util import datetime_now_with_tz + + kwargs = kwargs or dict() + kwargs.setdefault( + "header_comment", + f"""`{qty.name}` data from MESSAGEix-Transport calibration. + +Generated: {datetime_now_with_tz().isoformat()} + +Units: {qty.units:~} +""", + ) + + operator.write_report(qty, path, kwargs)