diff --git a/.gitignore b/.gitignore index 6014294..9913866 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ dnu/ dist/ *.egg-info *.log -.env \ No newline at end of file +*.env +*.ruff_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..dbde08f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,84 @@ +# The repos are ordered in the way they are executed. +# This is important because `autoflake` should run before `flake8`, for example + +default_stages: [commit, push] +fail_fast: true +minimum_pre_commit_version: 2.15.0 +repos: + + - repo: local + hooks: + + # checks for files that would conflict in case-insensitive filesystems. + - id: check-case-conflict + name: check-case-conflict + entry: check-case-conflict + language: python + types: [ python ] + + # checks for files that contain merge conflict strings. + - id: check-merge-conflict + name: check-merge-conflict + entry: check-merge-conflict + language: python + types: [ python ] + + # ensures that a file is either empty, or ends with one newline. + - id: end-of-file-fixer + name: end-of-file-fixer + entry: end-of-file-fixer + language: python + types: [ python ] + + # removes utf-8 byte order marker. + - id: fix-byte-order-marker + name: fix-byte-order-marker + entry: fix-byte-order-marker + language: python + types: [ python ] + + # replaces or checks mixed line ending. + - id: mixed-line-ending + name: mixed-line-ending + entry: mixed-line-ending + language: python + types: [ python ] + + # trims trailing whitespace. + - id: trailing-whitespace-fixer + name: trailing-whitespace-fixer + entry: trailing-whitespace-fixer + language: python + types: [ python ] + + # + - id: black + name: black + entry: black + files: "^(src|tests)" + language: python + types: [ python ] + + - id: ruff + name: ruff + entry: ruff + files: "^(src|tests)" + language: python + types: [ python ] + args: [--fix, --exit-non-zero-on-fix] + + # + # - id: pylint + # name: pylint + # entry: pylint + # files: "^(src|tests)" + # language: python + # types: [ python ] + + # + - id: mypy + name: mypy + entry: mypy + files: "^(src/pyetm/sessions|src/pyetm/client|src/pyetm/profiles)" + language: python + types: [ python ] diff --git a/examples/introduction.ipynb b/examples/introduction.ipynb index 71effae..43118fd 100644 --- a/examples/introduction.ipynb +++ b/examples/introduction.ipynb @@ -17,7 +17,7 @@ "from pyetm import Client\n", "\n", "# create a new scenario from scratch\n", - "client = Client.from_scenario_parameters(end_year=2050, area_code=\"nl\")\n", + "client = Client.from_scenario_parameters(end_year=2050, area_code=\"nl2019\")\n", "\n", "# print scenario_id\n", "scenario_id = client.scenario_id\n", @@ -69,7 +69,7 @@ "outputs": [], "source": [ "# frst check which parameters can be set in the scenario\n", - "parameters = client.user_parameters\n", + "parameters = client.input_parameters\n", "parameters.iloc[41:46]" ] }, @@ -80,7 +80,7 @@ "outputs": [], "source": [ "# show parameters that are set by the user\n", - "client.user_values" + "client.get_input_parameters(user_only=True)" ] }, { @@ -98,8 +98,8 @@ "}\n", "\n", "# apply the changes to the scenario\n", - "client.user_values = user_values\n", - "client.user_values" + "client.input_parameters = user_values\n", + "client.get_input_parameters(user_only=True)" ] }, { @@ -138,15 +138,6 @@ "outputs": [], "source": [ "# show if there are custom curves attached in the scenario\n", - "client.get_custom_curve_keys()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ "client.get_custom_curve_settings()" ] }, @@ -189,7 +180,7 @@ "outputs": [], "source": [ "# set data as ccurves profiles\n", - "client.custom_curves = ccurves\n", + "# client.set_custom_curves(ccurves\n", "client.custom_curves.head()" ] }, diff --git a/pyproject.toml b/pyproject.toml index e363f2e..8ab06b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,13 +4,17 @@ build-backend = "setuptools.build_meta" [project] name = "pyetm" -version = "1.2.2" +version = "1.3.0" + description = "Python-ETM Connector" authors = [{name = "Rob Calon", email = "robcalon@protonmail.com"}] readme = "README.md" requires-python = ">=3.9" license = {file = "LICENSE"} -dependencies = ['requests>=2.26', 'pandas>=2.0'] +dependencies = [ + 'requests>=2.26', + 'pandas>=2.0', +] keywords = ["ETM", "Energy Transition Model"] classifiers = [ 'Development Status :: 4 - Beta', @@ -28,14 +32,33 @@ classifiers = [ 'Programming Language :: Python :: 3.11', ] -[project.optional-dependencies] -async = ["aiohttp>=3.8"] -io = ["xlsxwriter>=3.0", "openpyxl>=3.0"] -test = ["pytest", "responses", "aioresponses", "build", "twine"] -all = ["pyetm[async]", "pyetm[io]", "pyetm[test]"] - [project.urls] repository = "https://github.com/robcalon/pyetm" # [tool.distutils.bdist_wheel] # universal = true + +[project.optional-dependencies] +async = ["aiohttp>=3.8"] +excel = ["xlsxwriter>=3.0", "openpyxl>=3.0"] +dev = [ + "black", + "mypy>=1.4.1", + "pre-commit", + "pre-commit-hooks", + "pyetm[async, excel]", + # "pylint", + "ruff", + "pandas-stubs", +] + +[tool.setuptools.package-data] +"pyetm.data" = ["*.csv"] + +[tool.pylint] +max-args = 15 +max-local = 20 + +[tool.mypy] +disallow_untypes_defs = true +# disallow_incomplete_defs = true diff --git a/src/pyetm/__init__.py b/src/pyetm/__init__.py index 7dcc064..853aee9 100644 --- a/src/pyetm/__init__.py +++ b/src/pyetm/__init__.py @@ -1,2 +1,4 @@ """init main module""" from .client import Client + +__all__ = ["Client"] diff --git a/src/pyetm/client/__init__.py b/src/pyetm/client/__init__.py index 5ecbfa4..e838562 100644 --- a/src/pyetm/client/__init__.py +++ b/src/pyetm/client/__init__.py @@ -1,2 +1,4 @@ """init client module""" from .client import Client + +__all__ = ["Client"] diff --git a/src/pyetm/client/account.py b/src/pyetm/client/account.py index 623123f..76defcc 100644 --- a/src/pyetm/client/account.py +++ b/src/pyetm/client/account.py @@ -1,11 +1,13 @@ - +"""account""" from __future__ import annotations -from collections.abc import Iterable import math +from collections.abc import Iterable + import pandas as pd from pyetm.logger import get_modulelogger + from .session import SessionMethods logger = get_modulelogger(__name__) @@ -16,17 +18,17 @@ class AccountMethods(SessionMethods): @property def my_scenarios(self) -> pd.DataFrame: - """ all scenarios connected to account""" + """all scenarios connected to account""" # validate token permission self._validate_token_permission("scenarios:read") # set url - url = 'scenarios' + url = self.make_endpoint_url(endpoint="scenarios") # determine number of pages pages = self._get_objects(url=url, page=1, limit=1) - pages = math.ceil(pages['meta']['total'] / 100) + pages = math.ceil(pages["meta"]["total"] / 100) if pages == 0: return pd.DataFrame() @@ -34,15 +36,13 @@ def my_scenarios(self) -> pd.DataFrame: # newlist scenarios = [] for page in range(1, pages + 1): - # fetch pages and format scenarios - recs = self._get_objects(url, page=page, limit=100)['data'] + recs = self._get_objects(url, page=page, limit=100)["data"] - excl = ['user_values', 'balanced_values', 'metadata', 'url'] - scenarios.extend([ - self._format_object(scen, excl) for scen in recs]) + excl = ["user_values", "balanced_values", "metadata", "url"] + scenarios.extend([self._format_object(scen, excl) for scen in recs]) - return pd.DataFrame.from_records(scenarios, index='id') + return pd.DataFrame.from_records(scenarios, index="id") @property def my_saved_scenarios(self) -> pd.DataFrame: @@ -52,11 +52,11 @@ def my_saved_scenarios(self) -> pd.DataFrame: self._validate_token_permission("scenarios:read") # set url - url = 'saved_scenarios' + url = self.make_endpoint_url(endpoint="saved_scenarios") # determine number of pages pages = self._get_objects(url, page=1, limit=1) - pages = math.ceil(pages['meta']['total'] / 100) + pages = math.ceil(pages["meta"]["total"] / 100) if pages == 0: return pd.DataFrame() @@ -64,15 +64,13 @@ def my_saved_scenarios(self) -> pd.DataFrame: # newlist scenarios = [] for page in range(1, pages + 1): - # fetch pages and format scenarios - recs = self._get_objects(url, page=page, limit=100)['data'] + recs = self._get_objects(url, page=page, limit=100)["data"] - excl = ['scenario', 'scenario_id', 'scenario_id_history'] - scenarios.extend([ - self._format_object(scen, excl) for scen in recs]) + excl = ["scenario", "scenario_id", "scenario_id_history"] + scenarios.extend([self._format_object(scen, excl) for scen in recs]) - return pd.DataFrame.from_records(scenarios, index='id') + return pd.DataFrame.from_records(scenarios, index="id") @property def my_transition_paths(self) -> pd.DataFrame: @@ -82,11 +80,11 @@ def my_transition_paths(self) -> pd.DataFrame: self._validate_token_permission("scenarios:read") # set url - url = 'transition_paths' + url = self.make_endpoint_url(endpoint="transition_paths") # determine number of pages pages = self._get_objects(url, page=1, limit=1) - pages = math.ceil(pages['meta']['total'] / 100) + pages = math.ceil(pages["meta"]["total"] / 100) if pages == 0: return pd.DataFrame() @@ -94,14 +92,11 @@ def my_transition_paths(self) -> pd.DataFrame: # newlist paths = [] for page in range(1, pages + 1): - # fetch pages and format scenarios - recs = self._get_objects(url, page=page, limit=100)['data'] - - paths.extend([ - self._format_object(path) for path in recs]) + recs = self._get_objects(url, page=page, limit=100)["data"] + paths.extend([self._format_object(path) for path in recs]) - return pd.DataFrame.from_records(paths, index='id') + return pd.DataFrame.from_records(paths, index="id") def _format_object(self, obj: dict, exclude: Iterable | None = None): """helper function to reformat a object.""" @@ -115,43 +110,56 @@ def _format_object(self, obj: dict, exclude: Iterable | None = None): exclude = [exclude] # flatten passed keys - for key in ['owner']: + for key in ["owner"]: if key in obj: - # flatten items in dict item = obj.pop(key) - item = {f'{key}_{k}': v for k, v in item.items()} + item = {f"{key}_{k}": v for k, v in item.items()} # add back to scenario obj = {**obj, **item} # process datetimes - for key in ['created_at', 'updated_at']: + for key in ["created_at", "updated_at"]: if key in obj: if obj.get(key) is not None: obj[key] = pd.to_datetime(obj[key], utc=True) - for key in ['template']: + for key in ["template"]: if key in obj: if obj.get(key) is None: obj[key] = pd.NA # reduce items in scenario - return {k:v for k,v in obj.items() if k not in exclude} + return {k: v for k, v in obj.items() if k not in exclude} - def _get_objects(self, url: str, page: int = 1, limit: int = 25) -> dict: + def _get_objects(self, url: str, page: int = 1, limit: int = 25): """Get info about object in url that are connected - to the user token. Object can be scenarios, saved scenarios - or transition paths.""" + to the user token. Object can be scenarios, saved scenarios + or transition paths.""" # raise without required permission - self._validate_token_permission(scope='scenarios:read') + self._validate_token_permission(scope="scenarios:read") # format request - params = {'page': int(page), 'limit': int(limit)} - headers = {'content-type': 'application/json'} + params = {"page": int(page), "limit": int(limit)} + headers = {"content-type": "application/json"} # request response - resp = self.session.get( - url, params=params, decoder='json', headers=headers) + objects = self.session.get( + url, params=params, content_type="application/json", headers=headers + ) + + return objects + + def _get_saved_scenario_id(self, saved_scenario_id: int) -> int: + """get latest scenario id for saved scenario""" + + # make url + url = self.make_endpoint_url( + endpoint="saved_scenarios", extra=str(saved_scenario_id) + ) + + # get most recent scenario id + scenario = self.session.get(url, content_type="application/json") - return resp + return int(scenario["scenario_id"]) diff --git a/src/pyetm/client/client.py b/src/pyetm/client/client.py index 8ce3ae8..ca3feec 100644 --- a/src/pyetm/client/client.py +++ b/src/pyetm/client/client.py @@ -1,10 +1,13 @@ """client object""" from __future__ import annotations +from typing import Any, Iterable import pandas as pd from pyetm.logger import get_modulelogger -from pyetm.sessions import RequestsSession, AIOHTTPSession +from pyetm.sessions import AIOHTTPSession, RequestsSession +from pyetm.types import InterpolateOptions +from pyetm.utils.interpolation import interpolate from .account import AccountMethods from .curves import CurveMethods @@ -37,13 +40,13 @@ def from_scenario_parameters( area_code: str, end_year: int, metadata: dict | None = None, - keep_compatible: bool = False, + keep_compatible: bool | None = None, private: bool | None = None, - uvalues: dict | pd.Series | None = None, + input_parameters: pd.Series[Any] | None = None, forecast_storage_order: list[str] | None = None, heat_network_order: list[str] | None = None, ccurves: pd.DataFrame | None = None, - **kwargs + **kwargs, ): """Create a new scenario from parameters. @@ -61,8 +64,9 @@ def from_scenario_parameters( in original scenario. private : bool, default None Make the scenario private. - uvalues : dict, default None - The user value configuration in the scenario. + input_parameters : dict, default None + The user specified input parameter value + configuration of the scenario. forecast_storage_order : list[str], default None The forecast storage order in the scenario. heat_network_order : list[str], default None @@ -77,14 +81,20 @@ def from_scenario_parameters( ------ client : Client Returns initialized client object.""" + # initialize new scenario client = cls(**kwargs) - client.create_new_scenario(area_code, end_year, metadata=metadata, - keep_compatible=keep_compatible, private=private) + client.create_new_scenario( + area_code=area_code, + end_year=end_year, + metadata=metadata, + keep_compatible=keep_compatible, + private=private, + ) # set user values - if uvalues is not None: - client.user_values = uvalues + if input_parameters is not None: + client.input_parameters = input_parameters # set ccurves if ccurves is not None: @@ -103,11 +113,11 @@ def from_scenario_parameters( @classmethod def from_existing_scenario( cls, - scenario_id: str | None = None, + scenario_id: int | None = None, metadata: dict | None = None, keep_compatible: bool | None = None, private: bool | None = None, - **kwargs + **kwargs, ): """create a new scenario as a copy of an existing scenario. @@ -138,7 +148,8 @@ def from_existing_scenario( # copy scenario id client.copy_scenario( - scenario_id, metadata, keep_compatible, private, connect=True) + scenario_id, metadata, keep_compatible, private, connect=True + ) return client @@ -149,13 +160,13 @@ def from_saved_scenario_id( metadata: dict | None = None, keep_compatible: bool | None = None, private: bool | None = None, - **kwargs + **kwargs, ): """initialize client from saved scenario id Parameters ---------- - saved_scenario_id : int or str, default None + saved_scenario_id : int, default None The saved scenario id to which is connected. metadata : dict, default None metadata passed to scenario. @@ -176,26 +187,106 @@ def from_saved_scenario_id( # initialize client client = cls(**kwargs) + scenario_id = client._get_saved_scenario_id(saved_scenario_id) - # make request - url = f"saved_scenarios/{saved_scenario_id}" - headers = {'content-type': 'application/json'} + return cls.from_existing_scenario( + scenario_id, metadata, keep_compatible, private, **kwargs + ) - # connect to saved scenario and - scenario_id = client.session.get( - url, decoder='json', headers=headers)['scenario_id'] + @classmethod + def from_interpolation( + cls, + end_year: int, + scenario_ids: Iterable[int], + method: InterpolateOptions = "linear", + saved_scenario_ids: bool = False, + metadata: dict | None = None, + keep_compatible: bool | None = None, + private: bool | None = None, + forecast_storage_order: list[str] | None = None, + heat_network_order: list[str] | None = None, + ccurves: pd.DataFrame | None = None, + **kwargs, + ): + """Initialize from interpolation of existing scenarios. Note that + the custom orders are always returned to the default values. - return cls.from_existing_scenario( - scenario_id, metadata, keep_compatible, private, **kwargs) + Parameters + ---------- + year: int + The end year for which the interpolation is made. + scenario_ids: int or iterable of int. + The scenario ids that are used for interpolation. + method : string, default 'linear' + Method for filling continious user values + for the passed target year(s). + saved_scenario_ids : bool, default False + Passed scenario ids are saved scenario ids. + metadata : dict, default None + metadata passed to scenario. + keep_compatible : bool, default None + Keep scenario compatible with future + versions of ETM. Defaults to settings + in original scenario. + private : bool, default None + Make the scenario private. + forecast_storage_order : list[str], default None + The forecast storage order in the scenario. + heat_network_order : list[str], default None + The heat network order in the scenario. + ccurves : pd.DataFrame, default None + Custom curves to use in sceneario. + + **kwargs are passed to the default initialization + procedure of the client. + + Return + ------ + client : Client + Returns initialized client object.""" + + # handle scenario ids: + if saved_scenario_ids: + # Only perform read operations on these sids + # as saved scenario's history would otherwise be modified. + client = Client(**kwargs) + scenario_ids = [client._get_saved_scenario_id(sid) for sid in scenario_ids] + + clients = [Client(sid, **kwargs) for sid in scenario_ids] + + # initialize scenario ids and sort by end year + clients = [Client(sid, **kwargs) for sid in scenario_ids] + clients = sorted(clients, key=lambda cln: cln.end_year) + + # get interpolated input parameters + interpolated = interpolate(target=end_year, clients=clients, method=method) + input_parameters = interpolated[end_year] + + # get area code + client = clients[-1] + area_code = client.area_code + + return Client.from_scenario_parameters( + area_code=area_code, + end_year=end_year, + metadata=metadata, + private=private, + keep_compatible=keep_compatible, + input_parameters=input_parameters, + forecast_storage_order=forecast_storage_order, + heat_network_order=heat_network_order, + ccurves=ccurves, + **kwargs, + ) def __init__( self, - scenario_id: str | None = None, + scenario_id: int | None = None, engine_url: str | None = None, etm_url: str | None = None, token: str | None = None, session: RequestsSession | AIOHTTPSession | None = None, - **kwargs + **kwargs, ): """client object to process ETM requests via its public API @@ -234,7 +325,7 @@ def __init__( # set session self.__kwargs = kwargs - self._session = session + self.session = session # set engine and token self.engine_url = engine_url @@ -247,15 +338,7 @@ def __init__( # set default gqueries self.gqueries = [] - # make message - msg = ( - "Initialised new Client: " - f"'scenario_id={self.scenario_id}, " - f"area_code={self.area_code}, " - f"end_year={self.end_year}'" - ) - - logger.debug(msg) + logger.debug("Initialised new Client: %s", self) def __enter__(self): """enter conext manager""" @@ -279,13 +362,13 @@ def __repr__(self): **{ "scenario_id": self.scenario_id, "engine_url": self.engine_url, - "session": self.session - }, - **self.__kwargs - } + "session": self.session, + }, + **self.__kwargs, + } # object environment - env = ", ".join(f'{k}={v}' for k, v in params.items()) + env = ", ".join(f"{k}={v}" for k, v in params.items()) return f"Client({env})" @@ -295,9 +378,9 @@ def __str__(self): # make stringname strname = ( "Client(" - f"scenario_id={self.scenario_id}, " - f"area_code={self.area_code}, " - f"end_year={self.end_year})" + f"scenario_id={self.scenario_id}, " + f"area_code={self.area_code if self.scenario_id else None}, " + f"end_year={self.end_year if self.scenario_id else None})" ) return strname @@ -307,14 +390,14 @@ def _reset_cache(self): # clear parameter caches self._get_scenario_header.cache_clear() - self.get_input_values.cache_clear() + self._get_input_parameters.cache_clear() # clear frame caches self.get_application_demands.cache_clear() self.get_energy_flows.cache_clear() - self.get_heat_network_order.cache_clear() self.get_production_parameters.cache_clear() self.get_sankey.cache_clear() + self.get_storage_parameters.cache_clear() # reset gqueries self.get_gquery_results.cache_clear() diff --git a/src/pyetm/client/curves.py b/src/pyetm/client/curves.py index 659718e..0163ca6 100644 --- a/src/pyetm/client/curves.py +++ b/src/pyetm/client/curves.py @@ -3,203 +3,115 @@ import functools import pandas as pd - -from pyetm.logger import get_modulelogger from .session import SessionMethods -# get modulelogger -logger = get_modulelogger(__name__) - - -class CurveMethods(SessionMethods): - """hourly curves""" - - @property - def _merit_order_enabled(self) -> bool: - """see if merit order is enabled""" - - # target input parameter - key = "settings_enable_merit_order" - url = f"scenarios/{self.scenario_id}/inputs/{key}" - # make request - headers = {'content-type': 'application/json'} - resp: dict = self.session.get(url, headers=headers) +def _get_curves(client: SessionMethods, extra: str, **kwargs) -> pd.DataFrame: + """wrapper to fetch curves from curves-endpoint""" - # format setting - enabled = resp.get('user', resp['default']) + # request parameters + url = client.make_endpoint_url(endpoint="curves", extra=extra) + buffer = client.session.get(url, content_type="text/csv") - # check for iterpolation issues - if not ((enabled == 1) | (enabled == 0)): - raise ValueError(f"invalid setting: '{key}'={enabled}") + return pd.read_csv(buffer, **kwargs) - return bool(enabled) - def _validate_merit_order(self): - """check if merit order is enabled""" - - # warn for disabled merit order - if self._merit_order_enabled is False: - logger.warning("%s: merit order disabled", self) +class CurveMethods(SessionMethods): + """hourly curves""" @property - def hourly_electricity_curves(self) -> pd.DataFrame: + def hourly_electricity_curves(self): """hourly electricity curves""" return self.get_hourly_electricity_curves() @functools.lru_cache(maxsize=1) - def get_hourly_electricity_curves(self) -> pd.DataFrame: + def get_hourly_electricity_curves(self): """get the hourly electricity curves""" - # raise without scenario id - self._validate_scenario_id() - self._validate_merit_order() - - # return empty frame with disabled merit order - if self._merit_order_enabled is False: - return pd.DataFrame() + # get curves + curves = _get_curves(self, extra="merit_order", index_col="Time") - # make request - post = f'scenarios/{self.scenario_id}/curves/merit_order' - resp = self.session.get(post, decoder="BytesIO") - - # convert to frame and set periodindex - curves = pd.read_csv(resp, index_col='Time', parse_dates=True) - curves.index = curves.index.to_period(freq='H') + # set periodindex + curves.index = pd.PeriodIndex(curves.index, freq="H").set_names(None) - return curves.rename_axis(None, axis=0) + return curves @property - def hourly_electricity_price_curve(self) -> pd.Series: + def hourly_electricity_price_curve(self): """hourly electricity price""" return self.get_hourly_electricity_price_curve() @functools.lru_cache(maxsize=1) - def get_hourly_electricity_price_curve(self) -> pd.Series: + def get_hourly_electricity_price_curve(self): """get the hourly electricity price curve""" - # raise without scenario id - self._validate_scenario_id() - self._validate_merit_order() - - # return empty series with disabled merit order - if self._merit_order_enabled is False: - return pd.Series() - - # make request - post = f'scenarios/{self.scenario_id}/curves/electricity_price' - resp = self.session.get(post, decoder="BytesIO") - - # convert to series - curves = pd.read_csv(resp, index_col='Time', parse_dates=True) - curves: pd.Series = curves.squeeze(axis=1) + # get squeezed curves + curves: pd.Series = _get_curves( + self, extra="electricity_price", index_col="Time" + ).squeeze(axis=1) # set periodindex - curves.index = curves.index.to_period(freq='H') - curves = curves.rename_axis(None, axis=0) + curves.index = pd.PeriodIndex(curves.index, freq="H").set_names(None) return curves.round(2) @property - def hourly_heat_curves(self) -> pd.DataFrame: + def hourly_heat_curves(self): """hourly heat curves""" return self.get_hourly_heat_curves() @functools.lru_cache(maxsize=1) - def get_hourly_heat_curves(self) -> pd.DataFrame: + def get_hourly_heat_curves(self): """get the hourly heat network curves""" - # raise without scenario id - self._validate_scenario_id() - self._validate_merit_order() - - # return empty frame with disabled merit order - if self._merit_order_enabled is False: - return pd.DataFrame() - - # request response and convert to frame - post = f'scenarios/{self.scenario_id}/curves/heat_network' - resp = self.session.get(post, decoder="BytesIO") + # get curves + curves = _get_curves(self, extra="heat_network", index_col="Time") - # convert to frame and set periodindex - curves = pd.read_csv(resp, index_col='Time', parse_dates=True) - curves.index = curves.index.to_period(freq='H') + # set periodindex + curves.index = pd.PeriodIndex(curves.index, freq="H").set_names(None) - return curves.rename_axis(None, axis=0) + return curves @property - def hourly_household_curves(self) -> pd.DataFrame: + def hourly_household_curves(self): """hourly household curves""" return self.get_hourly_household_curves() @functools.lru_cache(maxsize=1) - def get_hourly_household_curves(self) -> pd.DataFrame: + def get_hourly_household_curves(self): """get the hourly household heat curves""" - - # raise without scenario id - self._validate_scenario_id() - self._validate_merit_order() - - # return empty frame with disabled merit order - if self._merit_order_enabled is False: - return pd.DataFrame() - - # make request - post = f'scenarios/{self.scenario_id}/curves/household_heat' - resp = self.session.get(post, decoder="BytesIO") - - return pd.read_csv(resp) + return _get_curves(self, extra="household_heat") @property - def hourly_hydrogen_curves(self) -> pd.DataFrame: + def hourly_hydrogen_curves(self): """hourly hydrogen curves""" return self.get_hourly_hydrogen_curves() @functools.lru_cache(maxsize=1) - def get_hourly_hydrogen_curves(self) -> pd.DataFrame: + def get_hourly_hydrogen_curves(self): """get the hourly hydrogen curves""" - # raise without scenario id - self._validate_scenario_id() - self._validate_merit_order() - - # return empty frame with disabled merit order - if self._merit_order_enabled is False: - return pd.DataFrame() + # get curves + curves = _get_curves(self, extra="hydrogen", index_col="Time") - # make request - post = f'scenarios/{self.scenario_id}/curves/hydrogen' - resp = self.session.get(post, decoder="BytesIO") - - # convert to frame and set periodindex - curves = pd.read_csv(resp, index_col='Time', parse_dates=True) - curves.index = curves.index.to_period(freq='H') + # set periodindex + curves.index = pd.PeriodIndex(curves.index, freq="H").set_names(None) - return curves.rename_axis(None, axis=0) + return curves @property - def hourly_methane_curves(self) -> pd.DataFrame: + def hourly_methane_curves(self): """hourly methane curves""" return self.get_hourly_methane_curves() @functools.lru_cache(maxsize=1) - def get_hourly_methane_curves(self) -> pd.DataFrame: + def get_hourly_methane_curves(self): """get the hourly methane curves""" - # raise without scenario id - self._validate_scenario_id() - self._validate_merit_order() + # get curves + curves = _get_curves(self, extra="network_gas", index_col="Time") - # return empty frame with disabled merit order - if self._merit_order_enabled is False: - return pd.DataFrame() - - # make request - post = f'scenarios/{self.scenario_id}/curves/network_gas' - resp = self.session.get(post, decoder="BytesIO") - - # convert to frame and set periodindex - curves = pd.read_csv(resp, index_col='Time', parse_dates=True) - curves.index = curves.index.to_period(freq='H') + # set periodindex + curves.index = pd.PeriodIndex(curves.index, freq="H").set_names(None) - return curves.rename_axis(None, axis=0) + return curves diff --git a/src/pyetm/client/customcurves.py b/src/pyetm/client/customcurves.py index 65c3d17..725e3a9 100644 --- a/src/pyetm/client/customcurves.py +++ b/src/pyetm/client/customcurves.py @@ -1,12 +1,15 @@ """custom curve methods""" from __future__ import annotations +import functools -from collections.abc import Iterable +from collections.abc import Iterable, Mapping +from typing import Any -import functools import pandas as pd from pyetm.logger import get_modulelogger +from pyetm.utils.general import bool_to_json + from .session import SessionMethods # get modulelogger @@ -16,59 +19,51 @@ class CustomCurveMethods(SessionMethods): """Custom Curve Methods""" - def __overview(self, include_unattached: bool = False, - include_internal: bool = False) -> pd.DataFrame: + def _get_overview( + self, include_unattached: bool = False, include_internal: bool = False + ) -> pd.DataFrame: """fetch custom curve descriptives""" - # raise without scenario id - self._validate_scenario_id() - # newdict - params = {} + params: dict[str, str] = {} - # convert boolean - if include_unattached: - include_unattached = str(bool(include_unattached)) - params['include_unattached'] = include_unattached.lower() + # include unattached + if include_unattached is True: + params["include_unattached"] = bool_to_json(include_unattached) - # convert boolean - if include_internal: - include_internal = str(bool(include_internal)) - params['include_internal'] = include_internal.lower() + # include internal + if include_internal is True: + params["include_internal"] = bool_to_json(include_internal) - # request repsonse - url = f'scenarios/{self.scenario_id}/custom_curves' - resp = self.session.get(url, params=params) + # make url + url = self.make_endpoint_url("custom_curves") - # check for response - if bool(resp) is True: + # get custom curves + records = self.session.get(url, params=params, content_type="application/json") - # convert response to frame - ccurves = pd.DataFrame.from_records(resp, index="key") + # check for response + if bool(records) is True: + ccurves = pd.DataFrame.from_records(records, index="key") # format datetime column if "date" in ccurves.columns: - ccurves = self._format_datetime(ccurves) + ccurves["date"] = pd.to_datetime(ccurves["date"]) else: - # return empty frame ccurves = pd.DataFrame() return ccurves - def get_custom_curve_keys(self, include_unattached: bool = False, - include_internal: bool = False) -> pd.Index: + def get_custom_curve_keys( + self, include_unattached: bool = False, include_internal: bool = False + ) -> list[str]: """get all custom curve keys""" + return self._get_overview(include_unattached, include_internal).index.to_list() - # subset keys - params = include_unattached, include_internal - keys = self.__overview(*params).copy() - - return pd.Index(keys.index, name='ccurve_keys') - - def get_custom_curve_settings(self, include_unattached: bool = False, - include_internal: bool = False) -> pd.DataFrame: + def get_custom_curve_settings( + self, include_unattached: bool = False, include_internal: bool = False + ) -> pd.DataFrame: """show overview of custom curve settings""" # get relevant keys @@ -76,327 +71,225 @@ def get_custom_curve_settings(self, include_unattached: bool = False, keys = self.get_custom_curve_keys(*params) # empty frame without returned keys - if keys.empty: + if not keys: return pd.DataFrame() # reformat overrides - ccurves = self.__overview(*params).copy() - ccurves.overrides = ccurves.overrides.apply(len) + ccurves = self._get_overview(*params).copy() + ccurves["overrides"] = ccurves["overrides"].apply(len) # drop messy stats column - if 'stats' in ccurves.columns: - ccurves = ccurves[ccurves.columns.drop('stats')] + if "stats" in ccurves.columns: + ccurves = ccurves[ccurves.columns.drop("stats")] # drop unattached keys if not include_unattached: - ccurves = ccurves.loc[ccurves.attached] + ccurves = ccurves.loc[ccurves["attached"]] ccurves = ccurves.drop(columns="attached") - return ccurves + return ccurves.sort_index() - def get_custom_curve_user_value_overrides(self, - include_unattached: bool = False, - include_internal: bool = False) -> pd.DataFrame: + def get_custom_curve_user_value_overrides( + self, include_unattached: bool = False, include_internal: bool = False + ): """get overrides of user value keys by custom curves""" # subset and explode overrides - cols = ['overrides', 'attached'] + cols = ["overrides", "attached"] # get overview curves params = include_unattached, include_internal - overview = self.__overview(*params).copy() + overview = self._get_overview(*params).copy() + + # review overview + if overview.empty: + return overview # explode and drop na - overrides = overview[cols].explode('overrides') + overrides = overview[cols].explode("overrides") overrides = overrides.dropna() # reset index overrides = overrides.reset_index() - overrides.columns = ['overriden_by', 'user_value_key', 'active'] + overrides.columns = pd.Index(["override_by", "user_value_key", "active"]) # set index - overrides = overrides.set_index('user_value_key') + overrides = overrides.set_index("user_value_key") # subset active if not include_unattached: - return overrides.overriden_by[overrides.active] + return overrides.loc[overrides["active"]] return overrides @property - def custom_curves(self) -> pd.DataFrame: + def custom_curves(self) -> pd.Series[Any] | pd.DataFrame: """fetch custom curves""" return self.get_custom_curves() @custom_curves.setter - def custom_curves(self, ccurves: pd.DataFrame) -> None: + def custom_curves(self, ccurves: pd.Series[Any] | pd.DataFrame | None): """set custom curves without option to set a name""" - # check for old custom curves - keys = self.get_custom_curve_keys() - if not keys.empty: + # upload ccurves + if ccurves: + self.set_custom_curves(ccurves) - # remove old custom curves + # delete all ccurves + if ccurves is None: self.delete_custom_curves() - # set single custom curve - if isinstance(ccurves, pd.Series): - - if ccurves.name is not None: - self.upload_custom_curve(ccurves, ccurves.name) - - else: - raise KeyError("passed custom curve has no name") - - elif isinstance(ccurves, pd.DataFrame): - self.upload_custom_curves(ccurves) - - else: - raise TypeError("custom curves must be a series, frame or None") - - def __delete_ccurve(self, key: str) -> None: - """delete without raising or resetting""" - - # validate key - key = self._validate_ccurve_key(key) - - # make request - url = f'scenarios/{self.scenario_id}/custom_curves/{key}' - self.session.delete(url) - - def _format_datetime(self, ccurves: pd.DataFrame) -> pd.DataFrame: - """format datetime""" - - # format datetime - dtype = 'datetime64[ns, UTC]' - ccurves.date = ccurves.date.astype(dtype) - - # rename column - cols = {'date': 'datetime'} - ccurves = ccurves.rename(columns=cols) - - return ccurves - - def __get_ccurve(self, key: str) -> pd.Series: - """get custom curve""" - - # validate key - key = self._validate_ccurve_key(key) - - # make request - url = f'scenarios/{self.scenario_id}/custom_curves/{key}' - resp = self.session.get(url, decoder='BytesIO') - - # convert to series - curve = pd.read_csv(resp, header=None, names=[key]) - - return curve.squeeze('columns') - - def __upload_ccurve(self, curve: pd.Series, key: str | None = None, - name: str | None = None) -> None: - """upload without raising or resetting""" - - # resolve None - if key is None: - - # check series object - if isinstance(curve, pd.Series): - - # use series name - if curve.name is not None: - key = curve.name - - # validate key - key = self._validate_ccurve_key(key) - - # delete specified ccurve - if curve is None: - self.delete_custom_curve(key) - - # check ccurve - curve = self._check_ccurve(curve, key) - - # make request - url = f'scenarios/{self.scenario_id}/custom_curves/{key}' - self.session.upload_series(url, curve, name=name) - - def _check_ccurve(self, curve: pd.Series, key: str) -> pd.Series: - """check if a ccurve is compatible""" - - # subset columns from frame - if isinstance(curve, pd.DataFrame): - curve = curve[key] - - # assume list-like - if not isinstance(curve, pd.Series): - curve = pd.Series(curve, name=key) - - # check length - if not len(curve) == 8760: - raise ValueError("curve must contain 8760 entries") - - return curve - - def _validate_ccurve_key(self, key: str) -> str: + # consider moving validation to endpoint + def validate_ccurve_key(self, key: str): """check if key is valid ccurve""" - # raise for None - if key is None: - raise KeyError("No key specified for custom curve") - # check if key in ccurve index - params = {'include_unattached': True, 'include_internal': True} - if key not in self.get_custom_curve_keys(**params): + if str(key) not in self.get_custom_curve_keys( + include_unattached=True, include_internal=True + ): raise KeyError(f"'{key}' is not a valid custom curve key") - return key - - def delete_custom_curve(self, key: str) -> None: - """delate an uploaded ccurve""" - - # raise without scenario id - self._validate_scenario_id() - - # if key is attached - if key in self.get_custom_curve_keys(): - - # delete ccurve and reset - self.__delete_ccurve(key) - self._reset_cache() - - else: - # warn user for attempt - msg = (f"%s: attempted to remove '{key}', " + - "while curve already unattached") %self - logger.warning(msg) - - def delete_custom_curves(self, - keys: Iterable[str] | None = None) -> None: - """delete all custom curves""" - - # raise without scenario id - self._validate_scenario_id() - - # default keys - if keys is None: - keys = [] - - # convert iterable - if not isinstance(keys, pd.Index): - keys = pd.Index(keys) - - # get keys that need deleting - attached = self.get_custom_curve_keys() - - # default keys - if keys.empty: - keys = attached - - else: - # subset attached keys - keys = keys[keys.isin(attached)] - - # check validity - if (not attached.empty) & (not keys.empty): - - # delete all ccurves - for key in keys: - self.__delete_ccurve(key) - - # reset session - self._reset_cache() - - else: - # warn user for attempt - msg = ("%s: attempted to remove custom curves, " + - "without any (specified) custom curves attached") %self - logger.warning(msg) - - def get_custom_curve(self, key: str): - """return specific custom curve""" - key = self._validate_ccurve_key(key) - return self.custom_curves[key] - @functools.lru_cache(maxsize=1) - def get_custom_curves(self, - keys: Iterable[str] | None = None) -> pd.DataFrame: - """return all attached curstom curves""" - - # raise without scenario id - self._validate_scenario_id() - - # default keys - if keys is None: - keys = [] + def get_custom_curves( + self, keys: str | Iterable[str] | None = None + ) -> pd.Series[Any] | pd.DataFrame: + """get custom curve""" - # convert iterable - if not isinstance(keys, pd.Index): - keys = pd.Index(keys) + # get all attached keys + attached = self.get_custom_curve_keys(False, True) - # get keys that need deleting - attached = self.get_custom_curve_keys() + # handle single key + if isinstance(keys, str): + keys = [keys] - # default keys - if keys.empty: + # default to all attached keys + if keys is None: keys = attached - else: - # subset attached keys - keys = keys[keys.isin(attached)] - - # check validity - if not attached.empty: - - # get attached curves - func = self.__get_ccurve - ccurves = pd.concat([func(key) for key in attached], axis=1) + # warn user + if not keys: + logger.info("attempting to retrieve custom curves without any attached") + + # warn user + for key in set(keys).symmetric_difference(attached): + logger.info( + "attempting to retrieve '%s' while custom curve not attached", key + ) + + # get curves + curves: list[pd.Series[Any]] = [] + for key in set(keys).intersection(attached): + # validate key + self.validate_ccurve_key(key) + + # make request + url = self.make_endpoint_url(endpoint="custom_curves", extra=key) + buffer = self.session.get(url, content_type="text/csv") + + # append as series + curves.append(pd.read_csv(buffer, header=None, names=[key]).squeeze(axis=1)) + + return pd.concat(curves, axis=1).squeeze(axis=1) + + def set_custom_curves( + self, + ccurves: pd.Series[Any] | pd.DataFrame, + filenames: str | Iterable[str | None] | Mapping[str, str] | None = None, + ) -> None: + """upload custom curves and delete curves for keys that are not + present in the uploaded custom curves.""" + + # get all attached keys + attached = self.get_custom_curve_keys(False, True) + + # delete keys that are not reuploaded. + if attached: + # transform series + if isinstance(ccurves, pd.Series): + ccurves = ccurves.to_frame() + + # delete keys + self.delete_custom_curves( + keys=set(ccurves.columns).symmetric_difference(attached) + ) + + # upload custom curves + self.upload_custom_curves(ccurves, filenames=filenames) + + def upload_custom_curves( + self, + ccurves: pd.Series[Any] | pd.DataFrame, + filenames: str | Iterable[str | None] | Mapping[str, str] | None = None, + ) -> None: + """upload custom curves without deleting curves for keys that are + not present in the uploaded custom curves.""" + + # handle ccurves + if isinstance(ccurves, pd.Series): + ccurves = ccurves.to_frame() - else: + # convert single file names or None to iterable + if isinstance(filenames, str) or (filenames is None): + filenames = [filenames for _ in ccurves.columns] - # empty dataframe - ccurves = pd.DataFrame() + # convert iterable to mapping + if isinstance(filenames, Iterable): + # check for lenght mismatches + if len(list(filenames)) != len(ccurves.columns): + raise ValueError("lenght mismatch between ccurves and file names") - return ccurves[keys] + # convert to mapping + filenames = dict(zip(ccurves.columns, list(filenames))) - def upload_custom_curve(self, curve: pd.Series, key: str | None = None, - name: str | None = None) -> None: - """upload custom curve""" + # upload columns sequentually + for key, curve in ccurves.items(): + # validate key + key = str(key) + self.validate_ccurve_key(key) - # raise without scenario id - self._validate_scenario_id() + # check curve length + if not len(curve) == 8760: + raise ValueError(f"ccurve '{key}' must contain 8760 entries") - # upload ccurve - self.__upload_ccurve(curve, key, name) + # make request + url = self.make_endpoint_url(endpoint="custom_curves", extra=key) + self.session.upload(url, curve, filename=filenames[key]) # reset session self._reset_cache() - def upload_custom_curves(self, ccurves: pd.DataFrame, - names: list[str] | None = None) -> None: - """upload multiple ccurves at once""" + def delete_custom_curves(self, keys: str | Iterable[str] | None = None) -> None: + """delete custom curves""" - # raise without scenario id - self._validate_scenario_id() + # get all attached keys + attached = self.get_custom_curve_keys(False, True) - # delete all ccurves - if ccurves is None: - self.delete_custom_curves() + # handle single key + if isinstance(keys, str): + keys = [keys] - # list of Nones - if names is None: - names = [None for _ in ccurves.columns] + # default to all attached keys + if keys is None: + keys = attached - # convert single to list - if isinstance(names, str): - names = [names for _ in ccurves.columns] + # warn user + if not keys: + logger.info("attempting to unattach custom curves without any attached") - # raise for errors - if len(names) != len(ccurves.columns): - raise ValueError('number of names does not match number of curves') + # warn user + for key in set(keys).symmetric_difference(attached): + logger.info( + "attempting to remove '%s' while custom curve already unattached", key + ) - # upload all ccurves to ETM - for idx, key in enumerate(ccurves.columns): - self.__upload_ccurve(ccurves[key], key, name=names[idx]) + # delete curves + for key in set(keys).intersection(attached): + # validate key + self.validate_ccurve_key(key) - # reset session + # make request + url = self.make_endpoint_url(endpoint="custom_curves", extra=key) + self.session.delete(url) + + # reset cache self._reset_cache() diff --git a/src/pyetm/client/gqueries.py b/src/pyetm/client/gqueries.py index 4c315d9..ea55046 100644 --- a/src/pyetm/client/gqueries.py +++ b/src/pyetm/client/gqueries.py @@ -1,5 +1,6 @@ """graph query methods""" import functools + import pandas as pd from .session import SessionMethods @@ -9,7 +10,7 @@ class GQueryMethods(SessionMethods): """Graph query methods""" @property - def gqueries(self) -> pd.Index: + def gqueries(self): """returns a list of set gqueries""" return self._gqueries @@ -41,10 +42,10 @@ def gquery_curves(self): # subset curves from gquery_results gqueries = self.gquery_results - gqueries = gqueries[gqueries.unit == 'curve'] + gqueries = gqueries[gqueries.unit == "curve"] # subset future column and convert to series - gqueries = gqueries.future.apply(pd.Series) + gqueries = gqueries.future.apply(pd.Series) return gqueries.T @@ -54,7 +55,7 @@ def gquery_deltas(self): # subset deltas from gquery_results gqueries = self.gquery_results - gqueries = gqueries[gqueries.unit != 'curve'] + gqueries = gqueries[gqueries.unit != "curve"] return gqueries @@ -74,14 +75,13 @@ def get_gquery_results(self): self._validate_scenario_id() # create gquery request - data = {'gqueries': self.gqueries} - url = f'scenarios/{self.scenario_id}' + data = {"gqueries": self.gqueries} + url = self.make_endpoint_url(endpoint="scenario_id") # evaluate post - response = self.session.put(url, json=data) + message = self.session.put(url, json=data) # transform into dataframe - records = response['gqueries'] - gquery_results = pd.DataFrame.from_dict(records, orient='index') + gquery_results = pd.DataFrame.from_dict(message["gqueries"], orient="index") return gquery_results diff --git a/src/pyetm/client/meritorder.py b/src/pyetm/client/meritorder.py index a6eb859..83ac9e6 100644 --- a/src/pyetm/client/meritorder.py +++ b/src/pyetm/client/meritorder.py @@ -5,6 +5,7 @@ import pandas as pd from pyetm.logger import get_modulelogger + from .session import SessionMethods # get modulelogger @@ -14,46 +15,49 @@ class MeritOrderMethods(SessionMethods): """Merit Order Methods""" - def _get_merit_configuration(self, include_curves=True): + def _get_merit_configuration(self, include_curves: bool = True): """get merit configuration JSON""" - # lower cased boolean for params - include_curves = str(bool(include_curves)) - - # raise without scenario id - self._validate_scenario_id() - - # prepare request - params = {'include_curves': include_curves.lower()} - url = f'scenarios/{self.scenario_id}/merit' + # request parameters + params = {"include_curves": str(bool(include_curves)).lower()} + url = self.make_endpoint_url(endpoint="scenario_id", extra="merit") - # request response - resp = self.session.get(url, params=params) + # make request + configuration = self.session.get( + url, params=params, content_type="application/json" + ) - return resp + return configuration def get_participants(self, subset=None): """get particpants from merit configuration""" # supported subtypes - supported = ['total_consumption', 'with_curve', 'generic', - 'storage', 'dispatchable', 'must_run', 'volatile'] + supported = [ + "total_consumption", + "with_curve", + "generic", + "storage", + "dispatchable", + "must_run", + "volatile", + ] # subset all types if subset is None: subset = supported # subset consumer types - elif (subset == 'consumer') | (subset == 'consumers'): - subset = ['total_consumption', 'with_curve'] + elif (subset == "consumer") | (subset == "consumers"): + subset = ["total_consumption", "with_curve"] # subset flexible types - elif (subset == 'flexible') | (subset == 'flexibles'): - subset = ['generic', 'storage'] + elif (subset == "flexible") | (subset == "flexibles"): + subset = ["generic", "storage"] # subset producer types - elif (subset == 'producer') | (subset == 'producers'): - subset = ['dispatchable', 'must_run', 'volatile'] + elif (subset == "producer") | (subset == "producers"): + subset = ["dispatchable", "must_run", "volatile"] # other keys always in list if isinstance(subset, str): @@ -64,22 +68,24 @@ def get_participants(self, subset=None): subset = list(subset) # correct response JSON - recs = self._get_merit_configuration(False)['participants'] - recs = [rec for rec in recs if rec.get('type') in subset] + recs = self._get_merit_configuration(False)["participants"] + recs = [rec for rec in recs if rec.get("type") in subset] def correct(rec): """null correction in recordings""" - return {k: (v if (v != 'null') & (v is not None) else np.nan) - for k, v in rec.items()} + return { + k: (v if (v != "null") & (v is not None) else np.nan) + for k, v in rec.items() + } # correct records to replace null with None recs = [correct(rec) for rec in recs] - frame = pd.DataFrame.from_records(recs, index='key') + frame = pd.DataFrame.from_records(recs, index="key") frame = frame.rename_axis(None, axis=0).sort_index() # drop curve column - if 'curve' in frame.columns: - frame = frame.drop(columns='curve') + if "curve" in frame.columns: + frame = frame.drop(columns="curve") return frame @@ -91,11 +97,11 @@ def get_participant_curves(self): # map participants to curve names # drops paricipants without curve - recs = response['participants'] - cmap = {rec['key']: rec['curve'] for rec in recs if rec['curve']} + recs = response["participants"] + cmap = {rec["key"]: rec["curve"] for rec in recs if rec["curve"]} # extract curves from response - curves = response['curves'] + curves = response["curves"] curves = pd.DataFrame.from_dict(curves) def sset_column(key, value): @@ -113,17 +119,13 @@ def get_dispatchables_bidladder(self): returns both marginal costs and installed capacity""" # get all dispatchable units - units = self.get_participants(subset='dispatchable') + units = self.get_participants(subset="dispatchable") # cap related keys - keys = [ - 'availability', - 'number_of_units', - 'output_capacity_per_unit' - ] + keys = ["availability", "number_of_units", "output_capacity_per_unit"] # evalaute capacity and specify relevant columns - units['capacity'] = units[keys].product(axis=1) - units = units[['marginal_costs', 'capacity']] + units["capacity"] = units[keys].product(axis=1) + units = units[["marginal_costs", "capacity"]] - return units.sort_values(by='marginal_costs') + return units.sort_values(by="marginal_costs") diff --git a/src/pyetm/client/parameters.py b/src/pyetm/client/parameters.py index 646d448..08f964a 100644 --- a/src/pyetm/client/parameters.py +++ b/src/pyetm/client/parameters.py @@ -1,11 +1,14 @@ """parameters object""" +from __future__ import annotations import functools -from pyetm.logger import get_modulelogger +from typing import overload, Literal, Any import numpy as np import pandas as pd +from pyetm.logger import get_modulelogger + from .session import SessionMethods logger = get_modulelogger(__name__) @@ -14,427 +17,283 @@ class ParameterMethods(SessionMethods): """collector class for parameter objects""" - @property - def application_demands(self): - """application demands""" - return self.get_application_demands() - - @functools.lru_cache(maxsize=1) - def get_application_demands(self): - """get the application demands""" - - # raise without scenario id - self._validate_scenario_id() - - # make request - url = f'scenarios/{self.scenario_id}/application_demands' - resp = self.session.get(url, decoder="BytesIO") - - return pd.read_csv(resp, index_col='key') + ## Inputs ## @property - def energy_flows(self): - """energy flows""" - return self.get_energy_flows() - - @functools.lru_cache(maxsize=1) - def get_energy_flows(self): - """get the energy flows""" - - # raise without scenario id - self._validate_scenario_id() + def input_parameters(self) -> pd.Series[Any]: + """scenario input parameters""" + return self.get_input_parameters(False, False, False) - # make request - url = f'scenarios/{self.scenario_id}/energy_flow' - resp = self.session.get(url, decoder="BytesIO") - - # convert to frame - flows = pd.read_csv(resp, index_col='key') - - return flows - - @property - def forecast_storage_order(self): - """forecast storage order""" - return self.get_forecast_storage_order() - - @forecast_storage_order.setter - def heat_netforecast_storage_orderwork_order(self, order): - self.change_forecast_storage_order(order) + @input_parameters.setter + def input_parameters( + self, inputs: dict[str, str | float] | pd.Series[Any] | None + ) -> None: + self.set_input_parameters(inputs) @functools.lru_cache(maxsize=1) - def get_forecast_storage_order(self): - """get the heat network order""" - - # raise without scenario id - self._validate_scenario_id() + def _get_input_parameters(self) -> pd.DataFrame: + """cached configuration""" # make request - url = f'scenarios/{self.scenario_id}/forecast_storage_order' - resp = self.session.get(url) + url = self.make_endpoint_url(endpoint="inputs") + records = self.session.get(url, content_type="application/json") - # get order - order = resp["order"] + # convert records to frame + parameters = pd.DataFrame.from_records(records).T + parameters = parameters.drop(columns="cache_error") - return order + # infer dtypes + parameters = parameters.infer_objects() - def change_forecast_storage_order(self, order): - """change forecast storage order - - parameters - ---------- - order : list - Desired forecast storage order""" + # add user to column when absent + if "user" not in parameters.columns: + parameters.insert(loc=5, column="user", value=np.nan) - # raise without scenario id - self._validate_scenario_id() + return parameters - # convert np array to list - if isinstance(order, np.ndarray): - order = order.tolist() + @overload + def get_input_parameters( + self, + user_only: bool = False, + include_disabled: bool = False, + detailed: Literal[False] = False, + share_group: str | None = None, + ) -> pd.Series[str | float]: + pass + + @overload + def get_input_parameters( + self, + user_only: bool = False, + include_disabled: bool = False, + detailed: Literal[True] = True, + share_group: str | None = None, + ) -> pd.DataFrame: + pass + + def get_input_parameters( + self, + user_only: bool = False, + include_disabled: bool = False, + detailed: bool = False, + share_group: str | None = None, + ) -> pd.Series[str | float] | pd.DataFrame: + """Get the scenario input parameters from the ETM server. + + Parameters + ---------- + user_only: boolean, default False + Exclude parameters not set by the user in the returned results. + include_disabled: boolean, default False + Include disabled parameters in returned results. + detailed: boolean, default False + Include additional information for each parameter in the + returned result, e.g. the parameter bounds. + share_group: optional string + Only return results for the specified share group. + + Return + ------ + parameters: Series or DataFrame + The scenario's input parameters. Returns a series by default + and returns a DataFrame when detailed is set to True.""" + + # exclude parameters without unit (seem to be irrelivant and disabled) + parameters = self._get_input_parameters() + parameters = parameters.loc[~parameters["unit"].isna()] + + # drop disabled + if not include_disabled: + parameters = parameters.loc[~parameters["disabled"]] + + # drop non-user configured parameters + if user_only: + user = ~parameters["user"].isna() + parameters = parameters.loc[user] + + # subset share group + if share_group is not None: + # check share group + if share_group not in parameters["share_group"].unique(): + raise ValueError(f"share group does not exist: {share_group}") + + # subset share group + parameters = parameters[parameters["share_group"] == share_group] + + # show all details + if detailed: + return parameters + + # set missing defaults + parameters["user"] = parameters["user"].fillna(parameters["default"]) + + # subset user set inputs + user = parameters["user"] + user.name = "inputs" + + return user + + def set_input_parameters( + self, inputs: dict[str, str | float] | pd.Series[Any] | pd.DataFrame | None + ) -> None: + """set scenario input parameters""" - # acces dict for order - if isinstance(order, dict): - order = order['order'] + # convert None to dict + if inputs is None: + inputs = {} - # check items in order - for item in order: - if item not in self.forecast_storage_order: - raise ValueError( - f"Invalid forecast storage order item: '{item}'") + # subset series from df + if isinstance(inputs, pd.DataFrame): + inputs = inputs["user"] - # map order to correct scenario parameter - data = {'order': order} + # prepare request + headers = {"content-type": "application/json"} + data = {"scenario": {"user_values": dict(inputs)}, "detailed": True} # make request - url = f'scenarios/{self.scenario_id}/forecast_storage_order' - self.session.put(url, json=data) + url = self.make_endpoint_url(endpoint="scenario_id") + self.session.put(url, json=data, headers=headers) - # reinitialize scenario + # reset cached parameters self._reset_cache() + ## Orders ## + @property - def heat_network_order(self): + def heat_network_order(self) -> list[str]: """heat network order""" - return self.get_heat_network_order() - - @heat_network_order.setter - def heat_network_order(self, order): - self.change_heat_network_order(order) - - @functools.lru_cache(maxsize=1) - def get_heat_network_order(self): - """get the heat network order""" - # raise without scenario id - self._validate_scenario_id() + # make url + extra = "heat_network_order" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) # make request - url = f'scenarios/{self.scenario_id}/heat_network_order' - resp = self.session.get(url) - - # get order - order = resp["order"] + order = self.session.get(url, content_type="application/json") - return order - - def change_heat_network_order(self, order): - """change heat network order - - parameters - ---------- - order : list - Desired heat network order""" - - # raise without scenario id - self._validate_scenario_id() - - # convert np array to list - if isinstance(order, np.ndarray): - order = order.tolist() - - # acces dict for order - if isinstance(order, dict): - order = order['order'] + return order["order"] + @heat_network_order.setter + def heat_network_order(self, order: list[str]): # check items in order for item in order: if item not in self.heat_network_order: - raise ValueError( - f"Invalid heat network order item: '{item}'") + raise ValueError(f"Invalid heat network order item: '{item}'") + + # request parameters + data = {"order": order} + headers = {"content-type": "application/json"} - # map order to correct scenario parameter - data = {'order': order} + # make url + extra = "heat_network_order" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) # make request - url = f'scenarios/{self.scenario_id}/heat_network_order' - self.session.put(url, json=data) + self.session.put(url, json=data, headers=headers) - # reinitialize scenario + # reset cached items self._reset_cache() @property - def input_values(self): - """input values""" - return self.get_input_values() - - @input_values.setter - def input_values(self, uparams): - raise AttributeError('protected attribute; change user values instead.') - - @functools.lru_cache(maxsize=1) - def get_input_values(self): - """get configuration information of all available input parameters. - direct dump of inputs json from engine.""" + def forecast_storage_order(self) -> list[str]: + """forecast storage order""" - # raise without scenario id - self._validate_scenario_id() + # make url + extra = "forecast_storage_order" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) # make request - url = f'scenarios/{self.scenario_id}/inputs' - resp = self.session.get(url) - - # convert to frame - ivalues = pd.DataFrame.from_dict(resp, orient='index') - - # add user to column when absent - if 'user' not in ivalues.columns: - ivalues.insert(loc=5, column='user', value=np.nan) + order = self.session.get(url, content_type="application/json") - # convert user dtype to object and set disabled - ivalues.user = ivalues.user.astype('object') - ivalues.disabled = ivalues.disabled.fillna(False) + return order["order"] - return ivalues - - @property - def production_parameters(self): - """production parameters""" - return self.get_production_parameters() + @forecast_storage_order.setter + def forecast_storage_order(self, order: list[str]) -> None: + # check items in order + for item in order: + if item not in self.forecast_storage_order: + raise ValueError(f"Invalid forecast storage order item: '{item}'") - @functools.lru_cache(maxsize=1) - def get_production_parameters(self): - """get the production parameters""" + # request parameters + data = {"order": order} + headers = {"content-type": "application/json"} - # raise without scenario id - self._validate_scenario_id() + # make url + extra = "forecast_storage_order" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) # make request - url = f'scenarios/{self.scenario_id}/production_parameters' - resp = self.session.get(url, decoder="BytesIO") + self.session.put(url, json=data, headers=headers) - return pd.read_csv(resp) + # reset cached items + self._reset_cache() - @property - def sankey(self): - """sankey diagram""" - return self.get_sankey() + ## MISC ## @functools.lru_cache(maxsize=1) - def get_sankey(self): - """get the sankey data""" - - # raise without scenario id - self._validate_scenario_id() - - # make request - url = f'scenarios/{self.scenario_id}/sankey' - resp = self.session.get(url, decoder="BytesIO") - - # convert to frame - cols = ['Group', 'Carrier', 'Category', 'Type'] - sankey = pd.read_csv(resp, index_col=cols) - - return sankey - - @property - def scenario_parameters(self): - """all user values including non-user defined parameters""" - - # get user and fillna with default - uparams = self.user_parameters - sparams = uparams.user.fillna(uparams.default) - - # set name of series - sparams.name = 'scenario' - - return sparams - - @scenario_parameters.setter - def scenario_parameters(self, sparams): - - # check and set scenario parameters - self._check_scenario_parameters(sparams) - self.change_user_values(sparams) - - def _check_scenario_parameters(self, sparams=None): - """Utility function to check the validity of the scenario - parameters that are set in the scenario.""" - - # default sparams - if sparams is None: - sparams = self.scenario_parameters - - # check passed parameters as user values - sparams = self._check_user_values(sparams) + def get_application_demands(self) -> pd.DataFrame: + """get the application demands""" - # ensure that they are complete - passed = self.scenario_parameters.index.isin(sparams.index) - if not passed.all(): - missing = self.scenario_parameters[~passed] + # make url + extra = "application_demands" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) - # warn for each missing key - for key in missing.index: - logger.warning(f"'{key}' not in passed scenario parameters") + # make request and convert to frame + buffer = self.session.get(url, content_type="text/csv") + demands = pd.read_csv(buffer, index_col="key") - @property - def storage_parameters(self): - """storage volumes and capacities""" - return self.get_storage_parameters() + return demands @functools.lru_cache(maxsize=1) - def get_storage_parameters(self): + def get_storage_parameters(self) -> pd.DataFrame: """get the storage parameter data""" - # raise without scenario id - self._validate_scenario_id() + # make request + extra = "storage_parameters" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) # make request - url = f'scenarios/{self.scenario_id}/storage_parameters' - resp = self.session.get(url, decoder="BytesIO") + buffer = self.session.get(url, content_type="text/csv") # convert to frame - cols = ['Group', 'Carrier', 'Key', 'Parameter'] - parameters = pd.read_csv(resp, index_col=cols) + cols = ["group", "carrier", "key", "parameter"] + parameters = pd.read_csv(buffer, index_col=cols) return parameters - @property - def user_parameters(self): - """user parameters""" - return self.get_user_parameters() - - @user_parameters.setter - def user_parameters(self, uparams): - raise AttributeError('protected attribute; change user values instead.') - - def get_user_parameters(self): - """get configuration information of all available user parameters""" - - # raise without scenario id - self._validate_scenario_id() - - # drop disabled parameters - ivalues = self.input_values - uparams = ivalues[~ivalues.disabled] - - return uparams - - @property - def user_values(self): - """all user set values without non-user defined parameters""" - return self.get_user_values() - - @user_values.setter - def user_values(self, uvalues): - self.change_user_values(uvalues) - - def get_user_values(self): - """get the parameters that are configued by the user""" - - # raise without scenario id - self._validate_scenario_id() - - # subset values from user parameter df - uvalues = self.user_parameters['user'] - uvalues = uvalues.dropna() - - return uvalues - - def change_user_values(self, uvalues): - """change the passed user values in the ETM. - - parameters - ---------- - uvalues : pandas.Series - collection of key, value pairs of user values.""" - - # raise without scenario id - self._validate_scenario_id() - - # validate passed user values - uvalues = self._check_user_values(uvalues) - - # convert uvalues to dict - uvalues = uvalues.to_dict() - - # map values to correct scenario parameters - data = {"scenario": {"user_values": uvalues}, "detailed": True} - - # evaluate request - url = f'scenarios/{self.scenario_id}' - self.session.put(url, json=data) - - # reinitialize scenario - self._reset_cache() - - def _check_user_values(self, uvalues): - """check if all user values can be passed to ETM.""" - - # convert None to dict - if uvalues is None: - uvalues = {} - - # convert dict to series - if isinstance(uvalues, dict): - uvalues = pd.Series(uvalues, name='user', dtype='object') - - # subset series from df - if isinstance(uvalues, pd.DataFrame): - uvalues = uvalues.user - - return uvalues - - def _get_sharegroup(self, key): - """return subset of parameters in share group""" + @functools.lru_cache(maxsize=1) + def get_production_parameters(self) -> pd.DataFrame: + """get the production parameters""" - # get user and scenario parameters - uparams = self.user_parameters - sparams = self.scenario_parameters + # make url + extra = "production_parameters" + url = self.make_endpoint_url(endpoint="scenario_id", extra=extra) - return sparams[uparams.share_group == key] + # make request and convert to frame + buffer = self.session.get(url, content_type="text/csv") + parameters = pd.read_csv(buffer) - # @property - # def _cvalues(self): - # """continous user values""" + return parameters - # # get relevant parameters - # keys = self._dvalues.index - # cvalues = self.scenario_parameters + @functools.lru_cache(maxsize=1) + def get_energy_flows(self) -> pd.DataFrame: + """get the energy flows""" - # # get continious parameters - # cvalues = cvalues[~cvalues.index.isin(keys)] + # make request + url = self.make_endpoint_url(endpoint="scenario_id", extra="energy_flow") + buffer = self.session.get(url, content_type="text/csv") - # return cvalues.astype('float64') + return pd.read_csv(buffer, index_col="key") - # @property - # def _dvalues(self): - # """discrete user values""" + @functools.lru_cache(maxsize=1) + def get_sankey(self) -> pd.DataFrame: + """get the sankey data""" - # keys = [ - # 'heat_storage_enabled', - # 'merit_order_subtype_of_energy_power_nuclear_uranium_oxide', - # 'settings_enable_merit_order', - # 'settings_enable_storage_optimisation_energy_flexibility_hv_opac_electricity', - # 'settings_enable_storage_optimisation_energy_flexibility_pumped_storage_electricity', - # 'settings_enable_storage_optimisation_energy_flexibility_mv_batteries_electricity', - # 'settings_enable_storage_optimisation_energy_flexibility_flow_batteries_electricity', - # 'settings_enable_storage_optimisation_transport_car_flexibility_p2p_electricity', - # 'settings_weather_curve_set', - # ] + # make request + url = self.make_endpoint_url(endpoint="scenario_id", extra="sankey") + buffer = self.session.get(url, content_type="text/csv") - # # get discrete parameters - # dvalues = self.scenario_parameters - # dvalues = dvalues[dvalues.index.isin(keys)] + # convert to frame + cols = ["group", "carrier", "category", "type"] + sankey = pd.read_csv(buffer, index_col=cols) - # return dvalues + return sankey diff --git a/src/pyetm/client/scenario.py b/src/pyetm/client/scenario.py index 7f1b00e..a63dfa9 100644 --- a/src/pyetm/client/scenario.py +++ b/src/pyetm/client/scenario.py @@ -2,10 +2,11 @@ from __future__ import annotations -import copy -from urllib.parse import urljoin +from typing import Any +import copy import pandas as pd + from .session import SessionMethods @@ -15,14 +16,14 @@ class ScenarioMethods(SessionMethods): @property def area_code(self) -> str: """code for the area that the scenario describes""" - return self._scenario_header.get('area_code') + return self._scenario_header["area_code"] @property - def created_at(self) -> pd.Timestamp: + def created_at(self) -> pd.Timestamp | None: """timestamp at which the scenario was created""" # get created at - datetime = self._scenario_header.get('created_at') + datetime = self._scenario_header.get("created_at") # format datetime if datetime is not None: @@ -33,89 +34,95 @@ def created_at(self) -> pd.Timestamp: @property def end_year(self) -> int: """target year for which the scenario is configured""" - return self._scenario_header.get('end_year') + return self._scenario_header["end_year"] @property - def esdl_exportable(self): + def esdl_exportable(self) -> str | None: """scenario can be exported as esdl""" - return self._scenario_header.get('esdl_exportable') + return self._scenario_header.get("esdl_exportable") @property - def keep_compatible(self) -> bool: + def keep_compatible(self) -> bool | None: """migrate scenario with ETM updates""" - return self._scenario_header.get('keep_compatible') + return self._scenario_header.get("keep_compatible") @keep_compatible.setter def keep_compatible(self, boolean: bool): - # format header and update - header = {'keep_compatible': str(bool(boolean)).lower()} + header = {"keep_compatible": str(bool(boolean)).lower()} self._update_scenario_header(header) @property - def metadata(self) -> dict: + def metadata(self) -> dict[str, Any]: """metadata tags""" - return self._scenario_header.get('metadata') + return self._scenario_header.get("metadata", {}) @metadata.setter - def metadata(self, metadata: dict): - + def metadata(self, metadata: dict[str, Any] | None): # format header and update - header = {'metadata': dict(metadata)} + + # remove metadata + if metadata is None: + metadata = {} + + # apply update + header = {"metadata": dict(metadata)} self._update_scenario_header(header) @property - def owner(self) -> dict: + def owner(self) -> dict | None: """scenario owner if created by logged in user""" return self._scenario_header.get("owner") @property - def private(self) -> bool: + def private(self) -> bool | None: """boolean that determines if the scenario is private""" - return self._scenario_header.get('private') + return self._scenario_header.get("private") @private.setter def private(self, boolean: bool): - # # validate token permission - self._validate_token_permission(scope='scenarios:write') + self._validate_token_permission(scope="scenarios:write") # format header and update - header = {'private': str(bool(boolean)).lower()} + header = {"private": str(bool(boolean)).lower()} self._update_scenario_header(header) - @property - def pro_url(self) -> str: - """get pro url for session id""" - return urljoin(self.etm_url, f'scenarios/{self.scenario_id}/load/') - @property def scaling(self): """applied scaling factor""" - return self._scenario_header.get('scaling') + return self._scenario_header.get("scaling") @property def source(self): """origin of the scenario""" - return self._scenario_header.get('source') + return self._scenario_header.get("source") @property - def start_year(self) -> int: + def start_year(self) -> int | None: """get the reference year on which the default settings are based""" - return self._scenario_header.get('start_year') + return self._scenario_header.get("start_year") @property def template(self) -> int | None: """the id of the scenario that was used as a template, or None if no template was used.""" - return str(self._scenario_header.get('template')) + + # get template scenario + template = self._scenario_header.get("template") + + # convert to id + if template is not None: + template = int(template) + + return template @property - def updated_at(self) -> pd.Timestamp: + def updated_at(self) -> pd.Timestamp | None: """get timestamp of latest change""" # get created at - datetime = self._scenario_header.get('updated_at') + datetime = self._scenario_header.get("updated_at") # format datetime if datetime is not None: @@ -123,24 +130,21 @@ def updated_at(self) -> pd.Timestamp: return datetime - @property - def url(self) -> str: - """get url""" - return self._scenario_header.get('url') - - def add_metadata(self, metadata: dict): + def add_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: """append metadata""" original = copy.deepcopy(self.metadata) self.metadata = {**original, **metadata} + return self.metadata + def copy_scenario( self, - scenario_id: str | None = None, + scenario_id: int | None = None, metadata: dict | None = None, keep_compatible: bool | None = None, private: bool | None = None, - connect: bool = True + connect: bool = True, ) -> int: """Create a new scenario that is a copy of an existing scenario based on its id. The client automatically connects to the the @@ -171,7 +175,6 @@ def copy_scenario( # use own scenario id if scenario_id is None: - # raise without scenario id self._validate_scenario_id() scenario_id = self.scenario_id @@ -179,14 +182,17 @@ def copy_scenario( # remember original scenario id. previous = copy.deepcopy(self.scenario_id) - # make and set scenario - scenario = {'scenario_id': str(scenario_id)} - data = {"scenario": scenario} + # request parameters + data = {"scenario": {"scenario_id": str(scenario_id)}} + headers = {"content-type": "application/json"} + + # make request + url = self.make_endpoint_url(endpoint="scenarios") + scenario = self.session.post(url, json=data, headers=headers) - # request scenario id and connect - url = 'scenarios' - self.scenario_id = self.session.post( - url, json=data)['id'] + # connect to new scenario id + scenario_id = int(scenario["id"]) + self.scenario_id = scenario_id # set metadata parmater if metadata is not None: @@ -212,8 +218,8 @@ def create_new_scenario( end_year: int, metadata: dict | None = None, keep_compatible: bool | None = None, - private: bool | None = None - ) -> None: + private: bool | None = None, + ) -> int: """Create a new scenario on the ETM server. Parameters @@ -236,17 +242,19 @@ def create_new_scenario( end_year = int(end_year) # make scenario dict based on args - scenario = {'end_year': end_year, 'area_code' : area_code} + scenario = {"end_year": end_year, "area_code": area_code} - # set scenario parameter + # request parameters data = {"scenario": scenario} + headers = {"content-type": "application/json"} + url = self.make_endpoint_url(endpoint="scenarios") - # make request - url = 'scenarios' - response = self.session.post(url, json=data) + # get scenario_id + scenario = self.session.post(url, json=data, headers=headers) - # update scenario_id - self.scenario_id = str(response['id']) + # connect to new scenario + scenario_id = int(scenario["id"]) + self.scenario_id = scenario_id # set scenario metadata if metadata is not None: @@ -260,23 +268,23 @@ def create_new_scenario( if private is not None: self.private = private - def delete_scenario(self, scenario_id: str | None = None) -> None: + return scenario_id + + def delete_scenario(self, scenario_id: int | None = None) -> None: """Delete scenario""" # validate token - self._validate_token_permission(scope='scenarios:delete') + self._validate_token_permission(scope="scenarios:delete") # use connected scenario previous = None - if (scenario_id is not None) & ((str(scenario_id)) != self.scenario_id): - - # remember original connected scenario - # and connect to passed scenario id - previous = copy.deepcopy(self.scenario_id) - self.scenario_id = scenario_id + if scenario_id is not None: + if int(scenario_id) != self.scenario_id: + previous = copy.deepcopy(self.scenario_id) + self.scenario_id = scenario_id # delete scenario - url = f'scenarios/{self.scenario_id}' + url = self.make_endpoint_url(endpoint="scenario_id") self.session.delete(url=url) # connect to previous or None @@ -296,18 +304,16 @@ def interpolate_scenario(self, ryear: int, connect: bool = False): # check scenario end year if self.end_year != 2050: - raise NotImplementedError( - 'Can only interpolate based on 2050 scenarios') - - # pass end year to interpolate tool - data = {'end_year': ryear} + raise NotImplementedError("Can only interpolate based on 2050 scenarios") - # make requestd - url = f'scenarios/{self.scenario_id}/interpolate' - scenario = self.session.post(url, json=data, decoder='json') + # request parameters + data = {"end_year": ryear} + headers = {"content-type": "application/json"} + url = self.make_endpoint_url(endpoint="scenario_id", extra="interpolate") - # get scenario id - scenario_id = scenario['id'] + # get scenario_id + scenario = self.session.post(url, json=data, headers=headers) + scenario_id = int(scenario["id"]) # connect to new scenario if connect is True: @@ -321,10 +327,11 @@ def reset_scenario(self) -> None: # set reset parameter data = {"reset": True} - url = f'scenarios/{self.scenario_id}' + headers = {"content-type": "application/json"} + url = self.make_endpoint_url(endpoint="scenario_id") # make request - self.session.put(url, json=data) + self.session.put(url, json=data, headers=headers) # reinitialize connected scenario self._reset_cache() @@ -363,38 +370,44 @@ def to_saved_scenario( self._validate_token_permission("scenarios:write") # prepare request - url = 'saved_scenarios' - headers = {'content-type': 'application/json'} - - # make data - data = {"scenario_id": self.copy_scenario(connect=False)} + headers = {"content-type": "application/json"} + data: dict[str, Any] = {"scenario_id": self.copy_scenario(connect=False)} # update exisiting saved scenario if saved_scenario_id is not None: + # make url + url = self.make_endpoint_url( + endpoint="saved_scenarios", extra=str(saved_scenario_id) + ) - # update url - url += f'/{saved_scenario_id}' + # make request + scenario = self.session.post(url, json=data, headers=headers) - return self.session.put( - url, json=data, headers=headers) + return int(scenario["id"]) # default title if title is None: - title = f'API Generated - {self.scenario_id}' + title = f"API Generated - {self.scenario_id}" # add title data["title"] = title # add privacy setting if private is not None: - data["private"]: str(bool(private)).lower() + data["private"] = bool(private) # add description if description is not None: data["description"] = str(description) - return self.session.post( - url, json=data, headers=headers) + # make url + url = self.make_endpoint_url(endpoint="saved_scenarios") + + # make request + print(data) + scenario = self.session.post(url, json=data, headers=headers) + + return int(scenario["id"]) # def to_dict(self): #, path: str | Path) -> None: # """export full scenario to dict""" diff --git a/src/pyetm/client/session.py b/src/pyetm/client/session.py index 0ef9812..59e363d 100644 --- a/src/pyetm/client/session.py +++ b/src/pyetm/client/session.py @@ -1,24 +1,21 @@ """Base methods and client""" from __future__ import annotations -from typing import Literal -import os -import re import copy import functools - -from urllib.parse import urljoin +import os +import re +from typing import Any import pandas as pd from pyetm.logger import get_modulelogger -from pyetm.sessions import RequestsSession, AIOHTTPSession +from pyetm.sessions.abc import SessionABC +from pyetm.types import TokenScope, Endpoint # get modulelogger logger = get_modulelogger(__name__) -SCOPE = Literal['public', 'read', 'write', 'delete'] - class SessionMethods: """Core methods for API interaction""" @@ -34,18 +31,17 @@ def connected_to_default_engine(self) -> bool: return self.engine_url == self._default_engine_url @property - def _scenario_header(self) -> dict: + def _scenario_header(self) -> dict[str, Any]: """get full scenario header""" return self._get_scenario_header() @property - def engine_url(self) -> str: + def engine_url(self): """engine URL""" return self._engine_url @engine_url.setter def engine_url(self, url: str | None): - # default url if url is None: url = self._default_engine_url @@ -55,13 +51,12 @@ def engine_url(self, url: str | None): # reset token and change base url self.token = None - self.session.base_url = self._engine_url # reset cache self._reset_cache() @property - def etm_url(self) -> str: + def etm_url(self): """model URL""" # raise error @@ -72,7 +67,6 @@ def etm_url(self) -> str: @etm_url.setter def etm_url(self, url: str | None): - # use default pro location if (url is None) & (self.connected_to_default_engine): url = "https://energytransitionmodel.com/" @@ -83,17 +77,16 @@ def etm_url(self, url: str | None): @property def scenario_id(self) -> int | None: """scenario id""" - return self._scenario_id if hasattr(self, '_scenario_id') else None + return self._scenario_id if hasattr(self, "_scenario_id") else None @scenario_id.setter def scenario_id(self, scenario_id: int | None): - # store previous scenario id previous = copy.deepcopy(self.scenario_id) # try accessing dict if isinstance(scenario_id, dict): - scenario_id = scenario_id['id'] + scenario_id = scenario_id["id"] # convert passed id to integer if scenario_id is not None: @@ -113,101 +106,172 @@ def scenario_id(self, scenario_id: int | None): # validate scenario id self._get_scenario_header() + def make_endpoint_url(self, endpoint: Endpoint, extra: str = "") -> str: + """The url of the API endpoint for the connected scenario""" + + if endpoint == "curves": + # validate merit order + self._validate_merit_order() + + return self.make_endpoint_url( + endpoint="scenario_id", extra=f"curves/{extra}" + ) + + if endpoint == "custom_curves": + return self.make_endpoint_url( + endpoint="scenario_id", extra=f"custom_curves/{extra}" + ) + + if endpoint == "inputs": + return self.make_endpoint_url( + endpoint="scenario_id", extra=f"inputs/{extra}" + ) + + if endpoint == "saved_scenarios": + return self.session.make_url( + self.engine_url, url=f"saved_scenarios/{extra}" + ) + + if endpoint == "scenario_id": + # validate scenario id + self._validate_scenario_id() + + return self.session.make_url( + self.engine_url, url=f"scenarios/{self.scenario_id}/{extra}" + ) + + if endpoint == "scenarios": + return self.session.make_url(self.engine_url, "scenarios") + + if endpoint == "token": + return self.session.make_url(self.engine_url, url="/oauth/token/info") + + if endpoint == "transition_paths": + return self.session.make_url(self.engine_url, url="transition_paths") + + if endpoint == "user": + # validate token permission + self._validate_token_permission("openid") + + return self.session.make_url(self.engine_url, url="oauth/userinfo") + + raise NotImplementedError(f"endpoint not implemented: '{endpoint}'") + + def to_etm_url(self, load: bool = False): + """make url to access scenario in etm gui""" + + # raise without scenario id + self._validate_scenario_id() + + # relative path + url = f"scenarios/{self.scenario_id}" + + # append load path + if load is True: + url = f"{url}/load" + + return self.session.make_url(self.etm_url, url) + @property - def token(self) -> pd.Series | None: + def token(self): """optional personal access token for authorized use""" # return None without token if self._token is None: return None - # make request - url = '/oauth/token/info' - headers = {'content-type': 'application/json'} + # request parameters + url = self.make_endpoint_url(endpoint="token") + headers = {"content-type": "application/json"} - # get token information - resp: dict = self.session.get( - url, decoder='json', headers=headers) + # make request + token = self.session.get(url, headers=headers, content_type="application/json") # convert utc timestamps - resp['created_at'] = pd.to_datetime(resp['created_at'], unit='s') - resp['expires_in'] = pd.Timedelta(resp['expires_in'], unit='s') + token["created_at"] = pd.to_datetime(token["created_at"], unit="s") - return pd.Series(resp, name='token') + # convert experiation delta + if isinstance(token["expires_in"], int): + token["expires_in"] = pd.Timedelta(token["expires_in"], unit="s") + + return pd.Series(token, name="token") @token.setter def token(self, token: str | None = None): - # check environment variables for token if token is None: - token = os.getenv('ETM_ACCESS_TOKEN') + token = os.getenv("ETM_ACCESS_TOKEN") # store token self._token = token # update persistent session headers if self._token is None: - # pop authorization if present - if 'Authorization' in self.session.headers.keys(): - self.session.headers.pop('Authorization') + if "Authorization" in self.session.headers.keys(): + self.session.headers.pop("Authorization") else: - # set authorization - authorization = {'Authorization': f'Bearer {self._token}'} + authorization = {"Authorization": f"Bearer {self._token}"} self.session.headers.update(authorization) @property def user(self) -> pd.Series: """info about token owner if token assigned""" - # validate token permission - self._validate_token_permission('openid') + # request parameters` + url = self.make_endpoint_url(endpoint="user") + headers = {"content-type": "application/json"} # make request - url = '/oauth/userinfo' - headers = {'content-type': 'application/json'} + user = self.session.get(url, headers=headers, content_type="application/json") - # get token information - resp: dict = self.session.get( - url, decoder='json', headers=headers) - - return pd.Series(resp, name='user') + return pd.Series(user, name="user") @property - def session(self) -> RequestsSession | AIOHTTPSession: - """session object that handles requests""" - return self._session if hasattr(self, '_session') else None + def session(self) -> SessionABC: + """set object that handles requests""" + return self._session + + @session.setter + def session(self, session: SessionABC) -> None: + self._session = session @functools.lru_cache(maxsize=1) - def _get_scenario_header(self): + def _get_scenario_header(self) -> dict[str, Any]: """get header of scenario""" # return no values if self.scenario_id is None: return {} - # raise without scenario id - self._validate_scenario_id() + # request parameters + url = self.make_endpoint_url(endpoint="scenario_id") + headers = {"content-type": "application/json"} # make request - url = f'scenarios/{self.scenario_id}' - header = self.session.get(url) + header = self.session.get(url, headers=headers, content_type="application/json") return header - def _get_session_id(self, scenario_id: int) -> int: + def _get_session_id(self) -> int: """get a session_id for a pro-environment scenario""" - # extract content from url - url = urljoin(self.etm_url, f'saved_scenarios/{scenario_id}/load/') - content = self.session.request("get", url, decoder='text') + # make url + url = self.to_etm_url(load=True) + + # make request + content = self.session.get(url, content_type="text/html") - # get session id from content + # get session id from response pattern = '"api_session_id":([0-9]{6,7})' session_id = re.search(pattern, content) + # handle non-match + if session_id is None: + raise ValueError(f"Failed to scrape api_session_id from URL: '{url}'") + return int(session_id.group(1)) def _validate_scenario_id(self): @@ -215,22 +279,61 @@ def _validate_scenario_id(self): # check if scenario id is None if self.scenario_id is None: - raise ValueError('scenario id is None') + raise ValueError("scenario id is None") - def _validate_token_permission(self, scope: SCOPE = 'public'): + def _validate_token_permission(self, scope: TokenScope = "public"): """validate token permission""" # raise without token - if self._token is None: + if self.token is None: raise ValueError("No personall access token asssigned") # check if scope is known if scope is None: raise ValueError(f"Unknown token scope: '{scope}'") - if scope not in self.token.get('scope'): + # validate token scope + if scope not in self.token.loc["scope"]: raise ValueError(f"Token has no '{scope}' permission.") + @property + def merit_order_enabled(self) -> bool: + """see if merit order is enabled""" + + # target input parameter + key = "settings_enable_merit_order" + + # prepare request + headers = {"content-type": "application/json"} + url = self.make_endpoint_url(endpoint="inputs", extra=key) + + # make request + parameter = self.session.get( + url, headers=headers, content_type="application/json" + ) + + # get relevant setting + enabled = parameter.get("user", parameter["default"]) + + # check for iterpolation issues + if not (enabled == 1) | (enabled == 0): + raise ValueError(f"invalid setting: '{key}'={enabled}") + + return bool(enabled) + + # @merit_order_enabled.setter + # def merit_order_enabled(self, boolean: bool): + + # # target input parameter + # key = "settings_enable_merit_order" + + def _validate_merit_order(self): + """check if merit order is enabled""" + + # raise for disabled merit order + if self.merit_order_enabled is False: + raise ValueError(f"{self}: merit order disabled") + def _reset_cache(self): """reset cached scenario properties""" @@ -240,12 +343,9 @@ def _reset_cache(self): def _update_scenario_header(self, header: dict): """change header of scenario""" - # raise without scenario id - self._validate_scenario_id() - # set data data = {"scenario": header} - url = f'scenarios/{self.scenario_id}' + url = self.make_endpoint_url(endpoint="scenario_id") # make request self.session.put(url, json=data) diff --git a/src/pyetm/client/utils.py b/src/pyetm/client/utils.py index 7059645..5a23e84 100644 --- a/src/pyetm/client/utils.py +++ b/src/pyetm/client/utils.py @@ -1,22 +1,27 @@ """utility methods""" from __future__ import annotations -from typing import Literal +from collections.abc import Iterable + import pandas as pd from pyetm.utils import categorise_curves, regionalise_curves, regionalise_node -from .session import SessionMethods +from pyetm.types import Carrier -Carrier = Literal['electricity', 'heat', 'hydrogen', 'methane'] +from .session import SessionMethods class UtilMethods(SessionMethods): """utility methods""" - def categorise_curves(self, carrier: Carrier, - mapping: pd.DataFrame | str, columns: list[str] | None = None, - include_keys: bool = False, invert_sign: bool = False, - pattern_level: str | int | None = None, **kwargs) -> pd.DataFrame: + def categorise_curves( + self, + carrier: Carrier, + mapping: pd.Series[str] | pd.DataFrame, + columns: list[str] | None = None, + include_keys: bool = False, + invert_sign: bool = False, + ) -> pd.DataFrame: """Categorise the hourly curves for a specific carrier with a specific mapping. @@ -26,13 +31,12 @@ def categorise_curves(self, carrier: Carrier, Parameters ---------- - carrier : str or DataFrame - The carrier-name or hourly curves for which the - categorization is applied. - mapping : DataFrame or str + carrier : str + The carrier-name of the carrier on which + the categorization is applied. + mapping : DataFrame DataFrame with mapping of ETM keys in index and mapping - values in columns. Alternatively a string to a csv-file - can be passed. + values in columns. columns : list, default None List of column names and order that will be included in the mapping. Defaults to all columns in mapping. @@ -42,14 +46,6 @@ def categorise_curves(self, carrier: Carrier, Inverts sign convention where demand is denoted with a negative sign. Demand will be denoted with a positve value and supply with a negative value. - pattern_level : str or int, default None - Column level in which sign convention pattern is located. - Assumes last level by default. - - - - **kwargs arguments are passed to pd.read_csv when - a filename is passed in the mapping argument. Return ------ @@ -58,41 +54,42 @@ def categorise_curves(self, carrier: Carrier, specified carrier. """ - # fetch relevant curves - if isinstance(carrier, str): - - # make client attribute from carrier - attribute = f'hourly_{carrier}_curves' - - # fetch curves or raise error - if hasattr(self, attribute): - carrier = getattr(self, attribute) + # make client attribute from carrier + attribute = f"get_hourly_{carrier}_curves" - else: - # attribute not implemented - raise NotImplementedError(f'"{attribute}" not implemented') + # raise error + if not hasattr(self, attribute): + raise NotImplementedError(f'"{attribute}" not implemented') - if not isinstance(carrier, pd.DataFrame): - raise TypeError('carrier must be of type string or DataFrame') + # fetch curves + curves = getattr(self, attribute) # use categorization function curves = categorise_curves( - curves=carrier, mapping=mapping, columns=columns, - include_keys=include_keys, invert_sign=invert_sign, - pattern_level=pattern_level, **kwargs) + curves=curves, + mapping=mapping, + columns=columns, + include_keys=include_keys, + invert_sign=invert_sign, + ) return curves - def regionalise_curves(self, carrier, reg, node=None, - sector=None, hours=None, **kwargs): - """Return the residual power of the curves based on a - regionalisation table. The kwargs are passed to pandas.read_csv - when the regionalisation argument is a passed as a filestring. + def regionalise_curves( + self, + carrier: Carrier, + reg: pd.DataFrame, + node: str | list[str] | None = None, + sector: str | list[str] | None = None, + hours: int | list[int] | None = None, + ) -> pd.DataFrame: + """Return the residual power curves per node + based on a regionalisation table. Parameters ---------- - carrier : str or DataFrame - The carrier-name or hourly curves for which the + carrier : str + The carrier-name for which the regionalization is applied. reg : DataFrame or str Regionalization table with nodes in index and @@ -114,9 +111,8 @@ def regionalise_curves(self, carrier, reg, node=None, # fetch relevant curves if isinstance(carrier, str): - # make client attribute from carrier - attribute = f'hourly_{carrier}_curves' + attribute = f"hourly_{carrier}_curves" # fetch curves or raise error if hasattr(self, attribute): @@ -127,28 +123,32 @@ def regionalise_curves(self, carrier, reg, node=None, raise NotImplementedError(f'"{attribute}" not implemented') if not isinstance(carrier, pd.DataFrame): - raise TypeError('carrier must be of type string or DataFrame') + raise TypeError("carrier must be of type string or DataFrame") # use regionalisation function - return regionalise_curves(carrier, reg, node=node, - sector=sector, hours=hours, **kwargs) - - def regionalise_node(self, carrier, reg, node, - sector=None, hours=None, **kwargs): - + return regionalise_curves(carrier, reg, node=node, sector=sector, hours=hours) + + def regionalise_node( + self, + carrier: Carrier, + reg: pd.DataFrame, + node: str, + sector: str | list[str] | None = None, + hours: int | list[int] | None = None, + ) -> pd.DataFrame: """Return the sector profiles for a node specified in the regionalisation table. The kwargs are passed to pandas.read_csv when the regionalisation argument is a passed as a filestring. Parameters ---------- - carrier : str or DataFrame - The carrier-name or hourly curves for which the + carrier : str + The carrier-name for which the regionalization is applied. reg : DataFrame or str Regionalization table with nodes in index and sectors in columns. - node : key or list of keys + node : key Specific node in regionalisation for which the profiles are returned. sector : key or list of keys, default None @@ -165,9 +165,8 @@ def regionalise_node(self, carrier, reg, node, # fetch relevant curves if isinstance(carrier, str): - # make client attribute from carrier - attribute = f'hourly_{carrier}_curves' + attribute = f"hourly_{carrier}_curves" # fetch curves or raise error if hasattr(self, attribute): @@ -178,13 +177,14 @@ def regionalise_node(self, carrier, reg, node, raise NotImplementedError(f'"{attribute}" not implemented') if not isinstance(carrier, pd.DataFrame): - raise TypeError('carrier must be of type string or DataFrame') + raise TypeError("carrier must be of type string or DataFrame") # use regionalisation function - return regionalise_node(carrier, reg, node, - sector=sector, hours=hours, **kwargs) + return regionalise_node(carrier, reg, node, sector=sector, hours=hours) - def make_output_mapping_template(self, carriers=None): + def create_hourly_curve_mapping_template( + self, carriers: str | Iterable[str] | None = None + ): """make output mapping template""" # add string to list @@ -193,36 +193,35 @@ def make_output_mapping_template(self, carriers=None): # carrier for which columns are fetched if carriers is None: - carriers = ['electricity', 'heat', 'hydrogen', 'methane'] + carriers = ["electricity", "heat", "hydrogen", "methane"] if not isinstance(carriers, list): - raise TypeError('carriers must be of type list') + raise TypeError("carriers must be of type list") # regex mapping for product group productmap = { - '^.*[.]output [(]MW[)]$': 'supply', - '^.*[.]input [(]MW[)]$': 'demand', - 'deficit': 'supply', + "^.*[.]output [(]MW[)]$": "supply", + "^.*[.]input [(]MW[)]$": "demand", + "deficit": "supply", } def get_params(carrier): """helper for list comprehension""" # get curve columns - curve = f'hourly_{carrier}_curves' - idx = getattr(self, curve).columns + curve: pd.DataFrame = getattr(self, f"hourly_{carrier}_curves") - return pd.Series(data=carrier, index=idx, dtype='str') + return pd.Series(data=carrier, index=curve.columns, dtype="str") # make output mapping - mapping = [get_params(carrier) for carrier in carriers] - mapping = pd.concat(mapping).to_frame(name='carrier') + cols = [get_params(carrier) for carrier in carriers] + mapping = pd.concat(cols).to_frame(name="carrier") # add product columns - mapping['product'] = mapping.index.copy() - mapping['product'] = mapping['product'].replace(productmap, regex=True) + mapping["product"] = mapping.index.copy() + mapping["product"] = mapping["product"].replace(productmap, regex=True) # set index name - mapping.index.name = 'ETM_key' + mapping.index.name = "ETM_key" return mapping diff --git a/src/pyetm/exceptions.py b/src/pyetm/exceptions.py index d99baac..1f87b1d 100644 --- a/src/pyetm/exceptions.py +++ b/src/pyetm/exceptions.py @@ -1,54 +1,9 @@ """exceptions""" -import re class UnprossesableEntityError(Exception): """Unprocessable Entity Error""" -def format_share_group_error(error): - """apply more readable format to - share group errors messages""" - # find share group - pattern = re.compile("\"[a-z_]*\"") - group = pattern.findall(error)[0] - - # find group total - pattern = re.compile("\d*[.]\d*") - group_sum = pattern.findall(error)[0] - - # reformat message - group = group.replace("\"", "\'") - group = f"Share_group {group} sums to {group_sum}" - - # find parameters in group - pattern = re.compile("[a-z_]*=[0-9.]*") - items = pattern.findall(error) - - # reformat message - items = [item.replace("=", "': ") for item in items] - items = "'" + ",\n '".join(items) - - return f"""{group}\n {{{items}}}""" - -def format_error_messages(errors): - """format and handle error message""" - - # newlist - errs = [] - - # iterate over messages - for error in errors: - - # format share group errors - if "group does not balance" in error: - error = format_share_group_error(error) - - # append to list - errs.append(error) - - # make final message - base = "ETEngine returned the following error(s):" - msg = """%s\n > {}""".format("\n > ".join(errs)) %base - - return msg +class BalanceError(Exception): + """Balance Error""" diff --git a/src/pyetm/exchange/__init__.py b/src/pyetm/exchange/__init__.py deleted file mode 100644 index 065dc48..0000000 --- a/src/pyetm/exchange/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""init module""" -from .market import Market diff --git a/src/pyetm/exchange/checks.py b/src/pyetm/exchange/checks.py deleted file mode 100644 index fcece42..0000000 --- a/src/pyetm/exchange/checks.py +++ /dev/null @@ -1,265 +0,0 @@ -"""validation methods""" -from __future__ import annotations - -import numpy as np -import pandas as pd - -from pyetm.logger import get_modulelogger - -_logger = get_modulelogger(__name__) - -"""BE MORE SPECIFIC ABOUT DROPPING INTERCONNECTORS IN LOGGING""" -"""REMOVE ISLANDS BEFORE SIGNING OFF ON FINAL LIST (UNCONNECTED REGIONS)""" -"""CONSIDER CLASS LIKE SETUP AS IN ENIGMA""" - -def validate_interconnectors(interconnectors: pd.DataFrame, regions: dict): - """validate interconnectors dataframe""" - - # ensure interconnectors is dataframe - if not isinstance(interconnectors, pd.DataFrame): - interconnectors = pd.DataFrame(interconnectors) - - # rename index - interconnectors.index.name = 'key' - - # if not specified default to zero for mpi percentage - interconnectors.mpi_perc = interconnectors.mpi_perc.fillna(0) - - # dropping of invalid entries (warnings) - interconnectors = _validate_region_names(interconnectors, regions) - interconnectors = _validate_availability(interconnectors) - - # internal consistency checks (errors) - _validate_from_to_region(interconnectors) - _validate_duplicates(interconnectors) - _validate_mpi_regions(interconnectors) - _validate_mpi_percentage(interconnectors) - - return interconnectors - -def _validate_region_names(interconnectors: pd.DataFrame, - regions: dict) -> pd.DataFrame: - """validate regions have a session id""" - - # get unique regions in dataframe - keys = ['from_region', 'to_region'] - names = list(np.unique(interconnectors[keys])) - - # get region names for which no scenairo id is provided - missing = [reg for reg in names if reg not in regions.keys()] - - # warn for missing regions - for region in missing: - _logger.warning("scenario_id for '%s' missing", region) - - # specify conditions to exclude interconnectors - con1 = interconnectors['from_region'].isin(regions) - con2 = interconnectors['to_region'].isin(regions) - - # drop excluded interconnectors - return interconnectors[con1 & con2] - -def _validate_availability(interconnectors: pd.DataFrame) -> pd.DataFrame: - """check for unavaialble interconnectors""" - - # availability conditions - powered = interconnectors.p_mw > 0 - scaled = interconnectors.scaling > 0 - in_service = interconnectors.in_service - - # get interconnectors that are not available - invalid = interconnectors[~(powered & scaled & in_service)] - - # warn for unavailable interconnectors - for key in invalid.index: - _logger.warning("interconnector '%s' is not powered", key) - - return interconnectors[powered & scaled & in_service] - -def _validate_from_to_region(interconnectors: pd.DataFrame) -> None: - """validate unique from and to region""" - - # get sorted region pairs - keys = ['from_region', 'to_region'] - pairs = interconnectors[keys].apply(lambda x: sorted(x.values), axis=1) - - # get length of unique entries of pairs - pairs = pairs.apply(set).apply(len) - - # get from and to with same region - errors = pairs[pairs < 2] - - # raise for same from and to region - if not errors.empty: - raise ValueError(f"same from and to region for '{list(errors)}'") - -def _validate_duplicates(interconnectors: pd.DataFrame) -> None: - """check if there are duplicate entries""" - - # get sorted region pairs - keys = ['from_region', 'to_region'] - pairs = interconnectors[keys].apply(lambda x: sorted(x.values), axis=1) - - # get duplicate entries - errors = pairs[pairs.duplicated()] - - # raise for duplicates - if not errors.empty: - raise ValueError(f"duplicate entry/entries for '{list(errors)}'") - -def _validate_mpi_regions(interconnectors: pd.DataFrame) -> None: - """check the mpi regions""" - - # subset mpi regions with mpi percentage - interconnectors = interconnectors[interconnectors.mpi_perc < 0.0] - - # interconnectors with mpi region in from or to region - keys = ['from_region', 'to_region'] - check = interconnectors[keys].isin(interconnectors['mpi_region']) - - # get errors from check - errors = interconnectors[~check.any(axis=1)].index - - # raise for invalid results - if not errors.empty: - - # make message - msg = f"invalid mpi_region for interconnectors'{list(errors)}'" - raise ValueError(msg) - -def _validate_mpi_percentage(interconnectors: pd.DataFrame) -> None: - """check mpi percentages""" - - # check mpi percentage range [0-100]: - passed = interconnectors.mpi_perc.between(0, 100) - if not passed.all(): - cases = passed[~passed].index - - # make message - msg = f"mpi percentages of {list(cases)} outside range [0=100]" - raise ValueError(msg) - -def validate_scenario_ids(regions: dict, - interconnectors: pd.DataFrame) -> dict: - """"validate regions""" - - # dropping of invalid entries (warnings) - regions = _validate_interconnector_regions(regions, interconnectors) - - # internal consistency checks (errors) - - return regions - -def _validate_interconnector_regions(regions: dict, - interconnectors: pd.DataFrame) -> pd.DataFrame: - """validate that all regions are in interconnection""" - - # get unique regions in dataframe - keys = ['from_region', 'to_region'] - names = list(np.unique(interconnectors[keys])) - - # get region names for which no interconnector is defined - missing = [reg for reg in regions.keys() if reg not in names] - - # warn for missing regions - for region in missing: - _logger.warning("no interconnection with '%s'", region) - - # remove missing regions - regions = regions.items() - regions = {k: v for k, v in regions if k not in missing} - - return regions - -def validate_mpi_profiles(mpi_profiles: pd.DataFrame, - interconnectors: pd.DataFrame) -> pd.DataFrame: - """validate mpi profiles""" - - # transform other input - if not isinstance(mpi_profiles, pd.DataFrame): - mpi_profiles = pd.DataFrame(mpi_profiles) - - # dropping of invalid entries (warnings) - _validate_missing_profiles(mpi_profiles, interconnectors) - _validate_missing_mpi_perc(mpi_profiles, interconnectors) - - # internal consistency checks (errors) - _validate_profile_ranges(mpi_profiles) - - # create default mpi profiles - cols = interconnectors.index - profiles = pd.DataFrame(0, index=range(8760), columns=cols) - - # update default with validated - profiles.update(mpi_profiles) - - # specify condition to determine profile orient - condition = (interconnectors['from_region'] == - interconnectors['mpi_region']) - - # get and apply profile orient - orient = np.where(condition, 1, -1) - profiles = profiles * orient - - # replace negative zeros - profiles = profiles.replace(-0.0, 0) - - return profiles - -def _validate_profile_ranges(mpi_profiles: pd.DataFrame) -> None: - """validate range of profile values of mpi profiles""" - - # check mpi profile range [0-1] - passed = ((mpi_profiles >= 0) & (mpi_profiles <= 1)).all() - if not passed.all(): - - # get and raise for cases - cases = passed[~passed].index - - # make message - msg = (f"mpi profiles for {list(cases)} contain " - "values outside range [0=1]") - - raise ValueError(msg) - -def _validate_missing_profiles(mpi_profiles: pd.DataFrame, - interconnectors: pd.DataFrame) -> None: - """validate if an mpi profile present for each - interconnector with mpi percentage.""" - - # get interconnectors with mpi percentage - mpi_perc = interconnectors['mpi_perc'] - conns = mpi_perc[mpi_perc > 0].index - - # missing - missing = [c for c in conns if c not in mpi_profiles.columns] - - # raise for missing profiles - if missing: - raise ValueError(f"mpi_profile missing for '{missing}'") - -def _validate_missing_mpi_perc(mpi_profiles: pd.DataFrame, - interconnectors: pd.DataFrame) -> pd.DataFrame: - """validate missing mpi percentage for specified mpi profile""" - - # check mpi profiles without interconnector - drop = [c for c in mpi_profiles if c not in interconnectors.index] - - # warn user - for key in drop: - _logger.warning("dropped mpi_profile '%s'", key) - - # drop mpi profiles for identified cases - mpi_profiles = mpi_profiles.drop(columns=drop) - - # get interconnectors with mpi percentage - mpi_perc = interconnectors['mpi_perc'] - conns = mpi_perc[mpi_perc > 0].index - - - # missing - missing = [c for c in mpi_profiles if c not in conns] - - # raise for missing profiles - if missing: - raise ValueError(f"mpi_perc missing for '{missing}'") diff --git a/src/pyetm/exchange/market.py b/src/pyetm/exchange/market.py deleted file mode 100755 index 123c36e..0000000 --- a/src/pyetm/exchange/market.py +++ /dev/null @@ -1,869 +0,0 @@ -"""market module""" -from __future__ import annotations - -from pathlib import Path -from functools import cached_property - - -import shutil -import numpy as np -import pandas as pd - -from pyetm.logger import get_modulelogger -from pyetm.utils import lookup_coordinates - -from .region import Region -from .checks import (validate_scenario_ids, - validate_interconnectors, validate_mpi_profiles) - -logger = get_modulelogger(__name__) - - -class Market: - """Market Object""" - - @property - def name(self) -> str: - """name of model""" - return str(self.__name) - - @property - def reset(self) -> bool: - """reset on initialisation""" - return bool(self.__reset) - - @property - def wdir(self) -> Path: - """working directory""" - return self.__wdir - - @property - def scenario_ids(self) -> dict: - """dict with scenario_id per region""" - return self.__scenario_ids - - @property - def interconnectors(self) -> pd.DataFrame: - """frame with interconnectors""" - return self.__interconnectors.copy() - - @property - def _regions(self) -> list[Region]: - """list of region objects""" - return self.__regions - - @property - def regions(self) -> list[str]: - """region names of region objects""" - return [region.name for region in self._regions] - - @property - def region_urls(self) -> dict: - """region urls for pro environment""" - return {str(region): region.client.pro_url for region in self._regions} - - @property - def _interconnector_mapping(self) -> pd.DataFrame: - """mapping of interconnectors to corresponding - ETM interconnector keys""" - return self.__make_mapping__() - - @property - def interconnector_capacity(self) -> pd.Series: - """effective interconnector capacity""" - cols = ['p_mw', 'scaling', 'in_service'] - return self.interconnectors[cols].prod(axis=1) - - @property - def mpi_profiles(self) -> pd.DataFrame: - """multi purpose interconnector utilization""" - return self.__mpi_profiles - - def __init__(self, interconnectors: pd.DataFrame, - scenario_ids: dict, mpi_profiles: pd.DataFrame | None = None, - name: str = "exchange", reset: bool = True, - wdir: str | Path | None = None, **kwargs): - """initialize object""" - - # set hidden variables - self.__name = name - self.__reset = reset - self.__kwargs = kwargs - self.__wdir = Path(wdir) if wdir else Path.cwd() - - # set iterations - self.__iterations = 0 - - # log event - logger.info("initialising exchange market '%s'", self) - - # initalize dirs to store traces - self.__initialize_dirs__() - - # warn for non-reset - if not reset: - logger.critical("'%s': regions not reset on initialisation", - self) - - # validate scenario ids - scenario_ids = validate_scenario_ids( - scenario_ids, interconnectors) - - # validate interconnectors - interconnectors = validate_interconnectors( - interconnectors, scenario_ids) - - # validate scenario ids again - scenario_ids = validate_scenario_ids( - scenario_ids, interconnectors) - - # validate mpi profiles - mpi_profiles = validate_mpi_profiles( - mpi_profiles, interconnectors) - - # set parameters - self.__scenario_ids = scenario_ids - self.__interconnectors = interconnectors - self.__mpi_profiles = mpi_profiles - - # handle and set interconnectors, scenario ids and regions - self.__initialize_regions__(reset=reset) - - # cache regions and market - self._cache_regions() - - # cache expensive interconnector props - self.available_interconnector_capacity - self.interconnector_utilization - - # write first traces - if not reset: - self._update_traces() - - # log event - logger.debug("'%s': initialisation completed", self) - - @classmethod - def from_excel(cls, filepath: str, name: str | None = None, **kwargs): - """initialise market from excel""" - - # default name - if name is None: - name = Path(filepath).stem - - # read excel - with pd.ExcelFile(filepath) as reader: - - # read interconnectors - sheet, idx = 'Interconnectors', 0 - interconnectors = reader.parse(sheet, index_col=idx) - - # read scenario ids - sheet, idx = 'Sessions', [*range(4)] - scenario_ids = reader.parse(sheet, index_col=idx) - - # drop levels - for level in ['STUDY', 'SCENARIO', 'YEAR']: - if level in scenario_ids.index.names: - scenario_ids = scenario_ids.droplevel(level) - - # squeeze columns - scenario_ids = scenario_ids.squeeze('columns') - - # read mpi profiles - mpi_profiles = None - if 'MPI Profiles' in reader.sheet_names: - mpi_profiles = reader.parse('MPI Profiles') - - # initialise model - model = cls(name=name, - scenario_ids=scenario_ids, mpi_profiles=mpi_profiles, - interconnectors=interconnectors, **kwargs) - - return model - - def __repr__(self) -> str: - return f"ExchangeModel({self.name})" - - def __str__(self) -> str: - return self.name - - def __initialize_dirs__(self): - """initialize dirs where traces are stored""" - - # specify relevant paths - paths = ['prices', 'utilization', 'difference', 'consistency'] - - # iterate over paths - for path in paths: - - # construct dirpath - path = self.wdir.joinpath("{self.name}/{path}") - - # remove existing results - if path.is_dir() & self.reset: - shutil.rmtree(str(path)) - - # create new dir - path.mkdir(parents=True, exist_ok=True) - - def __initialize_regions__(self, reset: bool = True) -> None: - """initialize Region classes with correct - interconnection and scenario_id setings""" - - # new list - regions = [] - - # reference properties - imap = self._interconnector_mapping - - # reset - if reset: - - # get scaled capacity for etm keys - capacities = imap.key.map(self.interconnector_capacity) - - # iterate over scenario ids dictonairy - for region, scenario_id in self.scenario_ids.items(): - - logger.info("'%s': initialising region '%s'", self, region.upper()) - - # get interconnector names - names = 'interconnector_' + imap.other.xs(region) - - # reset - if reset: - - # get capacity from capacities frame - capacity = capacities.xs(region, level=0) - capacity.index = 'electricity_' + capacity.index + '_capacity' - - else: - - # use None - capacity = None - - # initialze a region - region = Region(region, scenario_id, - reset=reset, capacities=capacity, - interconnector_names=names, **self.__kwargs) - - # append region to self - regions.append(region) - - # set regions - self.__regions = regions - - def __make_mapping__(self) -> pd.DataFrame: - """mapping of interconnector names to ETM - interconnector keys""" - - # reference conns - conns = self.interconnectors - - # concat from and to regions - series = [conns.from_region, conns.to_region] - mapping = pd.concat(series, axis=0) - - # make midx from concated series - names = ['region', 'key'] - arrays = [mapping.values, mapping.index] - midx = pd.MultiIndex.from_arrays(arrays, names=names) - - # make series and enumerate icons - mapping = pd.Series(index=midx, dtype='string') - mapping = mapping.groupby(level=0).cumcount() - - def make_key(integer): - """helper to construct interconnector key""" - return f'interconnector_{integer}' - - # transform number in ETM key - mapping = mapping.apply(lambda x: make_key(x + 1)) - - # add information on other region - series = [conns.to_region, conns.from_region] - series = pd.concat(series, axis=0) - - # set same index as mapping - series.index = midx - - # merge mappings - arrays, keys = [mapping, series], ['ETM_key', 'other'] - mapping = pd.concat(arrays, axis=1, keys=keys) - - # join information with interconnectors - mapping = conns.join(mapping, how='inner') - mapping = mapping.reset_index(level=1).set_index('ETM_key', append=True) - - return mapping.sort_index() - - def _cache_regions(self): - """cache relevant properties in regions""" - - # iterate over regions - for region in self._regions: - - # log event and cache properties - logger.info("'%s': caching region '%s'", self, region) - region.cache_properties() - - def _get_region(self, name: str) -> Region: - """get region object for region name""" - return self._regions[self.regions.index(name)] - - def _update_traces(self): - """update traces""" - - # get exchange prices - basedir = self.wdir.joinpath(self.name) - curves = self.electricity_prices.copy() - - for col in curves.columns: - - filename = basedir.joinpath(f'prices/{col}.csv') - curves[[col]].T.to_csv(filename, index=False, mode='a', - header=False, sep=';', decimal=',') - - # get interconnector utilization - curves = self.interconnector_utilization.copy() - - for col in curves.columns: - - filename = basedir.joinpath(f'utilization/{col}.csv') - curves[[col]].T.to_csv(filename, index=False, mode='a', - header=False, sep=';', decimal=',') - - # get difference with ETM utilization - curves = self.difference() - - for col in curves.columns: - - filename = basedir.joinpath(f'difference/{col}.csv') - curves[[col]].T.to_csv(filename, index=False, mode='a', - header=False, sep=';', decimal=',') - - # get consistency between ETM utilizations in - # from and to regions. - curves = self.consistency() - - for col in curves.columns: - - filename = basedir.joinpath(f'consistency/{col}.csv') - curves[[col]].T.to_csv(filename, index=False, mode='a', - header=False, sep=';', decimal=',') - - logger.debug("'%s': updated iteration traces", self) - - def consistency(self): - """difference between etm utilization of - from and to region.""" - - difference = [] - - # reference interconnector mapping - imap = self._interconnector_mapping - - # modify mapping index - imap = imap.reset_index(level=1) - imap = imap.set_index('key', append=True) - imap = imap.ETM_key - - def utilization(region, interconnector): - """get utilization""" - - # get region and utilization - _region = self._get_region(region) - interconnector = imap[(region, interconnector)] - - return _region.etm_utilization[interconnector] - - for conn, props in self.interconnectors.iterrows(): - - # get etm utilization in both sides - frm = utilization(props.from_region, conn) - tow = utilization(props.to_region, conn) - - # evaluate difference (add as oppositve signs) - diff = frm.add(tow).replace(-0.000, 0.000) - - difference.append(pd.Series(diff, name=conn)) - - return pd.concat(difference, axis=1) - - def difference(self): - """difference between utilization and etm utilization, - returns difference based on from orient. - - use consistency check to see if there are differences - in the etm utilization between the from and to region.""" - - difference = [] - - imap = self._interconnector_mapping - - for region in self.regions: - - # get difference - _region = self._get_region(region) - diff = _region.utilization - _region.etm_utilization - - diff = diff.round(3) - diff = diff.replace(-0.000, 0.000) - - difference.append(diff) - - # convert to frame - difference = pd.concat(difference, axis=1, keys=self.regions) - - cols = imap[imap.index.get_level_values('region') == imap.from_region] - difference = difference[cols.index] - - difference.columns = difference.columns.map(imap.key) - - return difference - - @cached_property - def interconnector_utilization(self) -> pd.DataFrame: - """interconnector utilization""" - - # newlist - utilization = [] - - # reference interconnector mapping - imap = self._interconnector_mapping - - # append utilization - for region in self._regions: - utilization.append(region.utilization) - - # convert to frame - utilization = pd.concat(utilization, axis=1, keys=self.regions) - - # get correctly oriented interconnectors - cols = imap[imap.index.get_level_values('region') == imap.from_region] - utilization = utilization[cols.index] - - # reconstruct original columns - utilization.columns = utilization.columns.map(imap.key) - - return utilization - - @property - def etm_utilization(self) -> pd.DataFrame: - """etm utilization""" - - # newlist - utilization = [] - - # reference interconnector mapping - imap = self._interconnector_mapping - - # append utilization - for region in self._regions: - utilization.append(region.etm_utilization) - - # convert to frame - utilization = pd.concat(utilization, axis=1, keys=self.regions) - - # get correctly oriented interconnectors - cols = imap[imap.index.get_level_values('region') == imap.from_region] - utilization = utilization[cols.index] - - # reconstruct original columns - utilization.columns = utilization.columns.map(imap.key) - - return utilization - - @property - def price_setting_units(self) -> pd.DataFrame: - """price setting units in each region""" - - # newlist - units = [] - - # append units - for region in self._regions: - units.append(region.price_setting_unit) - - return pd.concat(units, axis=1, keys=self.regions) - - @property - def price_setting_capacities(self) -> pd.DataFrame: - """used capacity of price setting units in each region""" - - # newlist - capacities = [] - - # append capacities - for region in self._regions: - capacities.append(region.price_setting_capacity) - - return pd.concat(capacities, axis=1, keys=self.regions) - - @property - def price_setting_utilization(self) -> pd.DataFrame: - """utilization of price setting units in each region""" - - # newlist - capacities = [] - - # append capacities - for region in self._regions: - capacities.append(region.price_setting_utilization) - - return pd.concat(capacities, axis=1, keys=self.regions) - - @property - def next_dispatchable_units(self) -> pd.DataFrame: - """next dispatchable units in each region""" - - # newlist - units = [] - - # append units - for region in self._regions: - units.append(region.next_dispatchable_unit) - - return pd.concat(units, axis=1, keys=self.regions) - - @property - def next_dispatchable_prices(self) -> pd.DataFrame: - """next dispatchable unit price in each region""" - - # newlist - prices = [] - - # append prices - for region in self._regions: - prices.append(region.next_dispatchable_price) - - return pd.concat(prices, axis=1, keys=self.regions) - - @property - def next_dispatchable_capacities(self) -> pd.DataFrame: - """next dispatchable unit surplus capacity""" - - # newlist - capacities = [] - - # append capacities - for region in self._regions: - capacities.append(region.next_dispatchable_capacity) - - return pd.concat(capacities, axis=1, keys=self.regions) - - @property - def electricity_prices(self) -> pd.DataFrame: - """electricity prices in each region""" - - # newlist - prices = [] - - # append prices - for region in self._regions: - prices.append(region.electricity_price) - - # make frame - frame = pd.concat(prices, axis=1, keys=self.regions) - - return frame - - @property - def mpi_utilization(self) -> pd.DataFrame: - """get utilization of multi purpose interconnector utilization""" - return (self.interconnectors['mpi_perc'] / 100) * self.mpi_profiles - - @cached_property - def available_interconnector_capacity(self) -> pd.DataFrame: - """get remaining available capacity for the import - and export orients of the interconnector""" - - # reference properties - mpi = self.mpi_utilization - capacity = self.interconnector_capacity - utilization = self.interconnector_utilization - - # determine exchange capacity at direction - imp = (1.0 - mpi - utilization) * capacity - exp = (1.0 - mpi + utilization) * capacity - - # make table - keys = ['import', 'export'] - frame = pd.concat([imp, exp], keys=keys, axis=1) - - return frame - - @property - def interconnector_price_deltas(self) -> pd.DataFrame: - """get price deltas between the from and - to region of each interconnector""" - - # reference properties - conns = self.interconnectors - prices = self.electricity_prices - dispatch = self.next_dispatchable_prices - - # make dictonairy - frm, tow = conns.from_region, conns.to_region - mapping = dict(zip(conns.index, list(zip(frm, tow)))) - - # helper function - def price_deltas(from_region, to_region): - """Evaluated the price signals between the from and - to region of an interconnector. - - In case the price of the next dispatchable unit at the - signaled orient negates the price delta, the price delta - is set to zero. There is no wellfare effect to be realized - from exchange via the interconnector""" - - # determine price delta between each region - delta = prices[from_region] - prices[to_region] - - # inspect if price delta also there at next dispatchable unit - imprt = prices[from_region] - dispatch[to_region] - exprt = dispatch[from_region] - prices[to_region] - - # aggregate result set validity of exchange - signal = np.where(delta >= 0, imprt, exprt) - signal = abs(delta - signal) <= abs(delta) - - # apply signal - return (delta * signal).replace(-0.0, 0.0) - - # evaluate price deltas for combinations - deltas = [price_deltas(frm, tow) for frm, tow in mapping.values()] - - return pd.concat(deltas, axis=1, keys=mapping.keys()) - - @property - def utilization_duration_curves(self) -> pd.DataFrame: - """sort all interconnector utilization values""" - - # reference utilization - util = self.interconnector_utilization - - # helper function - def sort_col(col): - """helper to sort column""" - return util[col].sort_values(ascending=False, ignore_index=True) - - # sort columns - curves = [sort_col(col) for col in util.columns] - - return pd.concat(curves, axis=1) - - def set_region_curves(self, utilization: pd.DataFrame, - prices: pd.DataFrame) -> None: - """set interconnector availability and price curves""" - - # reference properties - imap = self._interconnector_mapping - - # remap availability curves to ETM keys - availability = utilization[imap.key] - availability.columns = imap.index - - # remap price curves to ETM keys - prices = prices[imap.other] - prices.columns = imap.index - - # iterate over regions - for region in self._regions: - - logger.info("'%s': updating region '%s'", self, region) - - # set availability curves - # subset and set availability for region - curves = availability.xs(region.name, axis=1, level=0) - - # determine orient based on from region - conns = imap.xs(region.name, level=0) - data = np.where(conns.from_region == region.name, 1, -1) - - # change availability based on conn orientation - orient = pd.Series(data, index=conns.index) - curves = curves.mul(orient) - - # set availability curves - region.utilization = curves - - # subset and set prices for region - curves = prices.xs(region.name, axis=1, level=0) - region.exchange_prices = curves - - @property - def iterations(self) -> int: - """number of iterations""" - return self.__iterations - - @property - def exchange_bids(self) -> pd.DataFrame: - """market information sheet - - This sheet checks if there is exchange potential between the - from and to region on each interconnector. - - This is done by looking at the import or export side of the - interconnector based on the price signal of each hour in the year. - A negative price delta signals an export orient for the - interconnector, reasoned from the from region, and a positive price - delta signals a import orient of the interconnector. - - When the capacity of the interconnector at the signaled exchange - side is not saturated, the surplus capacity is communicated. This - capacity is always set to zero in cases where the price delta between - the from and to region is zero, as there is no added wellfare to - be realised from additional exchange.""" - - # reference properties - deltas = self.interconnector_price_deltas - capacity = self.available_interconnector_capacity - utilization = self.interconnector_utilization - - # check if import signals can be serviced - scalar = capacity.xs('import', axis=1, level=0) > 0 - imp = deltas.where(deltas >= 0, np.nan).mul(scalar) - - # check if export signals can be serviced - scalar = capacity.xs('export', axis=1, level=0) > 0 - exp = deltas.where(deltas < 0, np.nan).mul(scalar) - - # merge signals - deltas = imp.fillna(exp) - deltas = deltas.replace(-0.0, 0.0) - - # get highest exchange potential - conns = deltas.abs().idxmax(axis=1) - frame = conns.to_frame(name='node') - - # lookup corresponding exchange prices and utilization - deltas = lookup_coordinates(conns, deltas) - util = lookup_coordinates(conns, utilization) - - # get region information - to_region = conns.map(self.interconnectors['to_region']) - from_region = conns.map(self.interconnectors['from_region']) - - # assign importing and exporting regions - frame['import_region'] = np.where(deltas <= 0, to_region, from_region) - frame['export_region'] = np.where(deltas <= 0, from_region, to_region) - - # assign exchange orient and utiliztion percent - frame['orient'] = np.where(deltas >= 0, 'import', 'export') - frame['network_util'] = util - - # make coords for lookup of available network capacity - coords = list(zip(frame['orient'], frame['node'])) - coords = pd.Series(coords, index=frame.index, name='orient') - - # assign available network capacity for exchange - frame['network_mw'] = lookup_coordinates(coords, - self.available_interconnector_capacity) - - # lookup production surplus at exporting region - frame['supply_mw'] = lookup_coordinates(frame['export_region'], - self.next_dispatchable_capacities) - - # lookup replacable dispatch at importing region - frame['demand_mw'] = lookup_coordinates(frame['import_region'], - self.price_setting_capacities) - - # assign price delta - frame['price_delta'] = deltas.abs().round(2) - - return frame - - @property - def exchange_results(self) -> pd.DataFrame: - """evaluate the updated interconnector utilization - based on the current utilization and wellfare potential. - - The exchanged capacity is defined as the minimum value - of the available exchange capacity, the surplus production - at the exporting region and the production at the price - setting unit at the importing country.""" - - # reference properties - exchange = self.exchange_bids - - # determine additional exchange volume - keys = ['network_mw', 'supply_mw', 'demand_mw'] - volume = exchange[keys].min(axis=1) - - # assign additional exchange volume - exchange = exchange.drop(columns=keys) - volume *= np.where(exchange['orient'] == 'import', 1, -1) - - # get full capacity - capacity = self.interconnector_capacity - capacity = exchange['node'].map(capacity) - - # add addtional availability percentage - exchange['network_util'] += volume.div(capacity) - exchange.insert(loc=4, column='exchange_mw', value=volume) - - return exchange - - @property - def updated_interconnector_utilization(self) -> pd.DataFrame: - """transform the market result into an updated - interconnector utilization result for each interconnector""" - - # reference properties - utilization = self.interconnector_utilization.copy() - exchange = self.exchange_results - - # pivot utilization percent over interconnectors - columns, values = 'node', 'network_util' - util = exchange.pivot(columns=columns, values=values) - - # update utilization tables - utilization.update(util) - - return utilization - - def clear_market(self, iterations: int = 1) -> None: - """evalute the market and update the utilization - of each region for each iteration""" - - for _ in range(int(iterations)): - self._evaluate_next_iteration() - - def _evaluate_next_iteration(self) -> None: - """evaluate next iteration of market clearing""" - - try: - - # set next iterations - self.__iterations += 1 - - # reference properties - prices = self.electricity_prices - utilization = self.updated_interconnector_utilization - - # update regions curves and cache regions - self.set_region_curves(utilization, prices) - self._cache_regions() - - # (re)cache available interconnector capacity - del self.available_interconnector_capacity - self.available_interconnector_capacity - - # (re)cache interconnector utilization - del self.interconnector_utilization - self.interconnector_utilization - - # update traces - self._update_traces() - - # log event - logger.info("completed iteration '%s' for '%s'", - self.iterations, self) - - except Exception as error: - - # log event - logger.exception("'%s': an unexpected error occured", self) - raise error diff --git a/src/pyetm/exchange/plotting.py b/src/pyetm/exchange/plotting.py deleted file mode 100644 index 287a4e8..0000000 --- a/src/pyetm/exchange/plotting.py +++ /dev/null @@ -1,392 +0,0 @@ -"""plotting methods""" -from __future__ import annotations - -import math - -from pathlib import Path -from datetime import datetime -from dateutil import parser - -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt - -from matplotlib.axes import Axes - -def _read_items(hour: int, items: list[str], wdir: str | Path | None = None): - """read items""" - - # default wdir - if wdir is None: - wdir = Path.cwd() - - def read_curve(item): - """read indivual curve""" - - filename = Path(wdir).joinpath(f'{item}.csv') - curve = pd.read_csv(filename, sep=';', decimal=',', - header=None, usecols=[hour]).squeeze('columns') - - return pd.Series(curve, name=item, dtype='float64') - - return pd.concat([read_curve(item) for item in items], axis=1) - -def plot_price_convergence(hour: int, regions: list[str], - axs: Axes | None = None, wdir: str | Path | None = None, **kwargs): - """plot price convergence over regions""" - - if wdir is None: - wdir = Path.cwd() - - filedir = Path(wdir).joinpath('prices') - - # specify subplot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # read data - data = _read_items(hour, regions, filedir) - data.plot(axs=axs, linewidth=3) - - # set legend - ncol = math.ceil(len(regions) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # axis limits - axs.set_xlim(0, len(data) - 1) - axs.set_ylim(0, None) - - # set axis labels - axs.set_title('Price convergence', size=12) - axs.set_ylabel('Price', size=10) - axs.set_xlabel('Iteration [#]', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - - return axs - -def plot_utilization_traces(hour: int, interconnectors: list[str], - axs: Axes | None = None, wdir: str | Path | None = None, **kwargs): - """plot interconnector utilization traces""" - - if wdir is None: - wdir = Path.cwd() - - filedir = Path(wdir).joinpath('utilization') - - # specify subplot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # read data - data = _read_items(hour, interconnectors, filedir) - data.plot(axs=axs, linewidth=3) - - # set legend - ncol = math.ceil(len(interconnectors) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # axis limits - axs.set_xlim(0, len(data) - 1) - axs.set_ylim(-1.1, 1.1) - - # set axis labels - axs.set_title('Interconnector utilization', size=12) - axs.set_ylabel('Utilization / Capacity', size=10) - axs.set_xlabel('Iteration [#]', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - - return axs - -def plot_prediction_error(hour: int, interconnectors: list[str], - axs: Axes | None = None, wdir: str | Path | None = None, **kwargs): - """plot model prediction error traces""" - - if wdir is None: - wdir = Path.cwd() - - filedir = Path(wdir).joinpath('difference') - - # specify subplot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # read data - data = _read_items(hour, interconnectors, filedir) - data.plot(axs=axs, linewidth=3) - - # set legend - ncol = math.ceil(len(interconnectors) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # axis limits - axs.set_xlim(0, len(data) - 1) - axs.set_ylim(-1.1, 1.1) - - # set axis labels - axs.set_title('Prediction error', size=12) - axs.set_ylabel('Model minus ETM', size=10) - axs.set_xlabel('Iteration [#]', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - - return axs - -def plot_symmetry_error(hour: int, interconnectors: list[str], - axs: Axes | None = None, wdir: str | Path | None = None, **kwargs): - """plot model symmetry error traces""" - - if wdir is None: - wdir = Path.cwd() - - filedir = Path(wdir).joinpath('consistency') - - # specify subplot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # read data - data = _read_items(hour, interconnectors, filedir) - data.plot(axs=axs, linewidth=3) - - # set legend - ncol = math.ceil(len(interconnectors) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # axis limits - axs.set_xlim(0, len(data) - 1) - axs.set_ylim(-1.1, 1.1) - - # set axis labels - axs.set_title('Symmetry error', size=12) - axs.set_ylabel('From minus to', size=10) - axs.set_xlabel('Iteration [#]', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - - return axs - -def plot_monitor(regions: list[str], interconnectors: list[str], hour: int, - wdir: str | Path | None = None): - """plot trace monitioring for hour""" - - if wdir is None: - wdir = Path.cwd() - - # make figure - _, axs = plt.subplots(4, 1, figsize=(15, 10), sharex=True) - - # plot monitored metrics - plot_price_convergence(hour, regions, axs=axs[0], wdir=wdir) - plot_utilization_traces(hour, interconnectors, axs=axs[1], wdir=wdir) - plot_prediction_error(hour, interconnectors, axs=axs[2], wdir=wdir) - plot_symmetry_error(hour, interconnectors, axs=axs[3], wdir=wdir) - - return _, axs - -def exchange_volumes_for_region(region, model, utilization=None): - """get exchange volumes for all connector regions of the - specified region""" - - # subset region relevant interconnectors - imap = model._interconnector_mapping - conns = imap.xs(region, level=0).set_index('key') - - # default to model utilization - if utilization is None: - utilization = model.interconnector_utilization - - # subset relevant interconnectors - utilization = utilization[conns.index] - capacity = model.interconnector_capacity[conns.index] - - # scale and orient utilization - orient = np.where(conns['from_region'] == region, 1, -1) - utilization *= capacity * orient - - # rename columns and sort - utilization.columns = utilization.columns.map(conns['other']) - utilization = utilization.sort_index(axis=1) - - return utilization - -def plot_weekly_exchange(region, model, utilization=None, axs=None, **kwargs): - """plot weekly exchange volumes on week basis for a region""" - - # get exchange volumes for region - utilization = exchange_volumes_for_region(region, model, utilization) - utilization = utilization.loc[:8735] - - # default plot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # configuration - periods = 8736 - origin = parser.parse( - '01-01-2029 00:00', dayfirst=True) - - # make periodindex - utilization.index = pd.period_range(start=origin, - periods=periods, freq='H', name='DateTime') - - # # aggregate utilization on weekly basis - grouper = pd.Grouper(freq='W') - utilization = utilization.groupby(grouper).sum() / 1e6 - - # plot utilization - utilization.index = utilization.index.week - utilization.plot.bar(axs=axs, stacked=True) - - # set legend - ncol = math.ceil(len(utilization.columns) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # set axis labels - axs.set_title(f'Weekly exchange volumes {region}', size=12) - axs.set_ylabel('Volume [TWh]', size=10) - axs.set_xlabel('Week', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - # axs.xaxis.set_major_formatter(mdates.DateFormatter("%m")) - - return axs - -def plot_daily_exchange(region, model, utilization=None, - from_date=None, to_date=None, axs=None, **kwargs): - """plot weekly exchange volumes on week basis for a region""" - - # get exchange volumes for region - utilization = exchange_volumes_for_region(region, model, utilization) - - # default from date - if from_date is None: - from_date = parser.parse( - '01-01-2029 00:00', dayfirst=True) - - # default to date - if to_date is None: - to_date = parser.parse( - '01-03-2029 23:00', dayfirst=True) - - # default plot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # configuration - periods = 8760 - origin = parser.parse( - '01-01-2029 00:00', dayfirst=True) - - # make periodindex - utilization.index = pd.period_range(start=origin, - periods=periods, freq='H', name='DateTime') - - # convert from date from string - if not isinstance(from_date, datetime): - from_date = parser.parse(from_date, dayfirst=True).replace( - year=origin.year, hour=0, minute=0, second=0, microsecond=0) - - # convert to date from string - if not isinstance(to_date, datetime): - to_date = parser.parse(to_date, dayfirst=True).replace( - year=origin.year, hour=23, minute=0, second=0, microsecond=0) - - # subet period - utilization = utilization.loc[from_date:to_date] - - # # aggregate utilization on weekly basis - grouper = pd.Grouper(freq='D') - utilization = utilization.groupby(grouper).sum() / 1e3 - - # plot utilization - utilization.index = utilization.index.strftime('%b %d') - utilization.plot.bar(axs=axs, stacked=True) - - # set legend - ncol = math.ceil(len(utilization.columns) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # set axis labels - axs.set_title(f'Daily exchange volumes {region}', size=12) - axs.set_ylabel('Volume [GWh]', size=10) - axs.set_xlabel('Date', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - - return axs - -def plot_hourly_exchange(region, model, utilization=None, - from_date=None, to_date=None, axs=None, **kwargs): - """plot weekly exchange volumes on week basis for a region""" - - # get exchange volumes for region - utilization = exchange_volumes_for_region(region, model, utilization) - - # default from date - if from_date is None: - from_date = parser.parse( - '01-01-2029 00:00', dayfirst=True) - - # default to date - if to_date is None: - to_date = parser.parse( - '03-01-2029 23:00', dayfirst=True) - - # default plot - if axs is None: - _, axs = plt.subplots(**kwargs) - - # configuration - periods = 8760 - origin = parser.parse( - '01-01-2029 00:00', dayfirst=True) - - # make periodindex - utilization.index = pd.period_range(start=origin, - periods=periods, freq='H', name='DateTime') - - # convert from date from string - if not isinstance(from_date, datetime): - from_date = parser.parse(from_date, dayfirst=True).replace( - year=origin.year, minute=0, second=0, microsecond=0) - - # convert to date from string - if not isinstance(to_date, datetime): - to_date = parser.parse(to_date, dayfirst=True).replace( - year=origin.year, minute=0, second=0, microsecond=0) - - # subet period - utilization = utilization.loc[from_date:to_date] - - # plot utilization - utilization.index = utilization.index.strftime('%e %b - %H:%M') - utilization.plot.bar(axs=axs, stacked=True) - - # set legend - ncol = math.ceil(len(utilization.columns) / 5) - axs.legend(loc='center left', title='Legend', ncol=ncol, - fontsize=11, bbox_to_anchor=(1.01, 0.5)) - - # set axis labels - axs.set_title(f'Hourly exchange volumes {region}', size=12) - axs.set_ylabel('Volume [MWh]', size=10) - axs.set_xlabel('Date', size=10) - - # set ticks - axs.tick_params(axis='both', which='major', labelsize=10) - - return axs diff --git a/src/pyetm/exchange/region.py b/src/pyetm/exchange/region.py deleted file mode 100644 index bc34280..0000000 --- a/src/pyetm/exchange/region.py +++ /dev/null @@ -1,513 +0,0 @@ -"""region module""" -from typing import Optional - -import numpy as np -import pandas as pd - -from pyetm import Client -from pyetm.logger import get_modulelogger -from pyetm.utils.lookup import lookup_coordinates - -logger = get_modulelogger(__name__) - - -class Region: - """wrapper around pyETM clients. Is unaware of mappings for - interconnector keys and only uses ETM keys.""" - - @property - def name(self) -> str: - """region name""" - return self.__name - - @property - def scenario_id(self) -> str: - """scenario id for region""" - return self.client.scenario_id - - @property - def client(self) -> Client: - """client that connects to ETM""" - return self.__client - - @property - def interconnector_names(self) -> pd.Series: - """names of interconnectors""" - return self.__interconnector_names - - @property - def utilization(self) -> pd.DataFrame: - - # create pattern for relevant keys - pat1 = 'interconnector_\d{1,2}_export_availability' - pat2 = 'interconnector_\d{1,2}_import_availability' - pattern = '%s|%s' %(pat1, pat2) - - # get attached keys - keys = self.client.get_custom_curve_keys(False) - - # subset relevant keys - keys = keys[keys.str.match(pattern)] - availability = self.custom_curves[keys] - - # base pattern - base = 'interconnector_\d{1,2}' - cols = availability.columns - - # subset relevant keys - suffix = '_import_availability' - imprt = availability[cols[cols.str.match(base + suffix)]] - imprt.columns = imprt.columns.str.rstrip(suffix) - - # subset relevant keys - suffix = '_export_availability' - exprt = availability[cols[cols.str.match(base + suffix)]] - exprt.columns = exprt.columns.str.rstrip(suffix) - - # get utilization in sparse format - utilization = imprt.sub(exprt) - - return utilization - - @property - def etm_utilization(self) -> pd.DataFrame: - - # reference user values - uvalues = self.client.user_values - - # match all interconnectors - pattern = "electricity_(interconnector_\d{1,2})_capacity" - uvalues = uvalues[uvalues.index.str.match(pattern)] - - # subset all active interconnectors - uvalues = uvalues[uvalues > 0] - uvalues.index = uvalues.index.str.extract(pattern, expand=False) - - # convert dtype - uvalues = uvalues.astype('float64') - - # reference hourly ecurves - curves = self.hourly_electricity_curves - - # helper function - def process_conn(conn): - """helper to fetch residual profile""" - - # specify keys - imprt = "energy_%s_imported_electricity.output (MW)" %conn - exprt = "energy_%s_exported_electricity.input (MW)" %conn - - # get residual curve - curve = curves[imprt] - curves[exprt] - - return pd.Series(curve, name=conn, dtype='float64') - - # get interconnector utilization profile for ETM curves - curves = [process_conn(conn) for conn in uvalues.index] - - return pd.concat(curves, axis=1) / uvalues - - @utilization.setter - def utilization(self, utilization: pd.DataFrame) -> None: - """utilization setter""" - - # import availability - imprt = utilization.where(utilization > 0, 0) - imprt.columns += '_import_availability' - - # export availability - exprt = utilization.where(utilization < 0, 0).abs() - exprt.columns += '_export_availability' - - # merge and sort availability - availability = pd.concat([imprt, exprt], axis=1) - - # specify curve names - names = self.interconnector_names - names = pd.concat([names, names]).to_list() - - # set interconnector availability - self.client.upload_custom_curves(availability, names=names) - - logger.debug("'%s': uploaded availability curves", self) - - @property - def exchange_prices(self) -> pd.DataFrame: - """interconnector price curves""" - - # create pattern for relevant keys - pattern = 'interconnector_\d{1,2}_price' - - # get attached keys and subset prices - keys = self.client.get_custom_curve_keys(False) - keys = keys[keys.str.match(pattern)] - - # get and format price curves - prices = self.custom_curves[keys] - prices.columns = prices.columns.str.rstrip('_price') - - return prices - - @property - def custom_curves(self) -> pd.DataFrame: - """attached custom curves""" - - # check for cached frames - cache = self.client.get_custom_curves.cache_info() - ccurves = self.client.custom_curves - - if cache.hits == 0: - logger.debug("'%s': downloaded attached ccurves", self) - - return ccurves - - @exchange_prices.setter - def exchange_prices(self, prices) -> None: - """interconnector price curves""" - - # make key and upload curves - prices.columns += '_price' - names = self.interconnector_names.to_list() - - # set price curves - self.client.upload_custom_curves(prices, names=names) - - logger.debug("'%s': uploaded exchange prices", self) - - @property - def exchange_capacity(self) -> pd.Series: - """set capacity for each interconnector""" - - # get pattern and uvalues from client - pattern = 'electricity_interconnector_\d{1,2}_capacity' - uvalues = self.client.scenario_parameters - - return uvalues[uvalues.index.str.match(pattern)] - - @property - def hourly_electricity_curves(self) -> pd.DataFrame: - """hourly electricity prices curves""" - - # check for cached frames - cache = self.client.get_hourly_electricity_curves.cache_info() - curves = self.client.hourly_electricity_curves - - if cache.hits == 0: - logger.debug("'%s': downloaded electricity curves", self) - - return curves - - @property - def bidladder(self) -> pd.DataFrame: - """cache bidladder over all iterations""" - return self.__bidladder - - @property - def electricity_price(self) -> pd.Series: - """hourly electricity price curve""" - - # check for cached frames - cache = self.client.get_hourly_electricity_price_curve.cache_info() - prices = self.client.hourly_electricity_price_curve - - if cache.hits == 0: - logger.debug("'%s': downloaded electricity prices", self) - - return prices - - @property - def price_setting_unit(self) -> pd.Series: - """name of price setting unit""" - - # get dispatchable utilization - util = self.dispatchable_utilization.copy() - - # invalidate zeros and reverse order - util = util.replace(0.00, np.nan) - util = util[util.columns[::-1]] - - # get price setting unit - unit = util.idxmin(axis=1) - unit = unit.str.rstrip('.output (MW)') - - return pd.Series(unit, name='unit', dtype='str') - - @property - def price_setting_capacity(self) -> pd.Series: - """used capacity of price setting unit""" - - # get units and dispatch - units = self.price_setting_unit.copy() - ecurves = self.hourly_electricity_curves - - # get dispatched volume - units = units + '.output (MW)' - utilized = lookup_coordinates(units, ecurves).round(2) - - return pd.Series(utilized, name='capacity', dtype='float64') - - @property - def price_setting_utilization(self) -> pd.Series: - """utilization of price setting unit""" - - # get units and utilization - units = self.price_setting_unit.copy() - utilization = self.dispatchable_utilization - - # lookup utilization - util = lookup_coordinates(units, utilization) - - return pd.Series(util, name='utilization', dtype='float64') - - @property - def next_dispatchable_unit(self) -> pd.Series: - """hourly unit name for next dispatchable unit""" - - # get units and match hourly curve format - ladder = self.bidladder.copy() - ladder.index = ladder.index + '.output (MW)' - - # get electricity prices and subset relevant curves - ecurves = self.hourly_electricity_curves - ecurves = ecurves[ladder.index] - - # find standby unit based on utilization - # round to prevent merit/python rounding errors - util = ecurves.div(ladder.capacity).round(2) - default = util.columns[-1] - - def return_position(utilization): - """helper to get index of dispatchable""" - return next((idx for idx, value in utilization.items() - if value < 1), default) - - # get index of next dispatchable unit - unit = util.apply(lambda row: return_position(row), axis=1) - unit = unit.str.rstrip('.output (MW)') - - return pd.Series(unit, name='unit', dtype='str') - - @property - def next_dispatchable_price(self) -> pd.Series: - """hourly price curve for next dispatchable unit""" - - # evaluate prices - units = self.next_dispatchable_unit - prices = units.map(self.bidladder.marginal_costs) - - return pd.Series(prices, name='price', dtype='float64') - - @property - def next_dispatchable_capacity(self) -> pd.Series: - """hourly surplus capacity of next dispatchable unit""" - - # get bidladder - ladder = self.bidladder.copy() - ladder.index = ladder.index + '.output (MW)' - - # get hourly electricity curves - ecurves = self.hourly_electricity_curves - ecurves = ecurves[ladder.index] - - # get units - units = self.next_dispatchable_unit.copy() - units = units + '.output (MW)' - - # get capacity and dispatch - capacity = units.map(ladder.capacity).round(2) - utilized = lookup_coordinates(units, ecurves).round(2) - - # evaluate capacity - capacity = capacity - utilized - - return pd.Series(capacity, name='capacity', dtype='float64') - - @property - def dispatchable_utilization(self) -> pd.DataFrame: - - # get bidladder - ladder = self.bidladder.copy() - ladder.index = ladder.index + '.output (MW)' - - # get hourly electricity curves - ecurves = self.hourly_electricity_curves - ecurves = ecurves[ladder.index] - - # evaluate capacity and strip suffix - ecurves = ecurves / ladder.capacity - ecurves.columns = ecurves.columns.str.rstrip('.output (MW)') - - return ecurves.round(2) - - # @property - # def surplus_bidlevel(self) -> pd.Series: - # """reached bidlevel of priceladder at each hour of the year. - # The reached bidlevel is based on the hourly electricity prices.""" - - # # get dispatchables and match hourly curve format - # ladder = self.bidladder.copy() - # ladder.index = ladder.index + '.output (MW)' - - # # get hourly electricity price curve - # prices = self.electricity_price - - # # reference vars for helper function - # mcosts = ladder.marginal_costs - # default = mcosts.index[-1] - - # def return_position(price): - # """helper to get index of dispatchable""" - # return next((idx for idx, value in mcosts.items() - # if price <= value), default) - - # # get index of active bidladder at eprice - # bidlevel = prices.apply(return_position) - # bidlevel = bidlevel.str.rstrip('.output (MW)') - - # return pd.Series(bidlevel, name='bidlevel', dtype='str') - - # @property - # def surplus_capacity(self) -> pd.Series: - # """"capacity surplus at each hour. Based on the reached - # bidlevel for the electricity price in each hour.""" - - # # get bidladder - # ladder = self.bidladder.copy() - # ladder.index = ladder.index + '.output (MW)' - - # # get hourly electricity curves - # curves = self.hourly_electricity_curves - - # # determine dispatchable utilization - # # correction for rounding errors in merit - # curves = curves[ladder.index] - # util = curves.sum(axis=1).round(2) - - # # get bidlevel - # bidlevel = self.surplus_bidlevel - # bidlevel = bidlevel + '.output (MW)' - - # # determine capacity - # # correction for rounding erros - # capacity = ladder.capacity.cumsum() - # capacity = bidlevel.map(capacity).round(2) - - # return capacity - util - - def __init__(self, name: str, scenario_id = str, - reset: bool = True, capacities: Optional[pd.Series] = None, - interconnector_names: Optional[pd.Series] = None, **kwargs) -> None: - """kwargs used for client, be aware that the proxies - argument changes to proxy when enabling a queue.""" - - # set properties - self.__name = name - self.__client = Client(scenario_id, **kwargs) - self.__interconnector_names = interconnector_names - - # reset interconnectors - if reset: - self.__reset_interconnectors(capacities) - - # cache bidladder in region - self.__bidladder = self.client.get_dispatchables_bidladder() - logger.debug("'%s': downloaded bidladder", self) - - def __repr__(self) -> str: - """reproduction string""" - return "Region(%s, %s)" %(self.name, self.scenario_id) - - def __str__(self) -> str: - """string name""" - return self.name.upper() - - def cache_properties(self) -> None: - """ensure all variables are cached""" - - # call relevant variables - self.utilization - self.electricity_price - self.hourly_electricity_curves - - def __reset_interconnectors(self, capacities: pd.Series) -> None: - """sets capacities of specified interconnectors and - disables all other interconnectors. All availability - and price related custom curves are removed from all - interconnectors. - - capacities : pd.Series - interconnector name ('interconnector_x') in index - and capacity of interconnector as value.""" - - # drop capacities with non-positive capacities - capacities = capacities[capacities > 0] - - # get uvalues for client - uvalues = self.client.scenario_parameters - - # subset interconnectors - pattern = 'electricity_interconnector_\d{1,2}_capacity' - conns = uvalues[uvalues.index.str.match(pattern)] - - # set interconnection - # capacity to zero - conns.loc[:] = 0 - conns.name = 'ETM_key' - - # check if new capacities need to be placed - if capacities is not None: - - # update capacities - conns.update(capacities) - self.client.user_values = conns - - logger.debug("'%s': uploaded capacity settings", self) - - # create pattern for relevant keys - pat1 = 'interconnector_\d{1,2}_export_availability' - pat2 = 'interconnector_\d{1,2}_import_availability' - pattern = '%s|%s' %(pat1, pat2) - - # get all keys and subset availability keys - keys = self.client.get_custom_curve_keys(True) - keys = keys[keys.str.match(pattern)] - - # subset interconnectors that are set - conns = range(1, len(capacities) + 1) - pattern = [f"interconnector_{nr}_" for nr in conns] - - # check pattern - pattern = "|".join(pattern) - columns = keys[keys.str.contains(pattern, regex=True)] - - # get interconnectors - pattern = '(interconnector_\d{1,2})' - ccolumns = columns.str.extract(pattern, expand=False).unique() - - # replace existing keys - utilization = pd.DataFrame(0, index=range(8760), columns=ccolumns) - self.utilization = utilization - - logger.debug("'%s': uploaded initial ccurves", self) - - else: - - logger.debug("'%s': all capacities set to zero", self) - - # get custom curve keys without unattached - keys = self.client.get_custom_curve_keys(False) - - # drop non interconnector related keys - pattern = 'interconnector_\d{1,2}_' - keys = keys[keys.str.contains(pattern)] - - # drop keys for configured profiles - if capacities is not None: - keys = keys[~keys.isin(columns)] - - # unattach keys - if not keys.empty: - - # delete custom curves - self.client.delete_custom_curves(keys=keys) - logger.debug("'%s': deleted superfluous ccurves", self) diff --git a/src/pyetm/logger.py b/src/pyetm/logger.py index 109aa6b..297a0ad 100644 --- a/src/pyetm/logger.py +++ b/src/pyetm/logger.py @@ -2,13 +2,14 @@ from __future__ import annotations import copy -import shutil import logging +import shutil -from datetime import datetime +from os import PathLike from pathlib import Path -def find_dirpath(dirname: str, dirpath: str) -> Path: + +def find_dirpath(dirname: str | PathLike, dirpath: str | PathLike) -> Path: """reduce dirpath until dirname in path found""" # convert to Path @@ -20,10 +21,8 @@ def find_dirpath(dirname: str, dirpath: str) -> Path: # iterate over dirpath until basename matched dirname while mdirpath.stem != dirname: - # limit number of recursions if recursions >= max_recursions: - # make message msg = f"Could not find '{dirname} in '{dirpath}'" @@ -35,11 +34,12 @@ def find_dirpath(dirname: str, dirpath: str) -> Path: return mdirpath -def _create_mainlogger(logdir) -> None: + +def _create_mainlogger(logdir) -> logging.Logger: """create mainlogger""" # make logdir - logdir = Path(logdir).joinpath('logs') + logdir = Path(logdir).joinpath("logs") Path.mkdir(logdir, exist_ok=True) # get rootlogger @@ -47,13 +47,13 @@ def _create_mainlogger(logdir) -> None: logger.setLevel(logging.DEBUG) # create formatter - fmt = '%(asctime)s | %(levelname)s | %(name)s | %(message)s' - datefmt = '%Y-%m-%d %H:%M' + fmt = "%(asctime)s | %(levelname)s | %(name)s | %(message)s" + datefmt = "%Y-%m-%d %H:%M" formatter = logging.Formatter(fmt, datefmt=datefmt) # create file handler - filepath = logdir.joinpath(PACKAGENAME + '.log') - file_handler = logging.FileHandler(filepath, mode='w+') + filepath = logdir.joinpath(PACKAGENAME + ".log") + file_handler = logging.FileHandler(filepath, mode="w+") file_handler.setFormatter(formatter) file_handler.setLevel(logging.DEBUG) @@ -68,6 +68,7 @@ def _create_mainlogger(logdir) -> None: return logger + def get_modulelogger(name: str) -> logging.Logger: """get instance of modulelogger""" @@ -77,7 +78,8 @@ def get_modulelogger(name: str) -> logging.Logger: return _moduleloggers[name] -def export_logfile(dst: str | None = None) -> None: + +def export_logfile(dst: str | PathLike | None = None) -> None: """Export logfile to targetfolder, defaults to current working directory.""" @@ -88,32 +90,34 @@ def export_logfile(dst: str | None = None) -> None: # export file shutil.copyfile(LOGDIR, dst) -def log_exception(exc: Exception, - logger: logging.Logger | None = None) -> None: - """report error message and export logs""" - # default logger - if logger is None: - logger = get_modulelogger(__name__) +# def log_exception(exc: Exception, logger: logging.Logger | None = None +# ) -> None: +# """report error message and export logs""" + +# # default logger +# if logger is None: +# logger = get_modulelogger(__name__) + +# # get current time +# now = datetime.now() +# now = now.strftime("%Y%m%d%H%M") - # get current time - now = datetime.now() - now = now.strftime("%Y%m%d%H%M") +# # make filepath +# filepath = Path.cwd().joinpath(now + ".log") - # make filepath - filepath = Path.cwd().joinpath(now + '.log') +# # log exception as error +# logger.error("Encountered error: exported logs to '%s'", filepath) +# logger.debug("Traceback for encountered error:", exc_info=True) - # log exception as error - logger.error("Encountered error: exported logs to '%s'", filepath) - logger.debug("Traceback for encountered error:", exc_info=True) +# # export logfile +# export_logfile(filepath) - # export logfile - export_logfile(filepath) +# raise exc - raise exc # package globals -PACKAGENAME = 'pyetm' +PACKAGENAME = "pyetm" PACKAGEPATH = find_dirpath(PACKAGENAME, __file__) # logger globals diff --git a/src/pyetm/myc/__init__.py b/src/pyetm/myc/__init__.py index 9e47342..52198de 100644 --- a/src/pyetm/myc/__init__.py +++ b/src/pyetm/myc/__init__.py @@ -1,2 +1,4 @@ """init myc module""" from .model import MYCClient + +__all__ = ["MYCClient"] diff --git a/src/pyetm/myc/model.py b/src/pyetm/myc/model.py index 4c4e8c3..f075e3f 100644 --- a/src/pyetm/myc/model.py +++ b/src/pyetm/myc/model.py @@ -1,49 +1,75 @@ """myc access""" from __future__ import annotations -from typing import TYPE_CHECKING, Literal - import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Literal, get_args, Iterable + import numpy as np import pandas as pd from pyetm import Client +from pyetm.logger import get_modulelogger +from pyetm.optional import import_optional_dependency from pyetm.utils import categorise_curves +from pyetm.utils.excel import add_frame, add_series from pyetm.utils.url import make_myc_url, set_url_parameters -from pyetm.logger import get_modulelogger, log_exception -from pyetm.optional import import_optional_dependency _logger = get_modulelogger(__name__) -Carrier = Literal['electricity', 'heat', 'hydrogen', 'methane'] - """externalize hard coded ETM parameters""" +if TYPE_CHECKING: + # import xlswriter + pass + +Carrier = Literal["electricity", "heat", "hydrogen", "methane"] + + +def check_carriers(carriers: Carrier | Iterable[Carrier] | None): + """check carrier""" + + # default carriers + if carriers is None: + carriers = get_args(Carrier) + + # add string to list + if isinstance(carriers, str): + carriers = [carriers] + + # check passed carriers + for carrier in carriers: + if carrier.lower() not in get_args(Carrier): + raise ValueError("carrier '{carrier}' not supported") + + # ensure lower cased carrier names + carrier = [carrier.lower() for carrier in carriers] + + return carrier + + def sort_frame( - frame: pd.DataFrame, - axis: int = 0, - reference: str | None = None + frame: pd.DataFrame, axis: int = 0, reference: str | None = None ) -> pd.DataFrame: """sort frame with reference scenario at start position""" # sort columns and get scenarios frame = frame.sort_index(axis=axis) - scenarios = frame.axes[axis].get_level_values(level='SCENARIO') + scenarios = frame.axes[axis].get_level_values(level="SCENARIO") # handle reference scenario if (reference is not None) & (reference in scenarios): - # subset reference from frame - ref = frame.xs(reference, level='SCENARIO', axis=axis, - drop_level=False) + ref = frame.xs(reference, level="SCENARIO", axis=axis, drop_level=False) # drop reference from frame and apply custom order - frame = frame.drop(reference, level='SCENARIO', axis=axis) + frame = frame.drop(reference, level="SCENARIO", axis=axis) frame = pd.concat([ref, frame], axis=axis) return frame -class MYCClient(): + +class MYCClient: """Multi Year Chart Client""" @property @@ -53,9 +79,7 @@ def myc_url(self) -> str: @myc_url.setter def myc_url(self, url: str | None): - if url is None: - # check for default engine with Client(**self._kwargs) as client: default_engine = client.connected_to_default_engine @@ -66,82 +90,80 @@ def myc_url(self, url: str | None): else: # raise for missing myc URL - raise ValueError("must specify the related " - "custom myc_url for the specified custom engine_url.") + raise ValueError( + "must specify the related " + "custom myc_url for the specified custom engine_url." + ) self._myc_url = str(url) @property def session_ids(self) -> pd.Series: """series of scenario_ids""" - return self.__session_ids + return self._session_ids @session_ids.setter def session_ids(self, session_ids: pd.Series): - # convert to series if not isinstance(session_ids, pd.Series): session_ids = pd.Series(session_ids) # set uniform index names - keys = ['STUDY', 'SCENARIO', 'REGION', 'YEAR'] + keys = ["STUDY", "SCENARIO", "REGION", "YEAR"] session_ids.index.names = keys # set session ids - self.__session_ids = session_ids + self._session_ids = session_ids # revalidate reference scenario - if hasattr(self, '_reference'): + if hasattr(self, "_reference"): self.reference = self.reference @property def parameters(self) -> pd.Series: """parameters""" - return self.__parameters + return self._parameters @parameters.setter def parameters(self, parameters: pd.Series): - # convert to series if not isinstance(parameters, pd.Series): parameters = pd.Series(parameters) # set parameters - self.__parameters = parameters + self._parameters = parameters @property def gqueries(self) -> pd.Series: """gqueries""" - return self.__gqueries + return self._gqueries @gqueries.setter def gqueries(self, gqueries: pd.Series): - # convert to series if not isinstance(gqueries, pd.Series): gqueries = pd.Series(gqueries) # set gqueries - self.__gqueries = gqueries + self._gqueries = gqueries @property - def mapping(self) -> pd.DataFrame: + def mapping(self) -> pd.DataFrame | None: """mapping""" - return self.__mapping + return self._mapping @mapping.setter - def mapping(self, mapping: pd.DataFrame | None): - + def mapping(self, mapping: pd.Series | pd.DataFrame | None): # convert to dataframe if (not isinstance(mapping, pd.DataFrame)) & (mapping is not None): mapping = pd.DataFrame(mapping) # set default index names if isinstance(mapping, pd.DataFrame): - mapping.index.names = ['KEY', 'CARRIER'] + mapping.index.names = ["KEY", "CARRIER"] # set mapping - self.__mapping = mapping + self._mapping = mapping @property def depricated(self) -> list[str]: @@ -149,20 +171,20 @@ def depricated(self) -> list[str]: # subset depricated parameters depricated = [ - 'areable_land', - 'buildings_roof_surface_available_for_pv', - 'co2_emission_1990', - 'co2_emission_1990_aviation_bunkers', - 'co2_emission_1990_marine_bunkers', - 'coast_line', - 'number_of_buildings_both', - 'number_of_buildings_present', - 'number_of_inhabitants', - 'number_of_inhabitants_present', - 'number_of_residences', - 'offshore_suitable_for_wind', - 'residences_roof_surface_available_for_pv', - ] + "areable_land", + "buildings_roof_surface_available_for_pv", + "co2_emission_1990", + "co2_emission_1990_aviation_bunkers", + "co2_emission_1990_marine_bunkers", + "coast_line", + "number_of_buildings_both", + "number_of_buildings_present", + "number_of_inhabitants", + "number_of_inhabitants_present", + "number_of_residences", + "offshore_suitable_for_wind", + "residences_roof_surface_available_for_pv", + ] return depricated @@ -172,30 +194,13 @@ def excluded(self) -> list[str]: # list of excluded parameters for TYNDP excluded = [ - # hidden parameter for HOLON project - 'holon_gas_households_useful_demand_heat_per_person_absolute', - - # issues with regard to min/max ranges and non-demand units - 'efficiency_industry_chp_combined_cycle_gas_power_fuelmix_electricity', - 'efficiency_industry_chp_combined_cycle_gas_power_fuelmix_heat', - 'efficiency_industry_chp_engine_gas_power_fuelmix_electricity', - 'efficiency_industry_chp_engine_gas_power_fuelmix_heat', - 'efficiency_industry_chp_turbine_gas_power_fuelmix_electricity', - 'efficiency_industry_chp_turbine_gas_power_fuelmix_heat', - 'efficiency_industry_chp_ultra_supercritical_coal_electricity', - 'efficiency_industry_chp_ultra_supercritical_coal_heat', - 'efficiency_industry_chp_wood_pellets_electricity', - 'efficiency_industry_chp_wood_pellets_heat', - + "holon_gas_households_useful_demand_heat_per_person_absolute", # set flh parameters instead - 'flh_of_energy_power_wind_turbine_coastal_user_curve', - 'flh_of_energy_power_wind_turbine_inland_user_curve', - 'flh_of_energy_power_wind_turbine_offshore_user_curve', - 'flh_of_solar_pv_solar_radiation_user_curve', - - # breaking paramter for scenarios - 'capacity_of_industry_metal_flexibility_load_shifting_electricity', + "flh_of_energy_power_wind_turbine_coastal_user_curve", + "flh_of_energy_power_wind_turbine_inland_user_curve", + "flh_of_energy_power_wind_turbine_offshore_user_curve", + "flh_of_solar_pv_solar_radiation_user_curve", ] return excluded @@ -207,9 +212,8 @@ def reference(self) -> str | None: @reference.setter def reference(self, reference: str | None): - # validate reference key - scenarios = self.session_ids.index.unique(level='SCENARIO') + scenarios = self.session_ids.index.unique(level="SCENARIO") if (reference not in scenarios) & (reference is not None): raise KeyError(f"Invalid reference key: '{reference}'") @@ -219,11 +223,12 @@ def reference(self, reference: str | None): def __init__( self, session_ids: pd.Series, - parameters: pd.Series, gqueries: pd.Series, + parameters: pd.Series, + gqueries: pd.Series, mapping: pd.Series | pd.DataFrame | None = None, reference: str | None = None, myc_url: str | None = None, - **kwargs + **kwargs, ): """initialisation logic for Client. @@ -275,6 +280,7 @@ def __init__( If tuple; ('cert', 'key') pair.""" # set kwargs + self._source = None self._kwargs = kwargs # set optional parameters @@ -291,7 +297,7 @@ def from_excel( filepath: str, reference: str | None = None, myc_url: str | None = None, - **kwargs + **kwargs, ): """initate from excel file with standard structure @@ -315,26 +321,28 @@ def from_excel( xlsx = pd.ExcelFile(filepath) # get session ids - session_ids = pd.read_excel(xlsx, sheet_name='Sessions', - usecols=[*range(5)], index_col=[*range(4)]).squeeze('columns') + session_ids = pd.read_excel( + xlsx, sheet_name="Sessions", usecols=[*range(5)], index_col=[*range(4)] + ).squeeze("columns") # get paramters - parameters = pd.read_excel(xlsx, sheet_name='Parameters', - usecols=[*range(2)], index_col=[*range(1)]).squeeze('columns') + parameters = pd.read_excel( + xlsx, sheet_name="Parameters", usecols=[*range(2)], index_col=[*range(1)] + ).squeeze("columns") # get gqueries - gqueries = pd.read_excel(xlsx, sheet_name='GQueries', - usecols=[*range(2)], index_col=[*range(1)]).squeeze('columns') + gqueries = pd.read_excel( + xlsx, sheet_name="GQueries", usecols=[*range(2)], index_col=[*range(1)] + ).squeeze("columns") # check for optional mapping if "Mapping" in xlsx.sheet_names: - # load mapping - mapping = pd.read_excel(filepath, sheet_name='Mapping', - index_col=[*range(2)]) + mapping = pd.read_excel( + filepath, sheet_name="Mapping", index_col=[*range(2)] + ) else: - # default mapping mapping = None @@ -346,7 +354,7 @@ def from_excel( mapping=mapping, reference=reference, myc_url=myc_url, - **kwargs + **kwargs, ) # set source in model @@ -354,14 +362,12 @@ def from_excel( return model - def _check_for_unmapped_input_parameters(self, - client: Client) -> pd.Index: - + def _check_for_unmapped_input_parameters(self, client: Client) -> pd.Index: # get parameters from client side - parameters = client.scenario_parameters + parameters = client.get_input_parameters() # subset external coupling nodes - key = 'external_coupling' + key = "external_coupling" subset = parameters.index.str.contains(key, regex=True) # drop external coupling nodex @@ -374,12 +380,11 @@ def _check_for_unmapped_input_parameters(self, # keys not in parameters but in parameters missing = parameters[~parameters.index.isin(self.parameters.index)] - missing = pd.Index(missing.index, name='unmapped keys') + missing = pd.Index(missing.index, name="unmapped keys") return missing - def _make_midx(self, - midx: tuple | pd.MultiIndex | None = None) -> pd.MultiIndex: + def _make_midx(self, midx: tuple | pd.MultiIndex | None = None) -> pd.MultiIndex: """helper to handle passed multiindex""" # default to all @@ -397,7 +402,8 @@ def _make_midx(self, return midx def get_input_parameters( - self, midx: tuple | pd.MultiIndex | None = None) -> pd.DataFrame: + self, midx: tuple | pd.MultiIndex | None = None + ) -> pd.DataFrame: """get input parameters for single scenario""" # subset cases of interest @@ -406,61 +412,53 @@ def get_input_parameters( _logger.info("collecting input parameters") - try: - - # make client with context manager - with Client(**self._kwargs) as client: - - # newlist - values = [] - - # connect scenario and check for unmapped keys - client.gqueries = list(self.gqueries.index) - - # newset - warned = False - unmapped = set() + # make client with context manager + with Client(**self._kwargs) as client: + # newlist + values = [] - # get parameter settings - for case, scenario_id in cases.items(): + # connect scenario and check for unmapped keys + client.gqueries = list(self.gqueries.index) - # log event - _logger.debug("> collecting inputs for " + - "'%s', '%s', '%s', '%s'", *case) + # newset + warned = False + unmapped = set() - # connect scenario and check for unmapped keys - client.scenario_id = scenario_id - missing = self._check_for_unmapped_input_parameters(client) + # get parameter settings + for case, scenario_id in cases.items(): + # log event + _logger.debug( + "> collecting inputs for " + "'%s', '%s', '%s', '%s'", *case + ) - if (not missing.empty) & (not warned): - _logger.warning("unmapped parameters in scenario(s)") - warned = True + # connect scenario and check for unmapped keys + client.scenario_id = scenario_id + missing = self._check_for_unmapped_input_parameters(client) - unmapped.update(set(missing)) + if (not missing.empty) & (not warned): + _logger.warning("unmapped parameters in scenario(s)") + warned = True - # reference scenario parameters - parameters = client.scenario_parameters + unmapped.update(set(missing)) - # collect irrelevant parameters - drop = self.excluded + self.depricated - drop = self.parameters.index.isin(drop) + # reference scenario parameters + parameters = client.get_input_parameters() - # collect relevant parameters - keep = self.parameters.index[~drop] - keep = parameters.index.isin(keep) + # collect irrelevant parameters + drop = self.excluded + self.depricated + drop = self.parameters.index.isin(drop) - # make subset amd append - parameters = parameters[keep] - values.append(parameters) + # collect relevant parameters + keep = self.parameters.index[~drop] + keep = parameters.index.isin(keep) - # warn for unmapped keys - if unmapped: - _logger.warn("encountered unmapped parameters: %s", - unmapped) + # make subset amd append + parameters = parameters[keep] + values.append(parameters) - # handle exception - except Exception as exc: - log_exception(exc, logger=_logger) + # warn for unmapped keys + if unmapped: + _logger.warn("encountered unmapped parameters: %s", unmapped) # construct frame and handle nulls frame = pd.concat(values, axis=1, keys=midx) @@ -471,15 +469,14 @@ def get_input_parameters( frame = frame.set_index(units, append=True) # set names of index levels - frame.index.names = ['KEY', 'UNIT'] - frame.columns.names = ['STUDY', 'SCENARIO', 'REGION', 'YEAR'] + frame.index.names = ["KEY", "UNIT"] + frame.columns.names = ["STUDY", "SCENARIO", "REGION", "YEAR"] return sort_frame(frame, axis=1, reference=self.reference) def set_input_parameters( - self, - frame: pd.DataFrame, - allow_external_coupling_parameters: bool = False) -> None: + self, frame: pd.DataFrame, allow_external_coupling_parameters: bool = False + ) -> None: """set input parameters""" # convert series to frame @@ -491,12 +488,11 @@ def set_input_parameters( frame = pd.DataFrame(frame) # drop unit from index - if 'UNIT' in frame.index.names: - frame = frame.reset_index(level='UNIT', drop=True) + if "UNIT" in frame.index.names: + frame = frame.reset_index(level="UNIT", drop=True) # ensure dataframe is consistent with session ids - errors = [midx for midx in frame.columns - if midx not in self.session_ids.index] + errors = [midx for midx in frame.columns if midx not in self.session_ids.index] # raise for errors if errors: @@ -508,9 +504,8 @@ def set_input_parameters( illegal = self.excluded + self.depricated if allow_external_coupling_parameters is False: - # list external coupling related keys - key = 'external_coupling' + key = "external_coupling" illegal += list(frame.index[frame.index.str.contains(key)]) # illegal parameters @@ -518,7 +513,6 @@ def set_input_parameters( # trigger warnings for mapped but disabled parameters if not illegal.empty: - # warn for keys for key in illegal: _logger.warning(f"excluded '{key}' from upload") @@ -526,32 +520,26 @@ def set_input_parameters( # drop excluded parameters frame = frame.drop(illegal) - try: - - # make client with context manager - with Client(**self._kwargs) as client: + # make client with context manager + with Client(**self._kwargs) as client: + # iterate over cases + for case, values in frame.items(): + # log event + _logger.debug( + "> changing input parameters for " + "'%s', '%s', '%s', '%s'", + *case, + ) - # iterate over cases - for case, values in frame.items(): + # drop unchanged keys + values = values.dropna() - # log event - _logger.debug("> changing input parameters for " + - "'%s', '%s', '%s', '%s'", *case) + # continue if no value specified + if values.empty: + continue - # drop unchanged keys - values = values.dropna() - - # continue if no value specified - if values.empty: - continue - - # connect to case and set values - client.scenario_id = self.session_ids.loc[case] - client.user_values = values - - # handle exception - except Exception as exc: - log_exception(exc, logger=_logger) + # connect to case and set values + client.scenario_id = self.session_ids.loc[case] + client.set_input_parameters(values) def get_hourly_carrier_curves( self, @@ -573,13 +561,12 @@ def get_hourly_carrier_curves( # get default mapping if (mapping is None) & (self.mapping is not None): - # lookup carriers in mapping and lower elements carriers = list(self.mapping.index.levels[1]) lcarriers = [carrier.lower() for carrier in carriers] # check if carrier is available - if not carrier in lcarriers: + if carrier not in lcarriers: raise KeyError("'%s' not specified as carrier in mapping") # subset correct carrier @@ -588,49 +575,47 @@ def get_hourly_carrier_curves( _logger.info("collecting hourly %s curves", carrier) - try: - - # make client with context manager - with Client(**self._kwargs) as client: - - # newlist - items = [] - - # iterate over cases - for case, scenario_id in cases.items(): - - # log event - _logger.debug("> collecting hourly %s curves for " + - "'%s', '%s', '%s', '%s'", carrier, *case) - - # connect scenario and get curves - client.scenario_id = scenario_id - - # get method for carrier curves - attr = f"hourly_{carrier}_curves" - curves = getattr(client, attr) - - # continue with disabled merit order - if curves.empty: - continue - - # set column name - curves.columns.names = ['KEY'] - - # check for categorisation - if mapping is not None: - - # categorise curves - curves = categorise_curves( - curves, mapping, columns=columns, - include_keys=include_keys, invert_sign=invert_sign) - - # append curves to list - items.append(curves.reset_index(drop=True)) - - # handle exception - except Exception as exc: - log_exception(exc, logger=_logger) + # make client with context manager + with Client(**self._kwargs) as client: + # newlist + items = [] + + # iterate over cases + for case, scenario_id in cases.items(): + # log event + _logger.debug( + "> collecting hourly %s curves for " + "'%s', '%s', '%s', '%s'", + carrier, + *case, + ) + + # connect scenario and get curves + client.scenario_id = scenario_id + + # get method for carrier curves + attr = f"hourly_{carrier}_curves" + curves = getattr(client, attr) + + # continue with disabled merit order + if curves.empty: + continue + + # set column name + curves.columns.names = ["KEY"] + + # check for categorisation + if mapping is not None: + # categorise curves + curves = categorise_curves( + curves, + mapping, + columns=columns, + include_keys=include_keys, + invert_sign=invert_sign, + ) + + # append curves to list + items.append(curves.reset_index(drop=True)) # construct frame for carrier frame = pd.concat(items, axis=1, keys=midx) @@ -649,35 +634,29 @@ def get_hourly_price_curves( _logger.info("collecting hourly price curves") - try: - - # make client with context manager - with Client(**self._kwargs) as client: - - # newlist - items = [] - - # iterate over cases - for case, scenario_id in cases.items(): - - # log event - _logger.debug("> collecting hourly price curve for " + - "'%s', '%s', '%s', '%s'", *case) + # make client with context manager + with Client(**self._kwargs) as client: + # newlist + items = [] - # connect scenario and get curves - client.scenario_id = scenario_id - curves = client.hourly_electricity_price_curve + # iterate over cases + for case, scenario_id in cases.items(): + # log event + _logger.debug( + "> collecting hourly price curve for " + "'%s', '%s', '%s', '%s'", + *case, + ) - # continue with disabled merit order - if curves.empty: - continue + # connect scenario and get curves + client.scenario_id = scenario_id + curves = client.hourly_electricity_price_curve - # append curves to list - items.append(curves.reset_index(drop=True)) + # continue with disabled merit order + if curves.empty: + continue - # handle exception - except Exception as exc: - log_exception(exc, logger=_logger) + # append curves to list + items.append(curves.reset_index(drop=True)) # construct frame for carrier frame = pd.concat(items, axis=1, keys=midx) @@ -696,34 +675,28 @@ def get_output_values( _logger.info("collecting gquery results") - try: + # make client with context manager + with Client(**self._kwargs) as client: + # newlist + values = [] - # make client with context manager - with Client(**self._kwargs) as client: - - # newlist - values = [] - - # connect scenario and check for unmapped keys - client.gqueries = list(self.gqueries.index) - - # get parameter settings - for case, scenario_id in cases.items(): - - # log event - _logger.debug("> collecting gquery results for " + - "'%s', '%s', '%s', '%s'", *case) + # connect scenario and check for unmapped keys + client.gqueries = list(self.gqueries.index) - # connect scenario and set gqueries - client.scenario_id = scenario_id - gqueries = client.gquery_results['future'] + # get parameter settings + for case, scenario_id in cases.items(): + # log event + _logger.debug( + "> collecting gquery results for " + "'%s', '%s', '%s', '%s'", + *case, + ) - # append gquery results - values.append(gqueries) + # connect scenario and set gqueries + client.scenario_id = scenario_id + gqueries = client.gquery_results["future"] - # handle exception - except Exception as exc: - log_exception(exc, logger=_logger) + # append gquery results + values.append(gqueries) # construct frame and handle nulls frame = pd.concat(values, axis=1, keys=midx) @@ -734,8 +707,8 @@ def get_output_values( frame = frame.set_index(units, append=True) # set names of index levels - frame.index.names = ['KEY', 'UNIT'] - frame.columns.names = ['STUDY', 'SCENARIO', 'REGION', 'YEAR'] + frame.index.names = ["KEY", "UNIT"] + frame.columns.names = ["STUDY", "SCENARIO", "REGION", "YEAR"] # fill missing values frame = frame.fillna(0) @@ -746,8 +719,8 @@ def make_myc_urls( self, midx: tuple | pd.MultiIndex | None = None, path: str | None = None, - params : dict[str, str] | None = None, - add_title: bool = True + params: dict[str, str] | None = None, + add_title: bool = True, ) -> pd.Series: """convert session ids excel to myc urls""" @@ -767,17 +740,17 @@ def make_myc_urls( cases = self.session_ids.loc[midx] if self.reference is not None: - # drop reference scenario - if self.reference in cases.index.unique(level='SCENARIO'): - cases = cases.drop(self.reference, level='SCENARIO', axis=0) + if self.reference in cases.index.unique(level="SCENARIO"): + cases = cases.drop(self.reference, level="SCENARIO", axis=0) # no cases dropped if cases.empty: - # warn user - _logger.warning("Cannot make URLs as model or passed subset " - "only contains reference scenarios") + _logger.warning( + "Cannot make URLs as model or passed subset " + "only contains reference scenarios" + ) return pd.Series() @@ -788,20 +761,23 @@ def make_myc_urls( urls = cases.astype(str).groupby(level=levels) # make urls - urls = urls.apply(lambda sids: make_myc_url( - url=self.myc_url, scenario_ids=sids, path=path, params=params)) + urls = urls.apply( + lambda sids: make_myc_url( + url=self.myc_url, scenario_ids=sids, path=path, params=params + ) + ) # add title if bool(add_title) is True: for idx, url in urls.items(): - # make title and append parameter inplace - params = {'title': " ".join(map(str, idx))} + params = {"title": " ".join(map(str, idx))} urls.at[idx] = set_url_parameters(url, params=params) - return pd.Series(urls, name='URL').sort_index() + return pd.Series(urls, name="URL").sort_index() - def to_excel(self, + def to_excel( + self, filepath: str | None = None, midx: tuple | pd.MultiIndex | None = None, input_parameters: bool | pd.DataFrame = True, @@ -814,7 +790,7 @@ def to_excel(self, columns: list[str] | None = None, include_keys: bool = False, invert_sign: bool = False, - ) -> None: + ) -> None: """Export results of model to Excel. Parameters @@ -864,44 +840,17 @@ def to_excel(self, a negative sign. Demand will be denoted with a positve value and supply with a negative value.""" - # pylint: disable=C0415 - # Due to optional import + # import optional dependency + xlsxwriter = import_optional_dependency("xlsxwriter") - from pathlib import Path - from pyetm.utils.excel import add_frame, add_series - - if TYPE_CHECKING: - # import xlswriter - import xlsxwriter - - else: - # import optional dependency - xlsxwriter = import_optional_dependency('xlsxwriter') - - # pylint: enable=C0415 - - # supported carriers - all_carriers = ['electricity', 'heat', 'hydrogen', 'methane'] - - # default carriers - if carriers is None: - carriers = all_carriers - - # add string to list - if isinstance(carriers, str): - carriers = [carriers] - - # check passed carriers - for carrier in carriers: - if carrier.lower() not in carriers: - raise ValueError("carrier '{carrier}' not supported") + # check carriers + carriers = check_carriers(carriers) # make filepath if filepath is None: - # default filepath now = datetime.datetime.now().strftime("%Y%m%d%H%M") - filepath = Path.cwd().joinpath(now + '.xlsx') + filepath = Path.cwd().joinpath(now + ".xlsx") # check filepath if not Path(filepath).parent.exists: @@ -916,8 +865,13 @@ def to_excel(self, # write input parameters if input_parameters is not False: - add_frame('INPUT_PARAMETERS', input_parameters, workbook, - index_width=[80, 18], column_width=18) + add_frame( + "INPUT_PARAMETERS", + input_parameters, + workbook, + index_width=[80, 18], + column_width=18, + ) # default outputs if output_values is True: @@ -925,17 +879,26 @@ def to_excel(self, # write output values if output_values is not False: - add_frame('OUTPUT_VALUES', output_values, workbook, - index_width=[80, 18], column_width=18) + add_frame( + "OUTPUT_VALUES", + output_values, + workbook, + index_width=[80, 18], + column_width=18, + ) # iterate over carriers if hourly_carrier_curves is True: for carrier in carriers: - # get carrier curves - curves = self.get_hourly_carrier_curves(carrier, - midx=midx, mapping=mapping, columns=columns, - include_keys=include_keys, invert_sign=invert_sign) + curves = self.get_hourly_carrier_curves( + carrier, + midx=midx, + mapping=mapping, + columns=columns, + include_keys=include_keys, + invert_sign=invert_sign, + ) # add to excel name = carrier.upper() @@ -944,7 +907,7 @@ def to_excel(self, # include hourly price curves if hourly_price_curves is True: curves = self.get_hourly_price_curves(midx=midx) - add_frame('EPRICE', curves, workbook, column_width=18) + add_frame("EPRICE", curves, workbook, column_width=18) # default urls if myc_urls is True: @@ -953,8 +916,9 @@ def to_excel(self, # add urls to workbook if myc_urls is not False: if not myc_urls.empty: - add_series('ETM_URLS', myc_urls, workbook, - index_width=18, column_width=80) + add_series( + "ETM_URLS", myc_urls, workbook, index_width=18, column_width=80 + ) # write workbook workbook.close() diff --git a/src/pyetm/optional.py b/src/pyetm/optional.py index 64d575c..7f6b91e 100644 --- a/src/pyetm/optional.py +++ b/src/pyetm/optional.py @@ -5,16 +5,12 @@ import importlib import sys -import warnings - from types import ModuleType -from pandas.util.version import Version -# A mapping from import name to package name (on PyPI) for packages where -# these two names are different. +from pandas.util.version import Version VERSIONS = {"aiohttp": "3.8.1", "xlsxwriter": "3.0"} -INSTALL_MAPPING = {} + def get_version(module: ModuleType) -> str: """get version""" @@ -24,17 +20,12 @@ def get_version(module: ModuleType) -> str: # raise error if version is None: - - # nest asyncio has no version - if module == 'nest_asyncio': - return '' - raise ImportError(f"Can't determine version for {module.__name__}") return version -def import_optional_dependency(name: str, extra: str = "", - errors: str = "raise", min_version: str | None = None): + +def import_optional_dependency(name: str, min_version: str | None = None): """Import an optional dependency. By default, if a dependency is missing an ImportError with a nice @@ -45,17 +36,6 @@ def import_optional_dependency(name: str, extra: str = "", ---------- name : str The module name. - extra : str - Additional text to include in the ImportError message. - errors : str {'raise', 'warn', 'ignore'} - What to do when a dependency is not found or its version is too old. - * raise : Raise an ImportError - * warn : Only applicable when a module's version is to old. - Warns that the version is too old and returns None - * ignore: If the module is not installed, return None, otherwise, - return the module, even if the version is too old. - It's expected that users validate the version locally when - using ``errors="ignore"`` (see. ``io/html.py``) min_version : str, default None Specify a minimum version that is different from the global pandas minimum version required. @@ -68,26 +48,16 @@ def import_optional_dependency(name: str, extra: str = "", is False, or when the package's version is too old and `errors` is ``'warn'``. """ - - assert errors in {"warn", "raise", "ignore"} - - package_name = INSTALL_MAPPING.get(name) - install_name = package_name if package_name is not None else name - - msg = ( - f"Missing optional dependency '{install_name}'. {extra} " - f"Use pip or conda to install {install_name}." - ) - try: module = importlib.import_module(name) except ImportError as exc: + msg = ( + f"Missing optional dependency '{name}'." + f"Use pip or conda to install {name}." + ) - if errors == "raise": - raise ImportError(msg) from exc - - return None + raise ImportError(msg) from exc # handle submodules parent = name.split(".")[0] @@ -108,11 +78,6 @@ def import_optional_dependency(name: str, extra: str = "", f"(version '{version}' currently installed)." ) - if errors == "warn": - warnings.warn(msg, UserWarning) - return None - - elif errors == "raise": - raise ImportError(msg) + raise ImportError(msg) return module diff --git a/src/pyetm/profiles/__init__.py b/src/pyetm/profiles/__init__.py index 0606fd2..453ea5e 100644 --- a/src/pyetm/profiles/__init__.py +++ b/src/pyetm/profiles/__init__.py @@ -1,3 +1,5 @@ """Initialize module""" -from .capacityfactors import CapacityFactorProfiles -from .weather import WeatherDemandProfiles +from .cfactors import validate_capacity_factors +from .heat import HeatDemandProfileGenerator + +__all__ = ["validate_capacity_factors", "HeatDemandProfileGenerator"] diff --git a/src/pyetm/profiles/capacityfactors/__init__.py b/src/pyetm/profiles/capacityfactors/__init__.py deleted file mode 100644 index ad1146e..0000000 --- a/src/pyetm/profiles/capacityfactors/__init__.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Loadfactor scaling, sigmoid adapted from: -https://stackoverflow.com/questions/3985619/how-to-calculate-a-logistic-sigmoid-function-in-python""" - -from __future__ import annotations -from pyetm.utils.profiles import validate_profile - -import numpy as np -import pandas as pd - - -class CapacityFactorProfiles: - """Loading factors for renewables""" - - @property - def year(self): - """year for which generator is configured""" - return self._year - - @property - def wind_offshore(self): - """validated wind offshore profile""" - return self._wind_offshore - - @property - def wind_onshore(self): - """validated wind onshore profile""" - return self._wind_onshore - - @property - def wind_coastal(self): - """validated wind coastal profile""" - return self._wind_coastal - - @property - def solar_pv(self): - """validated solar pv profile""" - return self._solar_pv - - @property - def solar_thermal(self): - """validated solar thermal profile""" - return self._solar_thermal - - def __init__( - self, - year : int, - wind_offshore : pd.Series, - wind_onshore : pd.Series, - wind_coastal : pd.Series, - solar_pv : pd.Series, - solar_thermal : pd.Series, - ) -> CapacityFactorProfiles: - """Initialize class object.""" - - # set year - self._year = year - - # set wind offshore - self._wind_offshore = validate_profile( - wind_offshore, name='weather/wind_offshore_baseline', year=year) - - # set wind onshore - self._wind_onshore = validate_profile( - wind_onshore, name='weather/wind_inland_baseline', year=year) - - # set wind coastal - self._wind_coastal = validate_profile( - wind_coastal, name='weather/wind_coastal_baseline', year=year) - - # set solar pv - self._solar_pv = validate_profile( - solar_pv, name='weather/solar_pv_profile_1', year=year) - - # set solar thermal - self._solar_thermal = validate_profile( - solar_thermal, name='weather/solar_thermal', year=year) - - def __enter__(self): - return self - - def __exit__(self, *args, **kwargs): - return None - - def __str__(self) -> str: - """String name""" - return f"LoadFactorProfiles(year={self.year})" - - def make_capacity_factors( - self, - wind_offshore_scalar: float | None = None, - wind_onshore_scalar: float | None = None, - wind_coastal_scalar: float | None = None, - solar_pv_scalar: float | None = None, - solar_thermal_scalar: float | None = None, - **kwargs -) -> pd.DataFrame: - """Scale profiles and return ETM compatible format""" - - # scale offshore profile - offshore = self.scale_profile( - profile=self.wind_offshore, scalar=wind_offshore_scalar, **kwargs) - - # scale onshore profile - onshore = self.scale_profile( - profile=self.wind_onshore, scalar=wind_onshore_scalar, **kwargs) - - # scale wind coastal profile - coastal = self.scale_profile( - profile=self.wind_coastal, scalar=wind_coastal_scalar, **kwargs) - - # scale solar pv profile - solar = self.scale_profile( - profile=self.solar_pv, scalar=solar_pv_scalar, **kwargs) - - # scale solar thermal profile - thermal = self.scale_profile( - profile=self.solar_thermal, scalar=solar_thermal_scalar, **kwargs) - - # merge profiles - profiles = pd.concat( - [offshore, onshore, coastal, solar, thermal], axis=1) - - return profiles.sort_index(axis=1) - - def scale_profile( - self, - profile: pd.Series, - scalar: float | None = None, - **kwargs - ) -> pd.Series: - """aply sigmoid?""" - - # check for values within range 0, 1 - - # don't scale profile - if (scalar is None) | (scalar == 1): - return profile - - # get derivative, normalize and scale with volume - factor = profile.apply(self._scaler) - factor = factor / factor.sum() * (scalar - 1) * profile.sum() - - # recheck if values within range or warn user. - - return profile.add(factor) - - def _sigmoid(self, factor: float) -> float: - """parameterised sigmoid function for normalized profiles""" - return -7.9921 + (9.2005 / (1 + np.exp(-1.7363*factor + -1.8928))) - - def _scaler(self, factor: float) -> float: - """scale by derivative of sigmoid""" - return self._sigmoid(factor) * (1 - self._sigmoid(factor)) - - # def sigmoid(x): - # return np.exp(-np.logaddexp(0, -x)) - - # def sigmoid(x): - # return 1 / (1 + np.exp(-(np.log(40000) / 1) * (x-1) + np.log(0.005))) diff --git a/src/pyetm/profiles/cfactors.py b/src/pyetm/profiles/cfactors.py new file mode 100644 index 0000000..ed319f2 --- /dev/null +++ b/src/pyetm/profiles/cfactors.py @@ -0,0 +1,25 @@ +"""Helper to assign capacity factor names""" +from __future__ import annotations + +import pandas as pd +from pyetm.utils.profiles import validate_profile + + +def validate_capacity_factors( + wind_offshore: pd.Series[float], + wind_onshore: pd.Series[float], + wind_coastal: pd.Series[float], + solar_pv: pd.Series[float], + solar_thermal: pd.Series[float], +) -> pd.DataFrame: + """validate profiles and assign correct keys""" + + profiles = [ + validate_profile(wind_offshore, name="weather/wind_offshore_baseline"), + validate_profile(wind_onshore, name="weather/wind_inland_baseline"), + validate_profile(wind_coastal, name="weather/wind_coastal_baseline"), + validate_profile(solar_pv, name="weather/solar_pv_profile_1"), + validate_profile(solar_thermal, name="weather/solar_thermal"), + ] + + return pd.concat(profiles, axis=1).sort_index(axis=1) diff --git a/src/pyetm/profiles/heat/__init__.py b/src/pyetm/profiles/heat/__init__.py new file mode 100644 index 0000000..030922c --- /dev/null +++ b/src/pyetm/profiles/heat/__init__.py @@ -0,0 +1,59 @@ +"""Initialize weather profile module""" + +from __future__ import annotations + +import pandas as pd + +from .buildings import Buildings +from .households import HousePortfolio + + +class HeatDemandProfileGenerator: + """Heat demand profile generator.""" + + @classmethod + def from_defaults(cls): + """Initialize with Quintel default settings.""" + + # default object configurations + households = HousePortfolio.from_defaults() + buildings = Buildings.from_defaults() + + return cls(households, buildings) + + def __init__(self, households: HousePortfolio, buildings: Buildings): + """Initialize class object. + + Parameters + ---------- + households : HousePortolio + HousePortfolio object. + buildings : Buildings + Buidlings object.""" + + # set objects + self.households = households + self.buildings = buildings + + def make_heat_demand_profiles( + self, + temperature: pd.Series[float], + irradiance: pd.Series[float], + wind_speed: pd.Series[float], + ) -> pd.DataFrame: + """heat demand related profiles""" + + # make household heat demand profiles + households = self.households.make_heat_demand_profiles(temperature, irradiance) + + # make buildings heat demand profile + buildings = self.buildings.make_heat_demand_profile(temperature, wind_speed) + + # add other profiles + temperature = pd.Series(temperature, name="weather/air_temperature") + agriculture = pd.Series(buildings, name="weather/agriculture_heating") + + # merge profiles + profiles = pd.concat([agriculture, temperature, buildings, households], axis=1) + + return profiles.sort_index(axis=1) diff --git a/src/pyetm/profiles/weather/buildings.py b/src/pyetm/profiles/heat/buildings.py similarity index 50% rename from src/pyetm/profiles/weather/buildings.py rename to src/pyetm/profiles/heat/buildings.py index bcc3c6a..bc2c213 100644 --- a/src/pyetm/profiles/weather/buildings.py +++ b/src/pyetm/profiles/heat/buildings.py @@ -3,17 +3,17 @@ from __future__ import annotations -from pyetm.logger import PACKAGEPATH -from pyetm.utils.profiles import validate_profile, validate_profile_lenght - import pandas as pd +from pyetm.logger import PACKAGEPATH +from pyetm.utils.profiles import validate_profile, make_period_index + class Buildings: """Aggregate heating model for buildings.""" @classmethod - def from_defaults(cls, name: str = 'default') -> Buildings: + def from_defaults(cls, name: str = "default") -> Buildings: """Initialize with Quintel default values. Parameters @@ -21,19 +21,35 @@ def from_defaults(cls, name: str = 'default') -> Buildings: name : str, default 'default' name of object.""" + # relevant columns + dtypes = {"reference": float, "slope": float, "constant": float} + + # filepath + file = PACKAGEPATH.joinpath("data/G2A_parameters.csv") + usecols = [key for key in dtypes] + # load G2A parameters - file = PACKAGEPATH.joinpath('data/G2A_parameters.csv') - parameters = pd.read_csv(file) + frame = pd.read_csv(file, usecols=usecols, dtype=dtypes) - return cls(name=name, **parameters) + # get relevant profiles + reference = frame["reference"] + slope = frame["slope"] + constant = frame["constant"] + + return cls( + name=name, + reference=reference, + slope=slope, + constant=constant, + ) def __init__( self, - reference: pd.Series, - slope: pd.Series, - constant: pd.Series, - name: str | None = None - ) -> Buildings: + reference: pd.Series[float], + slope: pd.Series[float], + constant: pd.Series[float], + name: str | None = None, + ): """Initialize class object. Parameters @@ -52,21 +68,17 @@ def __init__( # set parameters self.name = name - self.reference = validate_profile_lenght(reference) - self.slope = validate_profile_lenght(slope) - self.constant = validate_profile_lenght(constant) + self.reference = validate_profile(reference) + self.slope = validate_profile(slope) + self.constant = validate_profile(constant) def __repr__(self) -> str: """Reproduction string""" return f"Buildings(name={self.name})" def _calculate_heat_demand( - self, - effective: float, - reference: float, - slope: float, - constant: float - ) -> float: + self, effective: float, reference: float, slope: float, constant: float + ) -> float: """Calculates the required heating demand for the hour. Parameters @@ -85,9 +97,13 @@ def _calculate_heat_demand( ------ demand : float Required heating demand""" - return (reference - effective) * slope + constant if effective < reference else constant + return ( + (reference - effective) * slope + constant + if effective < reference + else constant + ) - def _make_parameters(self, effective: pd.Series) -> pd.DataFrame: + def _make_parameters(self, effective: pd.Series[float]) -> pd.DataFrame: """Make parameters frame from effective temperature and G2A parameters. @@ -102,17 +118,19 @@ def _make_parameters(self, effective: pd.Series) -> pd.DataFrame: Merged parameters with correct index.""" # resample effective temperature for each hour - effective = effective.resample('H').ffill() + effective = effective.resample("H").ffill() # check for index equality - if ((not self.reference.index.equals(self.slope.index)) - | (not self.slope.index.equals(self.constant.index))): - raise ValueError("indices for 'reference', 'slope' and " - "'constant' profiles are not alligned.") + if (not self.reference.index.equals(self.slope.index)) or ( + not self.slope.index.equals(self.constant.index) + ): + raise ValueError( + "indices for 'reference', 'slope' and " + "'constant' profiles are not alligned." + ) # merge G2A parameters - parameters = pd.concat( - [self.reference, self.slope, self.constant], axis=1) + parameters = pd.concat([self.reference, self.slope, self.constant], axis=1) # reindex parameters parameters.index = effective.index @@ -121,9 +139,9 @@ def _make_parameters(self, effective: pd.Series) -> pd.DataFrame: def make_heat_demand_profile( self, - temperature: pd.Series, - wind_speed : pd.Series, - year: int | None = None) -> pd.DataFrame: + temperature: pd.Series[float], + wind_speed: pd.Series[float], + ) -> pd.Series[float]: """Make heat demand profile for buildings. Effective temperature is defined as daily average temperature @@ -137,10 +155,6 @@ def make_heat_demand_profile( Celcius for 8760 hours. wind_speed : pd.Series Wind speed profile in m/s for 8760 hours. - year : int, default None - Optional year to help construct a - PeriodIndex when a series are passed - without PeriodIndex or DatetimeIndex. Return ------ @@ -148,35 +162,38 @@ def make_heat_demand_profile( Heat demand profile for buildings.""" # validate temperature profile - temperature = validate_profile( - temperature, name='temperature', year=year) + temperature = validate_profile(temperature, name="temperature") + wind_speed = validate_profile(wind_speed, name="wind_speed") - # validate irradiance profile - wind_speed = validate_profile( - wind_speed, name='wind_speed', year=year) + # # check for allignment + # if not temperature.index.equals(wind_speed.index): + # raise ValueError( + # "Periods or Datetimes of 'temperature' " + # "and 'wind_speed' profiles are not alligned." + # ) - # check for allignment - if not temperature.index.equals(wind_speed.index): - raise ValueError("Periods or Datetimes of 'temperature' " - "and 'wind_speed' profiles are not alligned.") + # merge profiles and assign periodindex + merged = pd.concat([temperature, wind_speed], axis=1) + merged.index = make_period_index(year=2019, periods=8760) - # merge profiles and get daily average - effective = pd.concat([temperature, wind_speed], axis=1) - effective = effective.groupby(pd.Grouper(freq='1D')).mean() + # evaluate daily average + merged = merged.groupby(pd.Grouper(freq="1D")).mean() # evaluate effective temperature - effective = effective['temperature'] - (effective['wind_speed'] / 1.5) - effective = pd.Series(effective, name='effective') + effective = merged["temperature"] - (merged["wind_speed"] / 1.5) + effective = pd.Series(effective, name="effective", dtype=float) # make parameters profiles = self._make_parameters(effective) # apply calculate demand functon - profile = profiles.apply( - lambda row: self._calculate_heat_demand(**row), axis=1) + profile = profiles.apply(lambda row: self._calculate_heat_demand(**row), axis=1) # name profile - name = 'weather/buildings_heating' - profile = pd.Series(profile, name=name, dtype='float64') + name = "weather/buildings_heating" + profile = pd.Series(profile, name=name, dtype=float) + + # scale profile values + profile = profile / profile.sum() / 3.6e3 - return profile / profile.sum() / 3.6e3 + return profile.reset_index(drop=True) diff --git a/src/pyetm/profiles/weather/cooling.py b/src/pyetm/profiles/heat/cooling.py similarity index 85% rename from src/pyetm/profiles/weather/cooling.py rename to src/pyetm/profiles/heat/cooling.py index 4fa912e..32251a4 100644 --- a/src/pyetm/profiles/weather/cooling.py +++ b/src/pyetm/profiles/heat/cooling.py @@ -2,6 +2,7 @@ https://ec.europa.eu/eurostat/statistics-explained/index.php?title=Heating_and_cooling_degree_days_-_statistics#Heating_and_cooling_degree_days_at_EU_level""" from __future__ import annotations + import pandas as pd @@ -12,8 +13,8 @@ def __init__( self, daily_threshold: float | None = None, hourly_threshold: float | None = None, - name : str | None = None - ) -> Cooling: + name: str | None = None, + ): """Initialize class object. Parameters @@ -66,10 +67,11 @@ def make_cooling_profile( # construct mask for daily threshold # daily average temperature exceeds daily threshold value - daily = temperature.groupby( - pd.Grouper(freq='1D'), group_keys=False).apply( - lambda group: pd.Series( - group.mean() > self.daily_threshold, index=group.index)) + daily = temperature.groupby(pd.Grouper(freq="1D"), group_keys=False).apply( + lambda group: pd.Series( + group.mean() > self.daily_threshold, index=group.index + ) + ) # construct mask for hourly threshold # hourly temperature exceeds hourly threshold value @@ -77,8 +79,8 @@ def make_cooling_profile( # set values outside mutual masks to zero # subtract hourly threshold value from cooling degrees - temperature = temperature.where( - daily & hourly, self.hourly_threshold).sub( - self.hourly_threshold) + temperature = temperature.where(daily & hourly, self.hourly_threshold).sub( + self.hourly_threshold + ) return temperature.div(temperature.sum()) diff --git a/src/pyetm/profiles/weather/households.py b/src/pyetm/profiles/heat/households.py similarity index 66% rename from src/pyetm/profiles/weather/households.py rename to src/pyetm/profiles/heat/households.py index ba40814..57f68d1 100644 --- a/src/pyetm/profiles/weather/households.py +++ b/src/pyetm/profiles/heat/households.py @@ -2,13 +2,14 @@ https://github.com/quintel/etdataset-public/tree/master/curves/demand/households/space_heating""" from __future__ import annotations -from collections.abc import Iterable -from pyetm.logger import PACKAGEPATH -from pyetm.utils.profiles import validate_profile +from collections.abc import Iterable import pandas as pd +from pyetm.logger import PACKAGEPATH +from pyetm.utils.profiles import validate_profile, make_period_index + from .smoothing import ProfileSmoother @@ -23,7 +24,7 @@ def p_concrete(self) -> float: @property def c_concrete(self) -> float: """Concrete thermal conductance in W/kg""" - return .88e3 + return 0.88e3 @property def u_value(self) -> float: @@ -56,31 +57,53 @@ def from_defaults(cls, house_type: str, insulation_level: str) -> Houses: insulation_level : str Name of default insulation type.""" + # relevant columns + dtypes = { + "house_type": str, + "insulation_level": str, + "behaviour": float, + "r_value": float, + "window_area": float, + "surface_area": float, + "wall_thickness": float, + } + + # filepath + file = PACKAGEPATH.joinpath("data/house_properties.csv") + usecols = [key for key in dtypes] + # load properties - file = PACKAGEPATH.joinpath('data/house_properties.csv') - properties = pd.read_csv( - file, index_col=['house_type', 'insulation_level']) + properties = pd.read_csv(file, usecols=usecols, index_col=[0, 1], dtype=dtypes) + props = properties.T[(house_type, insulation_level)] - # subset correct house and insulation profile - properties = properties.loc[(house_type, insulation_level)] + # get relevant properties + behaviour = props["behaviour"] + r_value = props["r_value"] + window_area = props["window_area"] + surface_area = props["surface_area"] + wall_thickness = props.loc["wall_thickness"] # load thermostat values - file = PACKAGEPATH.joinpath('data/thermostat_values.csv') - thermostat = pd.read_csv( - file, usecols = [insulation_level]).squeeze('columns') + file = PACKAGEPATH.joinpath("data/thermostat_values.csv") + thermostat = pd.read_csv(file, usecols=[insulation_level]).squeeze("columns") # initialize house house = cls( + behaviour=behaviour, + r_value=r_value, + window_area=window_area, + surface_area=surface_area, + wall_thickness=wall_thickness, thermostat=thermostat, house_type=house_type, insulation_level=insulation_level, smoother=ProfileSmoother(), - **properties, ) return house - def __init__(self, + def __init__( + self, behaviour: float, r_value: float, surface_area: float, @@ -89,9 +112,9 @@ def __init__(self, thermostat: Iterable[float], house_type: str | None = None, insulation_level: str | None = None, - smoother : ProfileSmoother | None = None, - **kwargs - ) -> Houses: + smoother: ProfileSmoother | None = None, + **kwargs, + ): """Initialize class object. Parameters @@ -121,11 +144,11 @@ def __init__(self, # default house type if house_type is None: - house_type = 'unnamed' + house_type = "unnamed" # default insulation type if insulation_level is None: - insulation_level = 'unnamed' + insulation_level = "unnamed" # default smoother if smoother is None: @@ -139,7 +162,7 @@ def __init__(self, self.window_area = window_area # set thermostat - self.thermostat = thermostat + self.thermostat = list(thermostat) self._inside = self.thermostat[0] # set type names @@ -153,15 +176,13 @@ def __repr__(self) -> str: """Reproduction string""" return ( f"House(house_type='{self.house_type}', " - f"insulation_level='{self.insulation_level}')") + f"insulation_level='{self.insulation_level}')" + ) def _calculate_heat_demand( - self, - temperature: float, - irradiance: float, - hour: int + self, temperature: float, irradiance: float, hour: int ) -> float: - """"Calculates the required heating demand for the hour. + """ "Calculates the required heating demand for the hour. Parameters ---------- @@ -178,11 +199,10 @@ def _calculate_heat_demand( Required heating demand.""" # determine energy demand at hour and update inside temperature - demand = (max(self.thermostat[hour] - self._inside, 0) - * self.heat_capacity) + demand = max(self.thermostat[int(hour)] - self._inside, 0) * self.heat_capacity # determine new inside temperature - self._inside = max(self.thermostat[hour], self._inside) + self._inside = max(self.thermostat[int(hour)], self._inside) # determine energy leakage and absorption leakage = (self._inside - temperature) * self.exchange_delta @@ -194,11 +214,8 @@ def _calculate_heat_demand( return demand def make_heat_demand_profile( - self, - temperature: pd.Series, - irradiance: pd.Series, - year: int | None = None - ) -> pd.DataFrame: + self, temperature: pd.Series[float], irradiance: pd.Series[float] + ) -> pd.Series[float]: """Make heat demand profile for house. Parameters @@ -208,10 +225,6 @@ def make_heat_demand_profile( Celcius for 8760 hours. irradiance : pd.Series Irradiance profile in W/m2 for 8760 hours. - year : int, default None - Optional year to help construct a - PeriodIndex when a series are passed - without PeriodIndex or DatetimeIndex. Return ------ @@ -219,34 +232,32 @@ def make_heat_demand_profile( Heat demand profile for house type at house insulation level.""" - # validate temperature profile - temperature = validate_profile( - temperature, name='temperature', year=year) + # validate profiles + temperature = validate_profile(temperature, name="temperature") + irradiance = validate_profile(irradiance, name="irradiance") - # validate irradiance profile - irradiance = validate_profile( - irradiance, name='irradiance', year=year) + # # check for allignment + # if not temperature.index.equals(irradiance.index): + # raise ValueError( + # "Periods or Datetimes of 'temperature' " + # "and 'irradiance' profiles are not alligned." + # ) - # check for allignment - if not temperature.index.equals(irradiance.index): - raise ValueError("Periods or Datetimes of 'temperature' " - "and 'irradiance' profiles are not alligned.") - - # merge profiles and get index hours - profile = pd.concat([temperature, irradiance], axis=1) - profile['hour'] = profile.index.hour + # merge profiles and assign hour of day + merged = pd.concat([temperature, irradiance], axis=1) + merged["hour"] = make_period_index(2019, periods=8760).hour # apply calculate demand function - profile = profile.apply( - lambda row: self._calculate_heat_demand(**row), axis=1) + profile = merged.apply(lambda row: self._calculate_heat_demand(**row), axis=1) # smooth resulting profile values = self.smoother.calculate_smoothed_demand( - profile.values, self.insulation_level) + profile.values, self.insulation_level + ) # name profile - name = f'weather/insulation_{self.house_type}_{self.insulation_level}' - profile = pd.Series(values, profile.index, dtype='float64', name=name) + name = f"weather/insulation_{self.house_type}_{self.insulation_level}" + profile = pd.Series(values, profile.index, dtype=float, name=name) return profile / profile.sum() / 3.6e3 @@ -266,39 +277,31 @@ def houses(self, houses: Iterable[Houses]): # check items in iterable for House for house in houses: if not isinstance(house, Houses): - raise TypeError('houses must be of type House') + raise TypeError("houses must be of type House") # set parameter self._houses = houses @classmethod - def from_defaults(cls, name: str = 'default') -> HousePortfolio: + def from_defaults(cls, name: str = "default") -> HousePortfolio: """From Quintel default house types and insulation levels.""" # load properties - file = PACKAGEPATH.joinpath('data/house_properties.csv') - properties = pd.read_csv( - file, usecols = ['house_type', 'insulation_level']) + file = PACKAGEPATH.joinpath("data/house_properties.csv") + properties = pd.read_csv(file, usecols=["house_type", "insulation_level"]) # newlist houses = [] # iterate over house types and insulation levels - for house_type in properties.house_type.unique(): - for insultation_level in properties.insulation_level.unique(): - + for house_type in properties["house_type"].unique(): + for insultation_level in properties["insulation_level"].unique(): # init house from default settings - houses.append( - Houses.from_defaults( - house_type, insultation_level)) + houses.append(Houses.from_defaults(house_type, insultation_level)) return cls(houses, name=name) - def __init__( - self, - houses: Iterable[Houses], - name: str | None = None - ) -> HousePortfolio: + def __init__(self, houses: Iterable[Houses], name: str | None = None): """Initialize class object. Parameters @@ -319,13 +322,10 @@ def __repr__(self) -> str: def __str__(self) -> str: """String name""" - return f'HousePortfolio(name={self.name})' + return f"HousePortfolio(name={self.name})" def make_heat_demand_profiles( - self, - temperature : pd.Series, - irradiance : pd.Series, - year: int | None = None + self, temperature: pd.Series[float], irradiance: pd.Series[float] ) -> pd.DataFrame: """Make heat demand profiles for all houses. @@ -336,10 +336,6 @@ def make_heat_demand_profiles( Celcius for 8760 hours. irradiance : pd.Series Irradiance profile in W/m2 for 8760 hours. - year : int, default None - Optional year to help construct a - PeriodIndex when a series are passed - without PeriodIndex or DatetimeIndex. Return ------ @@ -349,7 +345,8 @@ def make_heat_demand_profiles( # get heat profile for each house object profiles = [ - house.make_heat_demand_profile( - temperature, irradiance, year) for house in self.houses] + house.make_heat_demand_profile(temperature, irradiance) + for house in self.houses + ] return pd.concat(profiles, axis=1) diff --git a/src/pyetm/profiles/weather/smoothing.py b/src/pyetm/profiles/heat/smoothing.py similarity index 85% rename from src/pyetm/profiles/weather/smoothing.py rename to src/pyetm/profiles/heat/smoothing.py index a1e3d5a..4270a12 100644 --- a/src/pyetm/profiles/weather/smoothing.py +++ b/src/pyetm/profiles/heat/smoothing.py @@ -3,6 +3,7 @@ import numpy as np + class ProfileSmoother: """ The profiles generator is based on data for an individual household. @@ -26,14 +27,19 @@ class ProfileSmoother: X houses rather than an individual household. """ - def __init__(self, number_of_houses=300, hours_shifted=None, - interpolation_steps=10, random_seed=1337): + def __init__( + self, + number_of_houses=300, + hours_shifted=None, + interpolation_steps=10, + random_seed=1337, + ): """init""" # Standard deviation per insulation type. # See README for source if hours_shifted is None: - hours_shifted = {'low' : 2, 'medium' : 2.5, 'high' : 3} + hours_shifted = {"low": 2, "medium": 2.5, "high": 3} # interpolation steps # use intervals of 6 minutes when shifting curves @@ -44,7 +50,7 @@ def __init__(self, number_of_houses=300, hours_shifted=None, self.random_seed = random_seed def generate_deviations(self, size, scale): - ''' + """ Generate X random numbers with a standard deviation of Y hours Round to 1 decimal place and multiply by 10 to get integer value. The number designates the number of @@ -52,7 +58,7 @@ def generate_deviations(self, size, scale): compared to the original demand profile. E.g. '15' means that the demand profile will be shifted forward 1.5 hours, '-10' means it will be shifted backwards 1 hour - ''' + """ # (re)set random seed np.random.seed(self.random_seed) @@ -68,35 +74,35 @@ def generate_deviations(self, size, scale): return shifts.astype(int) def interpolate(self, arr, steps): - ''' + """ Interpolate the original demand profile to allow for smaller intervals than 1 hour (steps=10 means 6 minute intervals) - ''' + """ interpolated_arr = [] for index, value in enumerate(arr): start = value - if index == len(arr)-1: + if index == len(arr) - 1: stop = arr[0] else: - stop = arr[index+1] + stop = arr[index + 1] step_size = (stop - start) / steps for i in range(0, steps): - interpolated_arr.append(start + i*step_size) + interpolated_arr.append(start + i * step_size) return interpolated_arr def shift_curve(self, arr, num): - ''' + """ Rotate the elements of an array based on num. Example: if num = 5, each element will be shifted 5 places forwards. Elements at the end of the array will be put at the front. - ''' + """ return np.roll(arr, num) def trim_interpolated(self, arr, steps): - ''' + """ Converts the curve back to the original number of data points (8760) by taking the average of X data points before and X data points after each hour (where X = INTERPOLATION_STEPS) @@ -104,9 +110,9 @@ def trim_interpolated(self, arr, steps): points. Trimming it results in a curve with 8760 data points, where each data point (hour) is the average of 30 minutes before and after the whole hour. - ''' - arr = self.shift_curve(arr, self.interpolation_steps//2) - return [sum(arr[i:(i+steps)])/steps for i in range(0, len(arr), steps)] + """ + arr = self.shift_curve(arr, self.interpolation_steps // 2) + return [sum(arr[i : (i + steps)]) / steps for i in range(0, len(arr), steps)] def calculate_smoothed_demand(self, heat_demand, insulation_type): """calculate smoothed demand""" @@ -116,24 +122,24 @@ def calculate_smoothed_demand(self, heat_demand, insulation_type): # generate random numbers deviations = self.generate_deviations( - self.number_of_houses, self.hours_shifted[insulation_type]) + self.number_of_houses, self.hours_shifted[insulation_type] + ) # interpolate the demand curve to increase the number of data points # (i.e. reduce the time interval 1 hour to e.g. 6 minutes) - interpolated_demand = self.interpolate( - heat_demand, self.interpolation_steps) + interpolated_demand = self.interpolate(heat_demand, self.interpolation_steps) # for each random number, shift the demand curve X places forwards or # backwards (depending on the number value) and add it to the # cumulative demand array for num in deviations: demand_list = self.shift_curve(interpolated_demand, num) - cumulative_demand = [ - x + y for x, y in zip(cumulative_demand, demand_list)] + cumulative_demand = [x + y for x, y in zip(cumulative_demand, demand_list)] # Trim the cumulative demand array such that it has 8760 data points again # (hourly intervals instead of 6 minute intervals) smoothed_demand = self.trim_interpolated( - cumulative_demand, self.interpolation_steps) + cumulative_demand, self.interpolation_steps + ) return smoothed_demand diff --git a/src/pyetm/profiles/weather/__init__.py b/src/pyetm/profiles/weather/__init__.py deleted file mode 100644 index 851cfb2..0000000 --- a/src/pyetm/profiles/weather/__init__.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Initialize weather profile module""" - -from __future__ import annotations - -from pyetm.utils.profiles import validate_profile - -import pandas as pd - -from .buildings import Buildings -from .households import HousePortfolio -# from .cooling import Cooling - - -class WeatherDemandProfiles: - """Weather-related profile generator.""" - - @classmethod - def from_defaults(cls, name: str = 'default') -> WeatherDemandProfiles: - """Initialize with Quintel default settings.""" - - # default object configurations - households = HousePortfolio.from_defaults() - buildings = Buildings.from_defaults() - - return cls(households, buildings, name=name) - - def __init__( - self, - households: HousePortfolio, - buildings: Buildings, - name : str | None = None - ) -> WeatherDemandProfiles: - """Initialize class object. - - Parameters - ---------- - households : HousePortolio - HousePortfolio object. - buildings : Buildings - Buidlings object. - name : str, default None - Name of object.""" - - # set name - self.name = name - - # set parameters - self.households = households - self.buildings = buildings - - def __enter__(self): - return self - - def __exit__(self, *args, **kwargs): - return None - - def __str__(self) -> str: - """String name""" - return f"WeatherProfiles(name={self.name})" - - def make_demand_profiles( - self, - temperature: pd.Series, - irradiance: pd.Series, - wind_speed : pd.Series, - year: int | None = None - ) -> pd.DataFrame: - """weather related profiles""" - - # validate temperature profile - temperature = validate_profile( - temperature, name='temperature', year=year) - - # validate irradiance profile - irradiance = validate_profile( - irradiance, name='irradiance', year=year) - - # validate wind_speed profile - wind_speed = validate_profile( - wind_speed, name='wind_speed', year=year) - - # check for allignment - if not irradiance.index.equals(wind_speed.index): - raise ValueError("Periods or Datetimes of 'irradiance' " - "and 'wind_speed' profiles are not alligned.") - - # make household heat demand profiles - households = self.households.make_heat_demand_profiles( - temperature, irradiance, year) - - # make buildings heat demand profile - buildings = self.buildings.make_heat_demand_profile( - temperature, wind_speed, year) - - # make air temperature - key = 'weather/air_temperature' - temperature = pd.Series(temperature, name=key) - - # add agriculture - key = 'weather/agriculture_heating' - agriculture = pd.Series(buildings, name=key) - - # merge profiles - profiles = pd.concat( - [agriculture, temperature, buildings, households], axis=1) - - return profiles.sort_index(axis=1) diff --git a/src/pyetm/sessions/__init__.py b/src/pyetm/sessions/__init__.py index ddd94d5..3c28545 100644 --- a/src/pyetm/sessions/__init__.py +++ b/src/pyetm/sessions/__init__.py @@ -1,3 +1,5 @@ """init sessions module""" from .aiohttp import AIOHTTPSession from .requests import RequestsSession + +__all__ = ["AIOHTTPSession", "RequestsSession"] diff --git a/src/pyetm/sessions/abc.py b/src/pyetm/sessions/abc.py new file mode 100644 index 0000000..77b1ffb --- /dev/null +++ b/src/pyetm/sessions/abc.py @@ -0,0 +1,330 @@ +"""Base Session""" +from __future__ import annotations +from abc import ABC, abstractmethod +from io import BytesIO +from typing import Any, Literal, Mapping, overload +from urllib.parse import urljoin + +import re + +import pandas as pd + +from pyetm.exceptions import UnprossesableEntityError +from pyetm.types import ContentType, Method +from pyetm.utils.general import mapping_to_str + + +class SessionABC(ABC): + """Session abstract base class for properties and methods + accessed by ETM Client object.""" + + @property + @abstractmethod + def headers(self) -> dict[str, str]: + """headers send with each request""" + + @headers.setter + @abstractmethod + def headers(self, headers: dict[str, str] | None): + pass + + @abstractmethod + def connect(self) -> Any: + """open session connection""" + + @abstractmethod + def close(self) -> None: + """close session connection""" + + @abstractmethod + def delete( + self, url: str, headers: Mapping[str, str] | None = None + ) -> dict[str, Any]: + """delete request""" + + @abstractmethod + @overload + def get( + self, + url: str, + content_type: Literal["application/json"], + params: Mapping[str, str | int] | None = None, + headers: Mapping[str, str] | None = None, + ) -> dict[str, Any]: + pass + + @abstractmethod + @overload + def get( + self, + url: str, + content_type: Literal["text/csv"], + params: Mapping[str, str | int] | None = None, + headers: Mapping[str, str] | None = None, + ) -> BytesIO: + pass + + @abstractmethod + @overload + def get( + self, + url: str, + content_type: Literal["text/html"], + params: Mapping[str, str | int] | None = None, + headers: Mapping[str, str] | None = None, + ) -> str: + pass + + @abstractmethod + def get( + self, + url: str, + content_type: ContentType, + params: Mapping[str, str | int] | None = None, + headers: Mapping[str, str] | None = None, + ) -> dict[str, Any] | BytesIO | str: + """get request""" + + @abstractmethod + def post( + self, + url: str, + json: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + ) -> dict[str, Any]: + """post request""" + + @abstractmethod + def put( + self, + url: str, + json: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + ) -> dict[str, Any]: + """put request""" + + @abstractmethod + def upload( + self, url: str, series: pd.Series, filename: str | None = None + ) -> dict[str, Any]: + """upload series request""" + + @staticmethod + def make_url(base: str, url: str | None, allow_fragments: bool = True): + """join base url with relative path""" + return urljoin(base, url, allow_fragments) + + +class SessionTemplate(SessionABC): + """Predefined session template that is adopted + by the default session objects in pyetm.""" + + @property + def context(self): + """passed session environment parameters""" + + # check for attribute + if not hasattr(self, "_context"): + self.context = None + + return self._context + + @context.setter + def context(self, env: dict[str, Any] | None): + self._context = dict(env) if env else {} + + @property + def kwargs(self): + """passed session kwargs""" + + # check for attribute + if not hasattr(self, "_kwargs"): + self.kwargs = None + + return self._kwargs + + @kwargs.setter + def kwargs(self, kwargs: dict[str, Any] | None): + self._kwargs = dict(kwargs) if kwargs else {} + + def __str__(self): + return self.__class__.__name__ + + def __repr__(self): + return f"{self}({mapping_to_str({**self.kwargs, **self.context})})" + + def __enter__(self): + self.connect() + + def __exit__(self, *args, **kwargs): + self.close() + + @property + def headers(self): + """headers that are passed in each request""" + + # use default headers + if not hasattr(self, "_headers"): + self.headers = None + + return self._headers + + @headers.setter + def headers(self, headers: dict[str, str] | None) -> None: + self._headers = dict(headers) if headers else {} + + def delete( + self, + url: str, + headers: Mapping[str, str] | None = None, + ): + """delete request""" + return self.request( + method="delete", + url=url, + content_type="text/html", + headers=headers, + ) + + def get( + self, + url: str, + content_type: ContentType, + params: Mapping[str, str | int] | None = None, + headers: Mapping[str, str] | None = None, + ): + """get request""" + return self.request( + method="get", + url=url, + content_type=content_type, + headers=headers, + params=params, + ) + + def post( + self, + url: str, + json: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + ): + """post request""" + return self.request( + method="post", + url=url, + content_type="application/json", + json=json, + headers=headers, + ) + + def put( + self, + url: str, + json: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + ): + """put request""" + return self.request( + method="put", + url=url, + content_type="application/json", + json=json, + headers=headers, + ) + + @abstractmethod + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["application/json"], + **kwargs, + ) -> dict[str, Any]: + pass + + @abstractmethod + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["text/csv"], + **kwargs, + ) -> BytesIO: + pass + + @abstractmethod + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["text/html"], + **kwargs, + ) -> str: + pass + + @abstractmethod + def request( + self, method: Method, url: str, content_type: ContentType, **kwargs + ) -> Any | dict[str, Any] | BytesIO | str: + """make request""" + + def merge_headers(self, headers: dict[str, str] | None): + """merge headers""" + + # no additional headers + if headers is None: + return self.headers + + return {**headers, **self.headers} + + def raise_for_api_error(self, message: dict[str, str]): + """format API returned error messages""" + + # newlist + errs = [] + + # iterate over messages + for error in message["errors"]: + # format share group errors + if "group does not balance" in error: + error = self.format_share_group_error(error) + + # append to list + errs.append(error) + + # make final message + base = "ETEngine returned the following error(s):" + msg = """%s\n > {}""".format("\n > ".join(errs)) % base + + # format error messages(s) + raise UnprossesableEntityError(msg) + + @staticmethod + def format_share_group_error(error: str) -> str: + """apply more readable format to share group + errors messages""" + + # find share group + pattern = re.compile('"[a-z_]*"') + group: str = pattern.findall(error)[0] + + # find group total + pattern = re.compile(r"\d*[.]\d*") + group_sum = pattern.findall(error)[0] + + # reformat message + group = group.replace('"', "'") + group = f"Share_group {group} sums to {group_sum}" + + # find parameters in group + pattern = re.compile("[a-z_]*=[0-9.]*") + items: list[str] = pattern.findall(error) + + # reformat message + items = [item.replace("=", "': ") for item in items] + msg = "'" + ",\n '".join(items) + + return f"""{group}\n {{{msg}}}""" diff --git a/src/pyetm/sessions/aiohttp.py b/src/pyetm/sessions/aiohttp.py index 6852d23..0da531e 100644 --- a/src/pyetm/sessions/aiohttp.py +++ b/src/pyetm/sessions/aiohttp.py @@ -1,91 +1,51 @@ """aiohttp session object""" from __future__ import annotations -import io -import json -import asyncio +from io import BytesIO +from typing import Any, Literal, Mapping, overload, TYPE_CHECKING -from urllib.parse import urljoin -from collections.abc import Mapping -from typing import TYPE_CHECKING, Literal +import asyncio -from pyetm.utils.loop import _LOOP, _LOOP_THREAD -from pyetm.logger import get_modulelogger -from pyetm.optional import import_optional_dependency -from pyetm.exceptions import UnprossesableEntityError, format_error_messages +from typing_extensions import Self import pandas as pd -if TYPE_CHECKING: - - import ssl - import aiohttp +from pyetm.optional import import_optional_dependency +from pyetm.sessions.abc import SessionTemplate +from pyetm.types import ContentType, Method +from pyetm.utils.loop import _loop, _loop_thread +if TYPE_CHECKING: from yarl import URL - -logger = get_modulelogger(__name__) - -Decoder = Literal['bytes', 'BytesIO', 'json', 'text'] -Method = Literal['delete', 'get', 'post', 'put'] + from ssl import SSLContext + from aiohttp import ClientSession, FormData, Fingerprint, BasicAuth -class AIOHTTPSession: - """aiohttp-based session object""" +class AIOHTTPSession(SessionTemplate): + """aiohttps based adaptation""" @property def loop(self): """used event loop""" - return _LOOP + return _loop @property def loop_thread(self): """seperate thread for event loop""" - return _LOOP_THREAD - - @property - def base_url(self) -> str | None: - """base url used in make_url""" - return self.__base_url + return _loop_thread - @base_url.setter - def base_url(self, base_url: str | None) -> None: - - if base_url is None: - self.__base_url = base_url - - if base_url is not None: - self.__base_url = str(base_url) - - @property - def headers(self) -> dict: - """headers that are passed in each request""" - return self.__headers - - @headers.setter - def headers(self, headers: dict | None) -> None: - - if headers is None: - headers = {} - - self.__headers = dict(headers) - - def __init__(self, base_url: str | None = None, - headers: dict | None = None, + def __init__( + self, proxy: str | URL | None = None, - proxy_auth: aiohttp.BasicAuth | None = None, - ssl: ssl.SSLContext | bool | aiohttp.Fingerprint | None = None, + proxy_auth: BasicAuth | None = None, + ssl: SSLContext | bool | Fingerprint | None = None, proxy_headers: Mapping | None = None, - trust_env: bool = False): + trust_env: bool = False, + ): """session object for pyETM clients Parameters ---------- - base_url: str, default None - Base url to which the session connects, all request urls - will be merged with the base url to create a destination. - headers : dict, default None - Headers that are always passed during requests, e.g. an - authorization token. proxy: str or URL, default None Proxy URL proxy_auth : aiohttp.BasicAuth, default None @@ -102,252 +62,198 @@ def __init__(self, base_url: str | None = None, Should get proxies information from HTTP_PROXY / HTTPS_PROXY environment variables or ~/.netrc file if present.""" - # set parameters - self.base_url = base_url - self.headers = headers - # set environment kwargs for session construction - self._session_env = { - "trust_env": trust_env} + self.context = {"trust_env": trust_env} # set environment kwargs for method requests - self._request_env = { - "proxy": proxy, "proxy_auth": proxy_auth, - "ssl": ssl, "proxy_headers": proxy_headers} + self.kwargs = { + "proxy": proxy, + "proxy_auth": proxy_auth, + "ssl": ssl, + "proxy_headers": proxy_headers, + } # start loop thread if not already running if not self.loop_thread.is_alive(): self.loop_thread.start() - # set session - self._session = None + # # set session + self._session: ClientSession | None = None # start loop thread if not already running if not self.loop_thread.is_alive(): self.loop_thread.start() - def __repr__(self): - """reprodcution string""" - - # object environment - env = ", ".join(f'{k}={v}' for k, v in { - **self._request_env, **self._session_env}.items()) - - return f"AIOHTTPSession({env})" - - def __str__(self): - """stringname""" - return 'AIOHTTPSession' - - def __enter__(self): - """enter context manager""" - - # create session - self.connect() + async def __aenter__(self): + """enter async context manager""" + # start up session + await self.connect_async() return self - def __exit__(self, *args, **kwargs): - """exit context manager""" - - # close session - self.close() - - def make_url(self, url: str | None = None): - """join url with base url""" - return urljoin(self.base_url, url) + async def __aexit__(self, *args, **kwargs): + """exit async context manager""" + await self.close_async() - def connect(self): - """connect session""" + def connect(self) -> Self: + """sync wrapper for async session connect""" # specify coroutine and get future - coro = self.async_connect() + coro = self.connect_async() asyncio.run_coroutine_threadsafe(coro, self.loop).result() return self - def close(self): - """close session""" + async def connect_async(self): + """async session connect""" - # specify coroutine and get future - coro = self.async_close() - asyncio.run_coroutine_threadsafe(coro, self.loop).result() + # import module and create session + aiohttp = import_optional_dependency("aiohttp") + self._session = aiohttp.ClientSession(**self.context) - def request(self, method: Method, url: str, - decoder: Decoder = "bytes", **kwargs): - """request and handle api response""" + def close(self): + """sync wrapper for async session close""" # specify coroutine and get future - coro = self.async_request(method, url, decoder, **kwargs) - future = asyncio.run_coroutine_threadsafe(coro, self.loop) - - return future.result() - - def delete(self, url: str | None = None, - decoder: Decoder = 'text', **kwargs): - """delete request""" - return self.request("delete", self.make_url(url), decoder, **kwargs) + coro = self.close_async() + asyncio.run_coroutine_threadsafe(coro, self.loop).result() - def get(self, url: str | None = None, - decoder: Decoder = 'json', **kwargs): - """get request""" - return self.request("get", self.make_url(url), decoder, **kwargs) + async def close_async(self): + """async session close""" - def post(self, url: str | None = None, - decoder: Decoder = 'json', **kwargs): - """post request""" - return self.request("post", self.make_url(url), decoder, **kwargs) + # await session close + if self._session is not None: + await self._session.close() - def put(self, url: str | None = None, - decoder: Decoder = 'json', **kwargs): - """put request""" - return self.request("put", self.make_url(url), decoder, **kwargs) + # reset session + self._session = None - def upload_series(self, url: str | None = None, - series: pd.Series | None = None, name: str | None = None, **kwargs): - """upload series object""" + def upload( + self, + url: str, + series: pd.Series, + filename: str | None = None, + ) -> dict[str, Any]: + """upload series""" # optional module import - aiohttp = import_optional_dependency('aiohttp') - - # default to empty series - if series is None: - series = pd.Series() + aiohttp = import_optional_dependency("aiohttp") # set key as name - if name is None: - name = "not specified" + if filename is None: + filename = "filename not specified" - # convert values to string + # convert series to string data = series.to_string(index=False) # insert data in form - form = aiohttp.FormData() - form.add_field("file", data, filename=name) - - return self.put(url, data=form, **kwargs) - - async def __aenter__(self): - """enter async context manager""" - - # start up session - await self.async_connect() - - return self - - async def __aexit__(self, *args, **kwargs) -> None: - """exit async context manager""" - await self.async_close() - - async def async_connect(self): - """connect session""" - - # optional module import - aiohttp = import_optional_dependency('aiohttp') - - # create session - self._session = aiohttp.ClientSession(**self._session_env) - - async def async_close(self): - """close session""" - - # close and remove session - await self._session.close() - self._session = None - - async def async_request(self, method: Method, url: str, - decoder: Decoder = 'bytes', **kwargs): + form: FormData = aiohttp.FormData() + form.add_field("file", data, filename=filename) + + return self.request( + method="put", url=url, content_type="application/json", data=form + ) + + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["application/json"], + **kwargs, + ) -> dict[str, Any]: + pass + + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["text/csv"], + **kwargs, + ) -> BytesIO: + pass + + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["text/html"], + **kwargs, + ) -> str: + pass + + def request( + self, + method: Method, + url: str, + content_type: ContentType, + **kwargs, + ) -> dict[str, Any] | BytesIO | str: """make request to api session""" - # optional module import - aiohttp = import_optional_dependency('aiohttp') - - retries = 5 - while retries: - - try: - - # merge kwargs with session envioronment kwargs - kwargs = {**self._request_env, **kwargs} - - # add persistent headers - headers = kwargs.get('headers', {}) - kwargs['headers'] = {**headers, **self.headers} - - # reusable existing session - session = bool(self._session) - - # create session - if not session: - await self.async_connect() - - # make method request - request = getattr(self._session, method) - async with request(url, **kwargs) as resp: - - # check response - if not resp.status <= 400: - - # report error messages - if resp.status == 422: - await self._error_report(resp) - - # raise for status - resp.raise_for_status() - - # bytes decoding - if decoder == "bytes": - resp = await resp.read() - - # bytes as BytesIO - elif decoder == "BytesIO": - byts = await resp.read() - resp = io.BytesIO(byts) - - # json decoding - elif decoder == "json": - resp = await resp.json(encoding="utf-8") - - # text decoding - elif decoder == "text": - resp = await resp.text(encoding="utf-8") + # specify coroutine and get future + coro = self.make_async_request(method, url, content_type, **kwargs) + future = asyncio.run_coroutine_threadsafe(coro, self.loop) - else: - msg = f"decoding method '{decoder}' not implemented" - raise NotImplementedError(msg) + return future.result() - return resp + async def make_async_request( + self, + method: Method, + url: str, + content_type: ContentType, + **kwargs, + ): + """make request to api session""" - # except connectionerrors and retry - except aiohttp.ClientConnectorError as error: - retries -= 1 + # reusable existing session + session = bool(self._session) - # raise after retries - if not retries: - raise error + # create session + if not session: + await self.connect_async() - finally: + # merge base and request headers + headers = kwargs.get("headers") + kwargs["headers"] = self.merge_headers(headers) - # close session - if not session: - await self.async_close() + # merge base and request kwargs + kwargs = {**self.kwargs, **kwargs} - async def _error_report(self, resp: aiohttp.ClientResponse) -> None: - """create error report when api returns error messages.""" + # get request method + request = getattr(self._session, method) try: - - # attempt decode error message(s) - msg = await resp.json(encoding="utf-8") - errors = msg.get("errors") - - except json.decoder.JSONDecodeError: - - # no message returned - errors = None - - if errors: - - # format error message(s) - msg = format_error_messages(errors) - raise UnprossesableEntityError(msg) + # make request + async with request(url=url, **kwargs) as response: + # handle engine error message + if response.status == 422: + # raise for api error + self.raise_for_api_error(await response.json(encoding="utf-8")) + + # handle other error messages + response.raise_for_status() + + # decode application/json + if content_type == "application/json": + json: dict[str, Any] = await response.json(encoding="utf-8") + return json + + # decode text/csv + if content_type == "text/csv": + content: bytes = await response.read() + return BytesIO(content) + + # decode text/html + if content_type == "text/html": + text: str = await response.text(encoding="utf-8") + return text + + raise NotImplementedError(f"Content-type '{content_type}' not implemented") + + finally: + # handle session + if not session: + await self.close_async() diff --git a/src/pyetm/sessions/requests.py b/src/pyetm/sessions/requests.py index 28b2500..ba6bfc8 100644 --- a/src/pyetm/sessions/requests.py +++ b/src/pyetm/sessions/requests.py @@ -1,65 +1,29 @@ """requests session object""" from __future__ import annotations +from io import BytesIO +from typing import Any, overload, Literal -import io -import json - -from urllib.parse import urljoin -from typing import Literal - -from pyetm.exceptions import UnprossesableEntityError, format_error_messages - -import requests import pandas as pd +import requests -Decoder = Literal['bytes', 'BytesIO', 'json', 'text'] -Method = Literal['delete', 'get', 'post', 'put'] - - -class RequestsSession: - """requests-based session object""" - - @property - def base_url(self) -> str | None: - """base url used in make_url""" - return self.__base_url - - @base_url.setter - def base_url(self, base_url: str | None) -> None: - - if base_url is None: - self.__base_url = base_url - - if base_url is not None: - self.__base_url = str(base_url) - - @property - def headers(self) -> dict: - """headers that are passed in each request""" - return self.__headers - - @headers.setter - def headers(self, headers: dict | None) -> None: +from pyetm.sessions.abc import SessionTemplate +from pyetm.types import ContentType, Method - if headers is None: - headers = {} - self.__headers = dict(headers) +class RequestsSession(SessionTemplate): + """requests bases adaptation""" - def __init__(self, base_url: str | None = None, - headers: dict | None = None, proxies: dict | None = None, - stream: bool = False, verify: bool | str = True, - cert: str | tuple | None = None): + def __init__( + self, + proxies: dict | None = None, + stream: bool = False, + verify: bool | str = True, + cert: str | tuple | None = None, + ): """session object for pyETM clients Parameters ---------- - base_url: str, default None - Base url to which the session connects, all request urls - will be merged with the base url to create a destination. - headers : dict, default None - Headers that are always passed during requests, e.g. an - authorization token. proxies: dict, default None Dictionary mapping protocol or protocol and hostname to the URL of the proxy. @@ -78,170 +42,114 @@ def __init__(self, base_url: str | None = None, If string; path to ssl client cert file (.pem). If tuple; ('cert', 'key') pair.""" - # set parameters - self.base_url = base_url - self.headers = headers - # set environment kwargs for method requests - self._request_env = { - "proxies": proxies, "stream": stream, - "verify": verify, "cert": cert} + self.kwargs = { + "proxies": proxies, + "stream": stream, + "verify": verify, + "cert": cert, + } # set session self._session = requests.Session() - def __repr__(self): - """reproduction string""" - - # object environment - env = ", ".join(f"{k}={str(v)}" for k, v in - self._request_env.items()) - - return f"RequestsSession({env})" - - def __str__(self): - """stringname""" - return 'RequestsSession' - - def __enter__(self) -> RequestsSession: - """enter context manager""" - - # connect session - self.connect() - - return self - - def __exit__(self, *args, **kwargs) -> None: - """exit context manager""" - - # close session - self.close() - - def make_url(self, url: str | None = None): - """join url with base url""" - return urljoin(self.base_url, url) - def connect(self): """connect session""" def close(self): """close session""" - def request(self, method: Method, url: str, - decoder: Decoder = 'bytes', **kwargs): - """make request to api session""" - - retries = 5 - while retries: - - try: + def upload( + self, + url: str, + series: pd.Series, + filename: str | None = None, + ): + """upload series""" - # merge kwargs with session envioronment kwargs - kwargs = {**self._request_env, **kwargs} - - # add persistent headers - headers = kwargs.get('headers', {}) - kwargs['headers'] = {**headers, **self.headers} - - # make method request - request = getattr(self._session, method) - with request(url, **kwargs) as resp: - - # check response - if not resp.ok: - - # get debug message - if resp.status_code == 422: - self._error_report(resp) - - # raise for status - resp.raise_for_status() - - # bytes decoding - if decoder == "bytes": - resp = resp.content - - # bytes as BytesIO - elif decoder == "BytesIO": - byts = resp.content - resp = io.BytesIO(byts) - - # json decoding - elif decoder == "json": - resp = resp.json() - - # text decoding - elif decoder == "text": - resp = resp.text - - else: - msg = f"decoding method '{decoder}' not implemented" - raise NotImplementedError(msg) - - return resp - - # except connectionerrors and retry - except requests.exceptions.ConnectionError as error: - retries -= 1 - - # raise after retries - if not retries: - raise error - - def _error_report(self, resp: requests.Response) -> None: - """create error report when api returns error messages.""" - - try: - - # attempt decode error message(s) - msg = resp.json() - errors = msg.get("errors") - - except json.decoder.JSONDecodeError: - - # no message returned - errors = None - - if errors: + # set key as name + if filename is None: + filename = "filename not specified" - # format error message(s) - msg = format_error_messages(errors) - raise UnprossesableEntityError(msg) + # convert series to string + data = series.to_string(index=False) + form = {"file": (filename, data)} + + return self.request( + method="put", url=url, content_type="application/json", files=form + ) + + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["application/json"], + **kwargs, + ) -> dict[str, Any]: + pass + + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["text/csv"], + **kwargs, + ) -> BytesIO: + pass + + @overload + def request( + self, + method: Method, + url: str, + content_type: Literal["text/html"], + **kwargs, + ) -> str: + pass + + def request( + self, + method: Method, + url: str | None, + content_type: ContentType, + **kwargs, + ): + """make request to api session""" - def delete(self, url: str | None = None, - decoder: Decoder = 'text', **kwargs): - """delete request""" - return self.request("delete", self.make_url(url), decoder, **kwargs) + # merge base and request headers + headers = kwargs.get("headers") + kwargs["headers"] = self.merge_headers(headers) - def get(self, url: str | None = None, - decoder: Decoder = 'json', **kwargs): - """get request""" - return self.request("get", self.make_url(url), decoder, **kwargs) + # merge base and request kwargs + kwargs = {**self.kwargs, **kwargs} - def post(self, url: str | None = None, - decoder: Decoder = 'json', **kwargs): - """post request""" - return self.request("post", self.make_url(url), decoder, **kwargs) + # get request method + request = getattr(self._session, method) - def put(self, url: str | None = None, - decoder: Decoder = 'json', **kwargs): - """put request""" - return self.request("put", self.make_url(url), decoder, **kwargs) + # make request + with request(url=url, **kwargs) as response: + # handle engine error message + if response.status_code == 422: + return self.raise_for_api_error(response.json()) - def upload_series(self, url: str | None = None, - series: pd.Series | None = None, name: str | None = None, **kwargs): - """upload series object""" + # handle other error messages + response.raise_for_status() - # default to empty series - if series is None: - series = pd.Series() + # decode application/json + if content_type == "application/json": + json: dict[str, Any] = response.json() + return json - # set key as name - if name is None: - name = "not specified" + # decode text/csv + if content_type == "text/csv": + content: bytes = response.content + return BytesIO(content) - # convert series to string - data = series.to_string(index=False) - form = {"file": (name, data)} + # decode text/html + if content_type == "text/html": + text: str = response.text + return text - return self.put(url=url, files=form, **kwargs) + raise NotImplementedError(f"Content-type '{content_type}' not implemented") diff --git a/src/pyetm/types.py b/src/pyetm/types.py new file mode 100644 index 0000000..4285591 --- /dev/null +++ b/src/pyetm/types.py @@ -0,0 +1,78 @@ +"""module defined types""" +from __future__ import annotations +from typing import Literal + + +Carrier = Literal["electricity", "heat", "hydrogen", "methane"] + +ContentType = Literal["application/json", "text/csv", "text/html"] + +ErrorHandling = Literal["ignore", "warn", "raise"] + +Method = Literal["delete", "get", "post", "put"] + +TokenScope = Literal[ + "openid", "public", "scenarios:read", "scenarios:write", "scenarios:delete" +] + +Endpoint = Literal[ + "scenarios", + "scenario_id", + "curves", + "custom_curves", + "inputs", + "merit_configuration", + "user", + "transition_paths", + "token", + "saved_scenarios", +] + +# copied from pandas._typing +InterpolateOptions = Literal[ + "linear", + "time", + "index", + "values", + "nearest", + "zero", + "slinear", + "quadratic", + "cubic", + "barycentric", + "polynomial", + "krogh", + "piecewise_polynomial", + "spline", + "pchip", + "akima", + "cubicspline", + "from_derivatives", +] + +# class ETMTyped: +# """module defined object base class""" + + +# class HourlyElectricityPriceCurve(pd.Series, ETMTyped): +# """hourly electricity price curve""" + + +# class HourlyElectricityCurves(pd.DataFrame, ETMTyped): +# """hourly electricity curves""" + + +# class HourlyHydrogenCurves(pd.DataFrame, ETMTyped): +# """hourly hydrogen curves""" + + +# class HourlyMethaneCurves(pd.DataFrame, ETMTyped): +# """hourly methane curves""" + + +# class HourlyHeatCurves(pd.DataFrame, ETMTyped): +# """hourly heat curves""" + + +# class HourlyHouseholdCurves(pd.DataFrame, ETMTyped): +# """hourly household curves""" diff --git a/src/pyetm/utils/__init__.py b/src/pyetm/utils/__init__.py index 61e0986..16d6c25 100644 --- a/src/pyetm/utils/__init__.py +++ b/src/pyetm/utils/__init__.py @@ -1,6 +1,14 @@ """init util module""" from .categorisation import categorise_curves from .excel import add_frame, add_series -# from .interpolation import Interpolator, interpolate_clients from .lookup import lookup_coordinates from .regionalisation import regionalise_curves, regionalise_node + +__all__ = [ + "categorise_curves", + "add_frame", + "add_series", + "lookup_coordinates", + "regionalise_curves", + "regionalise_node", +] diff --git a/src/pyetm/utils/categorisation.py b/src/pyetm/utils/categorisation.py index 9619be2..61596e9 100644 --- a/src/pyetm/utils/categorisation.py +++ b/src/pyetm/utils/categorisation.py @@ -1,16 +1,68 @@ """categorisation method""" from __future__ import annotations - +from typing import Iterable import pandas as pd + from pyetm.logger import get_modulelogger +from pyetm.types import ErrorHandling +from pyetm.utils.general import iterable_to_str logger = get_modulelogger(__name__) -def categorise_curves(curves: pd.DataFrame, - mapping: pd.DataFrame | str, columns: list[str] | None =None, - include_keys: bool = False, invert_sign: bool = False, - pattern_level: str | int | None = None, **kwargs) -> pd.DataFrame: +def assigin_sign_convention( + curves: pd.DataFrame, invert_sign: bool = False +) -> pd.DataFrame: + """assign sign convention to hourly curves""" + + # ensure default convention + curves = curves.abs() + + # subset cols for sign convention + pattern = ".output (MW)" if invert_sign else ".input (MW)" + cols = curves.columns.get_level_values(level=-1).str.contains(pattern, regex=False) + + # apply sign convention + curves.loc[:, cols] = -curves.loc[:, cols] + + return curves.replace(-0, 0) + + +def validate_categorisation( + curves: pd.DataFrame, + mapping: pd.Series[str] | pd.DataFrame, + errors: ErrorHandling = "warn", +) -> None: + """validate categorisation""" + + # check if passed curves contains columns not specified in cat + missing_curves = curves.columns[~curves.columns.isin(mapping.index)] + if not missing_curves.empty: + # make message + errors = "', '".join(map(str, missing_curves)) + message = f"Missing key(s) in mapping: '{errors}'" + + raise KeyError(message) + + # check if cat specifies keys not in passed curves + superfluous_curves = mapping.index[~mapping.index.isin(curves.columns)] + if not superfluous_curves.empty: + if errors == "warn": + for key in superfluous_curves: + logger.warning("Unused key in mapping: %s", key) + + if errors == "raise": + error = iterable_to_str(superfluous_curves) + raise ValueError(f"Unsued key(s) in mapping: {error}") + + +def categorise_curves( + curves: pd.DataFrame, + mapping: pd.Series[str] | pd.DataFrame, + columns: str | Iterable[str] | None = None, + include_keys: bool = False, + invert_sign: bool = False, +) -> pd.DataFrame: """Categorize the hourly curves for a specific dataframe with a specific mapping. @@ -23,10 +75,9 @@ def categorise_curves(curves: pd.DataFrame, curves : DataFrame The hourly curves for which the categorization is applied. - mapping : DataFrame or str + mapping : DataFrame DataFrame with mapping of ETM keys in index and mapping - values in columns. Alternatively a string to a csv-file - can be passed. + values in columns. columns : list, default None List of column names and order that will be included in the mapping. Defaults to all columns in mapping. @@ -36,97 +87,53 @@ def categorise_curves(curves: pd.DataFrame, Inverts sign convention where demand is denoted with a negative sign. Demand will be denoted with a positve value and supply with a negative value. - pattern_level : str or int, default None - Column level in which sign convention pattern is located. - Assumes last level by default. - - **kwargs are passed to pd.read_csv when a filename is - passed in the mapping argument. Return ------ curves : DataFrame DataFrame with the categorized curves of the - specified carrier. - """ + specified carrier.""" + # copy curves curves = curves.copy() - # load categorization - if isinstance(mapping, str): - mapping = pd.read_csv(mapping, **kwargs) - if isinstance(mapping, pd.Series): - mapping = mapping.to_frame() - columns = mapping.columns + columns = mapping.to_frame().columns if curves.columns.nlevels != mapping.index.nlevels: - raise ValueError( - "Index levels of 'curves' and 'mapping' are not alligned") + raise ValueError("Index levels of 'curves' and 'mapping' are not alligned") - # check if passed curves contains columns not specified in cat - missing_curves = curves.columns[~curves.columns.isin(mapping.index)] - if not missing_curves.empty: - - # make message - missing_curves = "', '".join(map(str, missing_curves)) - message = f"Missing key(s) in mapping: '{missing_curves}'" - - raise KeyError(message) - - # check if cat specifies keys not in passed curves - superfluous_curves = mapping.index[~mapping.index.isin(curves.columns)] - if not superfluous_curves.empty: - - # make message - superfluous_curves = "', '".join(map(str, superfluous_curves)) - message = f"Unused key(s) in mapping: '{superfluous_curves}'" - - logger.warning(message) - - # determine pattern for desired sign convention - pattern = '[.]output [(]MW[)]' if invert_sign else '[.]input [(]MW[)]' - - # assume pattern in last level - if pattern_level is None: - pattern_level = curves.columns.nlevels - 1 - - # subset column positions with pattern - cols = curves.columns.get_level_values( - level=pattern_level).str.contains(pattern) - - # assign sign convention by pattern - curves.loc[:, cols] = -curves.loc[:, cols] + # apply sign convention + validate_categorisation(curves, mapping) + curves = assigin_sign_convention(curves, invert_sign=invert_sign) - # subset columns - if columns is not None: + # default columns + if columns is None: + columns = mapping.columns - # check columns argument - if isinstance(columns, str): - columns = [columns] + # check columns argument + if isinstance(columns, str): + columns = [columns] - # subset categorization - mapping = mapping[columns] + # subset categorization + mapping = mapping.loc[:, columns] # include index in mapping if include_keys is True: - # append index as column to mapping - keys = mapping.index.to_series(name='ETM_key') + keys = mapping.index.to_series(name="ETM_key") mapping = pd.concat([mapping, keys], axis=1) # include levels in mapping if mapping.index.nlevels > 1: - # transform index levels to frame - idx = mapping.index.droplevel(level=pattern_level) + idx = mapping.index.droplevel(level=-1) idx = idx.to_frame(index=False).set_index(mapping.index) # join frame with mapping mapping = pd.concat([idx, mapping], axis=1) if len(mapping.columns) == 1: - # extract column column = columns[0] @@ -138,13 +145,12 @@ def categorise_curves(curves: pd.DataFrame, curves = curves.T.groupby(by=column).sum().T else: - # make mapper for multiindex - names = list(mapping.columns) - mapping = dict(zip(mapping.index, pd.MultiIndex.from_frame(mapping))) + names = mapping.columns + mapper = dict(zip(mapping.index, pd.MultiIndex.from_frame(mapping))) # apply mapping to curves - midx = curves.columns.to_series().map(mapping) + midx = curves.columns.to_series().map(mapper) curves.columns = pd.MultiIndex.from_tuples(midx, names=names) # aggregate over levels diff --git a/src/pyetm/utils/converter.py b/src/pyetm/utils/converter.py index 5904f83..31153fa 100644 --- a/src/pyetm/utils/converter.py +++ b/src/pyetm/utils/converter.py @@ -1,19 +1,30 @@ """conversion methods""" from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING + import pandas as pd from pyetm import Client -from pyetm.myc import MYCClient from pyetm.logger import get_modulelogger +from pyetm.myc import MYCClient from pyetm.optional import import_optional_dependency +from pyetm.utils import add_frame, add_series _logger = get_modulelogger(__name__) -def copy_study_session_ids(session_ids: pd.Series | MYCClient, - study: str | None = None, metadata: dict | None = None, - keep_compatible: bool = False, **kwargs) -> pd.Series: +if TYPE_CHECKING: + pass + + +def copy_study_session_ids( + session_ids: pd.Series | MYCClient, + study: str | None = None, + metadata: dict | None = None, + keep_compatible: bool = False, + **kwargs, +) -> pd.Series: """make a copy of an existing study. The returned session ids are decoupled from the original study, but contain the same values""" @@ -25,15 +36,16 @@ def copy_study_session_ids(session_ids: pd.Series | MYCClient, # make series-like object if not isinstance(session_ids, pd.Series): - session_ids = pd.Series(session_ids, name='SESSION') + session_ids = pd.Series(session_ids, name="SESSION") # helper function def scenario_copy(session_id): """create compatible scenario copy""" # initiate client from existing scenairo - client = Client.from_existing_scenario(session_id, metadata=metadata, - keep_compatible=keep_compatible, **kwargs) + client = Client.from_existing_scenario( + session_id, metadata=metadata, keep_compatible=keep_compatible, **kwargs + ) return client.scenario_id @@ -42,30 +54,23 @@ def scenario_copy(session_id): # set study if applicable if study is not None: - session_ids = session_ids.index.set_levels([study], level='STUDY') + session_ids = session_ids.index.set_levels([study], level="STUDY") return session_ids -def copy_study_configuration(filepath: str, model: MYCClient, - study: str | None = None, copy_session_ids: bool = True, - metadata: dict | None = None, keep_compatible: bool = False) -> None: - """copy study configuration""" - - # pylint: disable=C0415 - # Due to optional import - - from pathlib import Path - from pyetm.utils import add_series, add_frame - if TYPE_CHECKING: - # import xlsxwriter - import xlsxwriter - - else: - # import optional dependency - xlsxwriter = import_optional_dependency('xlsxwriter') +def copy_study_configuration( + filepath: str, + model: MYCClient, + study: str | None = None, + copy_session_ids: bool = True, + metadata: dict | None = None, + keep_compatible: bool = False, +) -> None: + """copy study configuration""" - # pylint: enable-C0415 + # import optional dependency + xlsxwriter = import_optional_dependency("xlsxwriter") # check filepath if not Path(filepath).parent.exists: @@ -76,86 +81,80 @@ def copy_study_configuration(filepath: str, model: MYCClient, # get session ids if copy_session_ids: - # create copies of session ids - sessions = copy_study_session_ids(model, study=study, - metadata=metadata, keep_compatible=keep_compatible) + sessions = copy_study_session_ids( + model, study=study, metadata=metadata, keep_compatible=keep_compatible + ) else: - - _logger.warning("when using 'copy_session_ids=False', the " - + "'keep_compatible' argument is ignored.") + _logger.warning( + "when using 'copy_session_ids=False', the " + + "'keep_compatible' argument is ignored." + ) # keep original sessions = model.session_ids # set study if applicable if study is not None: - sessions = sessions.index.set_levels([study], level='STUDY') + sessions = sessions.index.set_levels([study], level="STUDY") - _logger.warning("'study' passed without copying session_ids, " + - "it is recommended to use 'copy_session_ids=True' instead to " + - "prevent referencing the same session_id by different names.") + _logger.warning( + "'study' passed without copying session_ids, " + + "it is recommended to use 'copy_session_ids=True' instead to " + + "prevent referencing the same session_id by different names." + ) if metadata is not None: - _logger.warning("'metadata' passed without copying session_ids, " + - "use 'copy_session_ids=True' instead.") + _logger.warning( + "'metadata' passed without copying session_ids, " + + "use 'copy_session_ids=True' instead." + ) # add sessions and set column width - add_series('Sessions', sessions, workbook, column_width=18) + add_series("Sessions", sessions, workbook, column_width=18) # add parameters and set column width - add_series('Parameters', model.parameters, workbook, - index_width=80, column_width=18) + add_series( + "Parameters", model.parameters, workbook, index_width=80, column_width=18 + ) # add gqueries and set column width - add_series('GQueries', model.gqueries, workbook, - index_width=80, column_width=18) + add_series("GQueries", model.gqueries, workbook, index_width=80, column_width=18) # add mapping and set column width if model.mapping is not None: - add_frame('Mapping', model.mapping, workbook, - index_width=[80, 18], column_width=18) + add_frame( + "Mapping", model.mapping, workbook, index_width=[80, 18], column_width=18 + ) # copy other tabs from source - if hasattr(model, '_source'): - + if hasattr(model, "_source"): _logger.debug("detected source file") - try: - - """merge together with model to also validate these values - before copying them""" - - # link source file - xlsx = pd.ExcelFile(model._source) - - # look for interconnectors - sheet = 'Interconnectors' - if sheet in xlsx.sheet_names: - - # read and write interconnectors - interconnectors = pd.read_excel(xlsx, sheet, index_col=0) - add_frame(sheet, interconnectors, workbook, column_width=18) - - _logger.debug("> included '%s' in copy", sheet) + """merge together with model to also validate these values + before copying them""" - # look for mpi profiles - sheet = 'MPI Profiles' - if sheet in xlsx.sheet_names: + # link source file + xlsx = pd.ExcelFile(model._source) - # read and write mpi profiles - profiles = pd.read_excel(xlsx, sheet) - add_frame(sheet, profiles, workbook, - index=False, column_width=18) + # look for interconnectors + sheet = "Interconnectors" + if sheet in xlsx.sheet_names: + # read and write interconnectors + interconnectors = pd.read_excel(xlsx, sheet, index_col=0) + add_frame(sheet, interconnectors, workbook, column_width=18) - _logger.debug("> included '%s' in copy", sheet) + _logger.debug("> included '%s' in copy", sheet) - except Exception as exc: + # look for mpi profiles + sheet = "MPI Profiles" + if sheet in xlsx.sheet_names: + # read and write mpi profiles + profiles = pd.read_excel(xlsx, sheet) + add_frame(sheet, profiles, workbook, index=False, column_width=18) - # report failure - _logger.debug("could not process detected source file", - exc_info=exc) + _logger.debug("> included '%s' in copy", sheet) # write workbook workbook.close() diff --git a/src/pyetm/utils/excel.py b/src/pyetm/utils/excel.py index 2fba35a..e935a5e 100644 --- a/src/pyetm/utils/excel.py +++ b/src/pyetm/utils/excel.py @@ -1,62 +1,68 @@ """write excel methods""" from __future__ import annotations +import math from collections.abc import Iterable from typing import TYPE_CHECKING, Literal -import math import numpy as np import pandas as pd - if TYPE_CHECKING: + from xlsxwriter.format import Format from xlsxwriter.workbook import Workbook from xlsxwriter.worksheet import Worksheet - from xlsxwriter.format import Format -def _handle_nans(worksheet: Worksheet, - row: int, col: int, number: float, cell_format=None) -> Literal[-1, 0]: + +def _handle_nans( + worksheet: Worksheet, row: int, col: int, number: float, cell_format=None +) -> Literal[-1, 0]: """handle nan values and convert float to numpy.float64""" # write NaN as NA if np.isnan(number): - return worksheet.write_formula(row, col, '=NA()', cell_format, '#N/A') + return worksheet.write_formula(row, col, "=NA()", cell_format, "#N/A") # set decimal precision number = math.ceil(number * 1e10) / 1e10 return worksheet.write_number(row, col, number, cell_format) + def _has_names(index: pd.Index | pd.MultiIndex) -> bool: """helper to check if index level(s) are named""" - return len(list(index.names)) != list(index.names).count(None) + return len(list(index.names)) != sum(x is not None for x in list(index.names)) -def _set_column_width(worksheet: Worksheet, - columns: pd.Index | pd.MultiIndex, offset: int, - column_width: int | list | None = None) -> None: + +def _set_column_width( + worksheet: Worksheet, + columns: pd.Index | pd.MultiIndex, + offset: int, + column_width: int | list | None = None, +) -> None: """set header column widths in worksheet""" # individual value for all header columns if isinstance(column_width, list): - # check for valid list if len(column_width) != len(columns): raise ValueError("column widths do not match number of columns") # set column widths individually for col_num, col_width in enumerate(column_width): - worksheet.set_column(col_num + offset, - col_num + offset, col_width) + worksheet.set_column(col_num + offset, col_num + offset, col_width) # single value for all header columns if isinstance(column_width, int): worksheet.set_column(offset, len(columns) + offset - 1, column_width) + def _set_index_width( worksheet: Worksheet, index: pd.Index | pd.MultiIndex, index_width: int | list | None = None, - column_width: int | list | None = None) -> None: + column_width: int | list | None = None, +) -> None: """set index column widths in worksheet""" # copy column width @@ -66,7 +72,6 @@ def _set_index_width( # individual value for all index columns if isinstance(index_width, list): - # check for valid list if len(index_width) != index.nlevels: raise ValueError("index widths do not match number of levels") @@ -79,12 +84,15 @@ def _set_index_width( if isinstance(index_width, int): worksheet.set_column(0, index.nlevels - 1, index_width) -def _write_index(worksheet: Worksheet, + +def _write_index( + worksheet: Worksheet, index: pd.Index | pd.MultiIndex, row_offset: int, index_width: int | list | None = None, column_width: int | list | None = None, - cell_format: Format | None = None) -> None: + cell_format: Format | None = None, +) -> None: """write index to worksheet""" # set index widths @@ -97,22 +105,25 @@ def _write_index(worksheet: Worksheet, # write index values if isinstance(index, pd.MultiIndex): - # write index values for multiindex for row_num, row_data in enumerate(index.values): for col_num, cell_data in enumerate(row_data): worksheet.write(row_num + row_offset, col_num, cell_data) else: - # write index values for regular index for row_num, cell_data in enumerate(index.values): worksheet.write(row_num + row_offset, 0, cell_data) -def add_frame(name: str, frame: pd.DataFrame, - workbook: Workbook, index: bool = True, - column_width: int | list | None = None, - index_width: int | list | None = None) -> Worksheet: + +def add_frame( + name: str, + frame: pd.DataFrame, + workbook: Workbook, + index: bool = True, + column_width: int | list | None = None, + index_width: int | list | None = None, +) -> Worksheet: """create worksheet from frame""" # add formats @@ -128,7 +139,6 @@ def add_frame(name: str, frame: pd.DataFrame, # write column values if isinstance(frame.columns, pd.MultiIndex): - # modify offset when index names are specified if _has_names(frame.index) & (index is True): skiprows += 1 @@ -141,11 +151,9 @@ def add_frame(name: str, frame: pd.DataFrame, # write colmns values for multiindex for col_num, col_data in enumerate(frame.columns.values): for row_num, cell_data in enumerate(col_data): - worksheet.write( - row_num, col_num + skipcolumns, cell_data, cell_format) + worksheet.write(row_num, col_num + skipcolumns, cell_data, cell_format) else: - # write column values for regular index for col_num, cell_data in enumerate(frame.columns.values): worksheet.write(0, col_num + skipcolumns, cell_data, cell_format) @@ -153,28 +161,41 @@ def add_frame(name: str, frame: pd.DataFrame, # freeze panes with rows and columns worksheet.freeze_panes(skiprows, skipcolumns) - # set column widths - _set_column_width(worksheet=worksheet, columns=frame.columns, - offset=skipcolumns, column_width=column_width) + # set column widths + _set_column_width( + worksheet=worksheet, + columns=frame.columns, + offset=skipcolumns, + column_width=column_width, + ) # write cell values in numeric format for row_num, row_data in enumerate(frame.values): for col_num, cell_data in enumerate(row_data): - worksheet.write(row_num + skiprows, - col_num + skipcolumns, cell_data) + worksheet.write(row_num + skiprows, col_num + skipcolumns, cell_data) # write index if index is True: - _write_index(worksheet=worksheet, - index=frame.index, row_offset=skiprows, index_width=index_width, - column_width=column_width, cell_format=cell_format) + _write_index( + worksheet=worksheet, + index=frame.index, + row_offset=skiprows, + index_width=index_width, + column_width=column_width, + cell_format=cell_format, + ) return worksheet -def add_series(name: str, series: pd.Series, - workbook: Workbook, index: bool = True, - column_width: int | None = None, - index_width: int | list | None = None) -> Worksheet: + +def add_series( + name: str, + series: pd.Series, + workbook: Workbook, + index: bool = True, + column_width: int | None = None, + index_width: int | list | None = None, +) -> Worksheet: """add series to workbook""" # add formats @@ -189,9 +210,9 @@ def add_series(name: str, series: pd.Series, worksheet.freeze_panes(1, skipcolumns) # handle iterable header - header = series.name + header = str(series.name) if isinstance(header, Iterable) & (not isinstance(header, str)): - header = '_'.join(map(str, header)) + header = "_".join(map(str, header)) # write header and set column width worksheet.write(0, skipcolumns, header, cell_format) @@ -203,8 +224,13 @@ def add_series(name: str, series: pd.Series, # include index if index is True: - _write_index(worksheet=worksheet, - index=series.index, row_offset=1, index_width=index_width, - column_width=column_width, cell_format=cell_format) + _write_index( + worksheet=worksheet, + index=series.index, + row_offset=1, + index_width=index_width, + column_width=column_width, + cell_format=cell_format, + ) return worksheet diff --git a/src/pyetm/utils/general.py b/src/pyetm/utils/general.py new file mode 100644 index 0000000..4f6a3c4 --- /dev/null +++ b/src/pyetm/utils/general.py @@ -0,0 +1,36 @@ +"""general utilities""" +from __future__ import annotations + +import re +from typing import Any, Iterable, Mapping + + +def bool_to_json(boolean: bool): + """convert boolean to json compatible string""" + return str(bool(boolean)).lower() + + +def iterable_to_str(iterable: Iterable[Any]): + """transform list to string""" + return ", ".join(map(str, iterable)) + + +def mapping_to_str(mapping: Mapping[Any, Any]): + """transform mapping to string""" + return ", ".join(f"{key}={value}" for key, value in mapping.items()) + + +def mapped_floats_to_str(mapping: Mapping[str, float | int], prec: int) -> str: + """transform mapping to string with rounding of floats""" + return ", ".join(f"{key}={value:.{prec}f}" for key, value in mapping.items()) + + +def snake_case_name(cls: object) -> str: + """convert camel case name to snake case name""" + + word = cls.__class__.__name__ + word = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", word) + word = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", word) + word = word.replace("-", "_") + + return word.lower() diff --git a/src/pyetm/utils/interpolation.py b/src/pyetm/utils/interpolation.py index 73c2da3..781b58b 100644 --- a/src/pyetm/utils/interpolation.py +++ b/src/pyetm/utils/interpolation.py @@ -1,168 +1,107 @@ -# import numpy -# import pandas -# import pyetm - -# def interpolate_clients(clients, cfill='linear', dfill='ffill'): -# """Interpolates the user values of the years between the -# passed clients. Uses a seperate method for continous and -# discrete user values. - -# Do note that the heat network order is not returned or -# interpolated by this function. - -# Parameters -# ---------- -# clients: list -# List of pyetm.client.Client objects that -# are used to interpolate the scenario. -# cfill : string, default 'linear' -# Method for filling continious user values -# between years of passed scenarios. Passed -# method is directly passed to interpolation -# function of a DataFrame. -# dfill : string, default 'ffill' -# Method for filling discrete user values -# between years of passed scenarios. Passed -# method is directly passed to fillna -# function of a DataFrame. - -# Returns -# ------- -# uvalues : DataFrame -# Returns the (interpolated) user values of each scenario -# that is inside the range of the years of the passed -# clients. The DataFrame also includes the user values for -# the years of the passed clients.""" - -# return Interpolator(clients).interpolate(cfill, dfill) - -# class Interpolator: -# """Interpolator class object""" - -# @property -# def clients(self): -# """client list""" -# return self.__clients - -# @clients.setter -# def clients(self, clients): -# """client list setter""" - -# # check if clients in listlike object -# if not isinstance(clients, list): -# clients = list(clients) - -# # check client in list -# for client in clients: -# if not isinstance(client, pyetm.Client): -# raise TypeError('client must be of type pyetm.client.Client') - -# # validate area codes and sort area codes -# self._validate_area_codes(clients) -# clients = self._sort_clients(clients) - -# # check scenario parameters -# self._validate_scenario_parameters(clients) - -# # set client list -# self.__clients = clients - -# def __init__(self, clients): -# """"initialize interpolator - -# Parameters -# ---------- -# clients : list -# List of pyetm.client.Client objects that -# are used to interpolate the scenario. Clients -# are by end year on initialization.""" - -# # set clients -# self.clients = clients - -# def interpolate(self, cfill='linear', dfill='ffill'): -# """Interpolates the user values of the years between the -# passed clients. Uses a seperate method for continous and -# discrete user values. -# Do note that the heat network order is not returned or -# interpolated by this function. -# Parameters -# ---------- -# cfill : string, default 'linear' -# Method for filling continious user values -# between years of passed scenarios. Passed -# method is directly passed to interpolation -# function of a DataFrame. -# dfill : string, default 'ffill' -# Method for filling discrete user values -# between years of passed scenarios. Passed -# method is directly passed to fillna -# function of a DataFrame. -# Returns -# ------- -# uvalues : DataFrame -# Returns the (interpolated) user values of each scenario -# that is inside the range of the years of the passed -# clients. The DataFrame also includes the user values for -# the years of the passed clients. -# make sure to check share groups after interpolation""" - -# # get end years of client to make annual series -# years = [client.end_year for client in self.clients] -# columns = [x for x in range(min(years), max(years) + 1)] - -# # get continous and discrete values for clients -# cvalues = [client._cvalues for client in self.clients] -# cvalues = pandas.concat(cvalues, axis=1, keys=years) - -# # make cvalues dataframe and interpolate -# cvalues = pandas.DataFrame(data=cvalues, columns=columns) -# cvalues = cvalues.interpolate(method=cfill, axis=1) - -# # get dvalues from passed clients -# dvalues = [client._dvalues for client in self.clients] -# dvalues = pandas.concat(dvalues, axis=1, keys=years) - -# # merge cvalues and dvalues -# uvalues = pandas.concat([cvalues, dvalues]) -# uvalues = uvalues.fillna(method=dfill, axis=1) - -# # sort user values -# sparams = self.clients[0].scenario_parameters -# uvalues = uvalues.loc[sparams.index] - -# return uvalues - -# def _sort_clients(self, clients): -# """sort list of clients based on end year""" - -# # get end years and sort clients based on end years -# years = [client.end_year for client in clients] -# clients = numpy.array(clients)[numpy.array(years).argsort()] - -# # check for duplicate end years -# if len(years) != len(numpy.unique(years)): -# raise ValueError("clients passed with duplicate scenario end year") - -# return list(clients) - -# def _validate_area_codes(self, clients): -# """check if all area codes of the passed scenarios are the same""" - -# # get all area codes of all clients -# codes = [client.area_code for client in clients] - -# if len(numpy.unique(codes)) != 1: -# raise ValueError("different area codes in passed clients") - -# def _validate_scenario_parameters(self, clients): -# """check if all set scenario paremeters are okay""" - -# for client in clients: -# try: -# client._check_scenario_parameters() - -# except ValueError: -# raise ValueError('errors in scenario parameters of ' + -# f'{client}, diagnose client with ' + -# 'client._check_scenario_parameters()') +"""scenario interpolation""" + +from __future__ import annotations +from typing import Iterable, TYPE_CHECKING + +import logging + +import pandas as pd +from pyetm.types import ErrorHandling, InterpolateOptions + +if TYPE_CHECKING: + from pyetm import Client + +logger = logging.getLogger(__name__) + + +def interpolate( + target: int | Iterable[int], + clients: list[Client], + method: InterpolateOptions = "linear", + if_errors: ErrorHandling = "raise", +) -> pd.DataFrame: + """Interpolates the user values of the years between the + passed clients. Uses a seperate method for continous and + discrete user values. + + Do note that the heat network order is not returned or + interpolated by this function. + + Parameters + ---------- + target : int or iterable of int + The target year(s) for which to + make an interpolation. + clients: list of Client + List of pyetm.client.Client objects that + are used to interpolate the scenario. + method : string, default 'linear' + Method for filling continious user values + for the passed target year(s). + + Returns + ------- + inputs : DataFrame + Returns the input parameters for all years of the + passed clients and the target year(s).""" + + # sort clients by end year + clients = sorted(clients, key=lambda client: client.end_year) + + # handle single target year + if isinstance(target, int): + target = [target] + + # validate area codes for clients + codes = [cln.area_code for cln in clients] + if len(set(codes)) != 1: + raise ValueError(f"Different area codes in passed clients: {codes}") + + # validate end years + years = [cln.end_year for cln in clients] + if len(set(years)) != len(clients): + raise ValueError(f"Duplicate end years in passed clients: {years}") + + # filter list + filtered = [yr for yr in target if min(years) < yr < max(years)] + if len(set(filtered)) != len(set(target)): + raise ValueError( + "Interpolation target(s) out of bound: " + f"{min(years)} < " + f"{list(set(filtered).symmetric_difference(target))} " + f"< {max(years)}." + ) + + # merge inputs and mask get input parameters + inputs = pd.concat([cln.input_parameters for cln in clients], axis=1, keys=years) + params = clients[0].get_input_parameters(include_disabled=True, detailed=True) + + # split input parameters by value type + mask = params["unit"].isin(["enum", "x"]) + cinputs, dinputs = inputs.loc[~mask], inputs.loc[mask] + + # check for equality of discrete values + errors = dinputs.loc[~dinputs.eq(dinputs.iloc[:, -1], axis=0).any(axis=1)] + if not errors.empty: + # make message + msg = ( + "Inconsistent scenario settings for input parameters: \n\n" + + errors.to_string() + ) + + if if_errors == "warn": + logger.warning(msg) + + if if_errors == "raise": + raise ValueError(msg) + + # expand subsets with target year columns + columns = sorted(set(years).union(filtered)) + cinputs = pd.DataFrame(data=cinputs, columns=columns, dtype=float) + dinputs = pd.DataFrame(data=dinputs, columns=columns) + + # handle interpolation + cinputs = cinputs.interpolate(method=method, axis=1) + dinputs = dinputs.bfill(axis=1) + + return pd.concat([cinputs, dinputs]) diff --git a/src/pyetm/utils/lookup.py b/src/pyetm/utils/lookup.py index 71c4158..ecee147 100644 --- a/src/pyetm/utils/lookup.py +++ b/src/pyetm/utils/lookup.py @@ -2,8 +2,8 @@ import numpy as np import pandas as pd -def lookup_coordinates(coords: pd.Series, - frame: pd.DataFrame, **kwargs) -> pd.Series: + +def lookup_coordinates(coords: pd.Series, frame: pd.DataFrame, **kwargs) -> pd.Series: """lookup function to get coordinate values from dataframe""" # reindex frame with factorized coords diff --git a/src/pyetm/utils/loop.py b/src/pyetm/utils/loop.py index d6e1b6b..d90cb81 100644 --- a/src/pyetm/utils/loop.py +++ b/src/pyetm/utils/loop.py @@ -10,11 +10,12 @@ # get modulelogger logger = get_modulelogger(__name__) + def _start_loop(loop): asyncio.set_event_loop(loop) loop.run_forever() + # create thread in which a new loop can run -_LOOP = asyncio.new_event_loop() -_LOOP_THREAD = threading.Thread(target=_start_loop, - args=[_LOOP], daemon=True) +_loop = asyncio.new_event_loop() +_loop_thread = threading.Thread(target=_start_loop, args=[_loop], daemon=True) diff --git a/src/pyetm/utils/profiles.py b/src/pyetm/utils/profiles.py index 546730d..8ef6039 100644 --- a/src/pyetm/utils/profiles.py +++ b/src/pyetm/utils/profiles.py @@ -1,8 +1,9 @@ """PeriodIndex related utilities.""" - from __future__ import annotations +from typing import Any import calendar + import pandas as pd @@ -10,7 +11,7 @@ def make_period_index( year: int, name: str | None = None, periods: int | None = None, - as_datetime: bool = False + as_datetime: bool = False, ) -> pd.PeriodIndex | pd.DatetimeIndex: """Make a PeriodIndex for a specific year. @@ -33,15 +34,15 @@ def make_period_index( The constructed index.""" # make start period - start = f'{year}-01-01 00:00' + start = f"{year}-01-01 00:00" # default name for datetime index if (name is None) & as_datetime: - name = 'Datetime' + name = "Datetime" # default name for period index if (name is None) & (not as_datetime): - name = 'Period' + name = "Period" # check periods if periods is None: @@ -52,20 +53,16 @@ def make_period_index( periods = len(periods) # make periodindex - index = pd.period_range( - start=start, periods=periods, freq='H', name=name) + index = pd.period_range(start=start, periods=periods, freq="H", name=name) # convert type if as_datetime: - index = index.astype('datetime64[ns]') + index = index.astype("datetime64[ns]") return index -def validate_profile( - series: pd.Series, - name: str | None = None, - year: int | None = None -) -> pd.Series: + +def validate_profile(series: pd.Series[Any], name: str | None = None) -> pd.Series[Any]: """Validate profile object. Parameters @@ -74,10 +71,6 @@ def validate_profile( Series to be validated. name : str Name of returned series. - year : int, default None - Optional year to help construct a - PeriodIndex when a series is passed - without PeriodIndex or DatetimeIndex Return ------ @@ -87,34 +80,41 @@ def validate_profile( # squeeze dataframe if isinstance(series, pd.DataFrame): - series = series.squeeze('columns') + squeezed = pd.DataFrame(series).squeeze(axis=1) + + # cannot process frames + if not isinstance(squeezed, pd.Series): + raise TypeError("Cannot squeeze DataFrame to Series") + + # assign squeezed + series = squeezed # default to series name if name is None: - name = series.name + name = str(series.name) # check series lenght series = validate_profile_lenght(series, length=8760) - # check index type - objs = (pd.DatetimeIndex, pd.PeriodIndex) - if (not isinstance(series.index, objs)) & (year is None): - - # raise for missing year parameter - raise ValueError(f"Must specify year for profile '{name}' when " - "passed without pd.DatetimeIndex or pd.PeriodIndex") + # # check index type + # objs = (pd.DatetimeIndex, pd.PeriodIndex) + # if not isinstance(series.index, objs): + # # check for year parameter + # if year is None: + # raise ValueError( + # f"Must specify year for profile '{name}' when " + # "passed without pd.DatetimeIndex or pd.PeriodIndex" + # ) - # assign periodindex otherwise - if not isinstance(series.index, objs): - series.index = make_period_index(year, periods=8760) + # # assign period index + # series.index = make_period_index(year, periods=8760) return pd.Series(series, name=name) + def validate_profile_lenght( - series: pd.Series, - name: str | None = None, - length: int | None = None -) -> pd.Series: + series: pd.Series[Any], name: str | None = None, length: int | None = None +) -> pd.Series[Any]: """Validate profile length. Parameters @@ -138,11 +138,13 @@ def validate_profile_lenght( # default name if name is None: - name = series.name + name = str(series.name) # check series lenght if len(series) != 8760: - raise ValueError(f"Profile '{name}' must contain 8760 values, " - f"counted '{len(series)}' instead.") + raise ValueError( + f"Profile '{name}' must contain 8760 values, " + f"counted '{len(series)}' instead." + ) return pd.Series(series, name=name) diff --git a/src/pyetm/utils/regionalisation.py b/src/pyetm/utils/regionalisation.py index 4f32f3d..c44c1d2 100644 --- a/src/pyetm/utils/regionalisation.py +++ b/src/pyetm/utils/regionalisation.py @@ -1,35 +1,99 @@ """regionalisation methods""" +from __future__ import annotations + import logging -import numpy as np import pandas as pd +from pyetm.exceptions import BalanceError +from pyetm.types import ErrorHandling +from pyetm.utils.general import iterable_to_str, mapped_floats_to_str + logger = logging.getLogger(__name__) -def _validate_regionalisation(curves, reg, **kwargs): - """helper function to validate regionalisation table""" - # load regionalization - if isinstance(reg, str): - reg = pd.read_csv(reg, **kwargs) +def validate_hourly_curves_balance( + curves: pd.DataFrame, precision: int = 1, errors: ErrorHandling = "warn" +) -> None: + """validate if deficits in curves""" - # check is reg specifies keys not in passed curves - for item in reg.columns[~reg.columns.isin(curves.columns)]: - raise ValueError(f"'{item}' is not present in the passed curves") + # copy curves + curves = curves.copy() + + # regex mapping for product group + productmap = { + "^.*[.]output [(]MW[)]$": "supply", + "^.*[.]input [(]MW[)]$": "demand", + } + + # make column mapping + cols = curves.columns.to_series(name="product") + cols = cols.replace(productmap, regex=True) + + # map column mapping + curves.columns = curves.columns.map(cols) + + # aggregate columns + curves = curves.groupby(level=0, axis=1).sum() + balance = curves["supply"] - curves["demand"] + + # handle deficits + if any(balance.round(precision) != 0): + message = "Deficits in curves" + + if errors == "warn": + logger.warning(message) + + if errors == "raise": + raise BalanceError(message) + + +def validate_regionalisation( + curves: pd.DataFrame, + reg: pd.DataFrame, + prec: int = 3, + errors: ErrorHandling = "warn", +) -> None: + """helper function to validate regionalisation table""" # check if passed curves specifies keys not specified in reg - for item in curves.columns[~curves.columns.isin(reg.columns)]: - raise ValueError(f"'{item}' not present in the regionalization") + missing_reg = curves.columns[~curves.columns.isin(reg.columns)] + if not missing_reg.empty: + raise KeyError( + f"Missing key(s) in regionalisation: {iterable_to_str(missing_reg)}" + ) - # check if regionalizations add up to 1.000 - sums = round(reg.sum(axis=0), 3) - for idx, value in sums[sums != 1].items(): - raise ValueError(f'"{idx}" regionalization sums to ' + - f'{value: .3f} instead of 1.000') + # check is reg specifies keys not in passed curves + superfluos_reg = reg.columns[~reg.columns.isin(curves.columns)] + if not superfluos_reg.empty: + if errors == "warn": + for key in superfluos_reg: + logger.warning("Unused key in regionalisation: %s", key) - return curves, reg + if errors == "raise": + error = iterable_to_str(superfluos_reg) + raise KeyError(f"Unused key(s) in regionalisation: {error}") -def regionalise_curves(curves, reg, node=None, - sector=None, hours=None, **kwargs): + # check if regionalizations add up to 1.000 + sums = reg.sum(axis=0).round(prec) + checksum_errors = sums[sums != 1] + if not checksum_errors.empty: + if errors == "warn": + for key, value in checksum_errors.items(): + error = f"{key}={value:.{prec}f}" + logger.warning("Regionalisation key does not sum to 1: %s", error) + + if errors == "raise": + error = mapped_floats_to_str((dict(checksum_errors)), prec=prec) + raise ValueError(f"Regionalisation key(s) do not sum to 1: {error}") + + +def regionalise_curves( + curves: pd.DataFrame, + reg: pd.DataFrame, + node: str | list[str] | None = None, + sector: str | list[str] | None = None, + hours: int | list[int] | None = None, +) -> pd.DataFrame: """Return the residual power of the curves based on a regionalisation table. The kwargs are passed to pd.read_csv when the regionalisation argument is a passed as a filestring. @@ -38,7 +102,7 @@ def regionalise_curves(curves, reg, node=None, ---------- curves : DataFrame Categorized ETM curves. - reg : DataFrame or str + reg : DataFrame Regionalization table with nodes in index and sectors in columns. node : key or list of keys, default None @@ -54,54 +118,57 @@ def regionalise_curves(curves, reg, node=None, Return ------ curves : DataFrame - Residual power profiles.""" + Residual power curves per regionalisation node.""" # validate regionalisation - curves, reg = _validate_regionalisation(curves, reg, **kwargs) - - """consider warning for curves that do not sum up to zero, - as this leads to incorrect regionalisations. Assigning a negative - sign to demand only happens during categorisation.""" + validate_hourly_curves_balance(curves, errors="raise") + validate_regionalisation(curves, reg) # handle node subsetting if node is not None: - # warn for subsettign multiple items if isinstance(node, list): + logger.warning("returning dot product for subset of multiple nodes") - msg = "returning dot product for subset of multiple nodes" - logger.warning(msg) - - else: - # ensure list + # handle string + if isinstance(node, str): node = [node] # subset node - reg = reg.loc[node] + reg = reg.loc[node, :] # handle sector subsetting if sector is not None: - # warn for subsetting multiple items if isinstance(sector, list): + logger.warning("returning dot product for subset of multiple sectors") - msg = "returning dot product for subset of multiple sectors" - logger.warning(msg) - - else: - # ensure list + # handle string + if isinstance(sector, str): sector = [sector] # subset sector - curves, reg = curves[sector], reg[sector] + curves, reg = curves.loc[:, sector], reg.loc[:, sector] # subset hours if hours is not None: - curves = curves.loc[hours] + # handle single hour + if isinstance(hours, int): + hours = [hours] + + # subset hours + curves = curves.iloc[hours, :] return curves.dot(reg.T) -def regionalise_node(curves, reg, node, sector=None, hours=None, **kwargs): + +def regionalise_node( + curves: pd.DataFrame, + reg: pd.DataFrame, + node: str, + sector: str | list[str] | None = None, + hours: int | list[int] | None = None, +) -> pd.DataFrame: """Return the sector profiles for a node specified in the regionalisation table. The kwargs are passed to pd.read_csv when the regionalisation argument is a passed as a filestring. @@ -113,7 +180,7 @@ def regionalise_node(curves, reg, node, sector=None, hours=None, **kwargs): reg : DataFrame or str Regionalization table with nodes in index and sectors in columns. - node : key or list of keys + node : key Specific node in regionalisation for which the profiles are returned. sector : key or list of keys, default None @@ -129,33 +196,31 @@ def regionalise_node(curves, reg, node, sector=None, hours=None, **kwargs): Sector profile per specified node.""" # validate regionalisation - curves, reg = _validate_regionalisation(curves, reg, **kwargs) + validate_hourly_curves_balance(curves) + validate_regionalisation(curves, reg) - # subset node(s) - reg = reg.loc[node] + if not isinstance(node, str): + node = str(node) - # subset hours - if hours is not None: - curves = curves.loc[hours] + # subset reg for node + nreg = reg.loc[node, :] - # handle single node - if not isinstance(node, list): - - # handle sector - if sector is not None: - return curves[sector].mul(reg[sector]) - - return curves.mul(reg) + # handle sector subsetting + if sector is not None: + # handle string + if isinstance(sector, str): + sector = [sector] - # prepare new index - levels = [reg.index, curves.index] - index = pd.MultiIndex.from_product(levels, names=None) + # subset sector + curves, nreg = curves.loc[:, sector], nreg.loc[sector] - # prepare new dataframe - columns = curves.columns - values = np.repeat(curves.values, reg.index.size, axis=0) + # subset hours + if hours is not None: + # handle single hour + if isinstance(hours, int): + hours = [hours] - # match index structure of regionalization - curves = pd.DataFrame(values, index=index, columns=columns) + # subset hours + curves = curves.iloc[hours, :] - return reg.mul(curves, level=0) + return curves.mul(nreg) diff --git a/src/pyetm/utils/url.py b/src/pyetm/utils/url.py index 1af860b..454e282 100644 --- a/src/pyetm/utils/url.py +++ b/src/pyetm/utils/url.py @@ -1,7 +1,9 @@ +"""url""" from __future__ import annotations from urllib.parse import parse_qsl, urlencode, urljoin, urlparse + def set_url_parameters(url, params: dict[str, str]): """change url parameters""" @@ -11,6 +13,7 @@ def set_url_parameters(url, params: dict[str, str]): return parsed.geturl() + def append_parameters_to_url(url, params: dict[str, str]): """add url parameters""" @@ -20,7 +23,8 @@ def append_parameters_to_url(url, params: dict[str, str]): return set_url_parameters(url, params=params) -def append_path_to_url(url, *args: str): + +def append_path_to_url(url, *args: tuple[str]): """add path to existing path""" # parse url @@ -28,19 +32,21 @@ def append_path_to_url(url, *args: str): # get path element and join args = [parsed.path] + list(args) - path = "/".join(map(lambda x: str(x).rstrip('/'), args)) + path = "/".join(map(lambda x: str(x).rstrip("/"), args)) # replace path parsed = parsed._replace(path=path) return parsed.geturl() + def make_myc_url( url: str, scenario_ids: list[int], path: str | None = None, - params: dict[str, str] | None = None + params: dict[str, str] | None = None, ) -> str: + """make myc url""" # make base url url = urljoin(url, ",".join(map(str, scenario_ids))) diff --git a/tests/fixtures.py b/tests/fixtures.py deleted file mode 100644 index 95cd673..0000000 --- a/tests/fixtures.py +++ /dev/null @@ -1,96 +0,0 @@ -import pandas as pd - -curves = pd.DataFrame( - data = { - "category_a.input (MW)": [40, 35, 21, 42, 34], - "category_b.input (MW)": [69, 52, 45, 64, 95], - "category_c.input (MW)": [0, 0, 0, 2, 1], - "category_d.input (MW)": [38, 35, 24, 65, 36], - "category_a.output (MW)": [95, 67, 0, 75, 0], - "category_b.output (MW)": [19, 50, 0, 63, 0], - "category_c.output (MW)": [33, 5, 90, 35, 0], - "deficit": [0, 0, 0, 0, 166], - } -) - -mapping = pd.Series( - data = { - "category_a.input (MW)": "mapping_a", - "category_b.input (MW)": "mapping_a", - "category_c.input (MW)": "mapping_b", - "category_d.input (MW)": "mapping_b", - "category_a.output (MW)": "mapping_a", - "category_b.output (MW)": "mapping_a", - "category_c.output (MW)": "mapping_b", - "deficit": "deficit", - }, - name="user_keys" -) - -reg = pd.DataFrame( - data = { - "category_a.input (MW)": { - "node_a": 1.00, - "node_b": 0.00, - "node_c": 0.00, - "node_d": 0.00, - "node_e": 0.00, - }, - - "category_b.input (MW)": { - "node_a": 0.00, - "node_b": 1.00, - "node_c": 0.00, - "node_d": 0.00, - "node_e": 0.00, - }, - - "category_c.input (MW)": { - "node_a": 0.00, - "node_b": 0.00, - "node_c": 1.00, - "node_d": 0.00, - "node_e": 0.00, - }, - - "category_d.input (MW)": { - "node_a": 0.00, - "node_b": 0.00, - "node_c": 0.00, - "node_d": 1.00, - "node_e": 0.00, - }, - - "category_a.output (MW)": { - "node_a": 1.00, - "node_b": 0.00, - "node_c": 0.00, - "node_d": 0.00, - "node_e": 0.00, - }, - - "category_b.output (MW)": { - "node_a": 0.00, - "node_b": 1.00, - "node_c": 0.00, - "node_d": 0.00, - "node_e": 0.00, - }, - - "category_c.output (MW)": { - "node_a": 0.00, - "node_b": 0.00, - "node_c": 1.00, - "node_d": 0.00, - "node_e": 0.00, - }, - - "deficit": { - "node_a": 0.00, - "node_b": 0.00, - "node_c": 0.00, - "node_d": 0.00, - "node_e": 1.00, - }, - } -) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index fb41a01..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,50 +0,0 @@ -import unittest -import pandas as pd - -from fixtures import reg, mapping, curves -from pyetm.utils import categorise_curves, regionalise_curves, regionalise_node - -class UtilsTester(unittest.TestCase): - - def test_categorise_curves(self): - - mapped = categorise_curves(curves, mapping) - sol = pd.DataFrame({ - 'deficit': [0, 0, 0, 0, 166], - 'mapping_a': [5, 30, -66, 32,-129], - 'mapping_b': [-5, -30, 66, -32, -37], - }) - - self.assertEqual(True, mapped.equals(sol)) - - def test_regionalise_curves(self): - - rec = regionalise_curves(curves, reg) - sol = pd.DataFrame({ - 'node_a': [135.0, 102.0, 21.0, 117.0, 34.0], - 'node_b': [88.0, 102.0, 45.0, 127.0, 95.0], - 'node_c': [33.0, 5.0, 90.0, 37.0, 1.0], - 'node_d': [38.0, 35.0, 24.0, 65.0, 36.0], - 'node_e': [0.0, 0.0, 0.0, 0.0, 166.0], - }) - - self.assertEqual(True, rec.equals(sol.T)) - - def test_regionalise_node(self): - - rec = regionalise_node(curves, reg, "node_a") - sol = pd.DataFrame({ - 'category_a.input (MW)': [40.0, 35.0, 21.0, 42.0, 34.0], - 'category_b.input (MW)': [0.0, 0.0, 0.0, 0.0, 0.0], - 'category_c.input (MW)': [0.0, 0.0, 0.0, 0.0, 0.0], - 'category_d.input (MW)': [0.0, 0.0, 0.0, 0.0, 0.0], - 'category_a.output (MW)': [95.0, 67.0, 0.0, 75.0, 0.0], - 'category_b.output (MW)': [0.0, 0.0, 0.0, 0.0, 0.0], - 'category_c.output (MW)': [0.0, 0.0, 0.0, 0.0, 0.0], - 'deficit': [0.0, 0.0, 0.0, 0.0, 0.0] - }) - - self.assertEqual(True, rec.equals(sol)) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file