Skip to content

Commit

Permalink
Use consistent names for global reporting vars; document
Browse files Browse the repository at this point in the history
  • Loading branch information
khaeru committed Oct 3, 2019
1 parent ddf476b commit 8e0c2ff
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 33 deletions.
38 changes: 27 additions & 11 deletions ixmp/reporting/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@

import yaml

from . import computations
from .describe import describe_recursive
from .key import Key
from .utils import (
REPLACE_UNITS,
RENAME_DIMS,
UNITS,
keys_for_quantity,
rename_dims,
ureg,
)
from . import computations
from .describe import describe_recursive


log = logging.getLogger(__name__)
Expand Down Expand Up @@ -547,13 +547,29 @@ def write(self, key, path):
def configure(path=None, **config):
"""Configure reporting globally.
Valid configuration keys include:
Modifies global variables that affect the behaviour of *all* Reporters and
computations, namely
:obj:`RENAME_DIMS <ixmp.reporting.utils.RENAME_DIMS>`,
:obj:`REPLACE_UNITS <ixmp.reporting.utils.REPLACE_UNITS>`, and
:obj:`UNITS <ixmp.reporting.utils.UNITS>`.
Valid configuration keys—passed as *config* keyword arguments—include:
Other Parameters
----------------
units : mapping
Configuration for handling of units. Valid sub-keys include:
- *units*:
- **replace** (mapping of str -> str): replace units before they are
parsed by :doc:`pint <pint:index>`. Added to :obj:`REPLACE_UNITS
<ixmp.reporting.utils.REPLACE_UNITS>`.
- **define** (:class:`str`): block of unit definitions, added to
:obj:`UNITS <ixmp.reporting.utils.UNITS>` so that units are
recognized by pint. See the :ref:`pint documentation
<pint:defining>`.
- *define*: a string, passed to :meth:`pint.UnitRegistry.define`.
- *replace*: a mapping from str to str, used to replace units before they
are parsed by pints
rename_dims : mapping of str -> str
Update :obj:`RENAME_DIMS <ixmp.reporting.utils.RENAME_DIMS>`.
Warns
-----
Expand All @@ -567,14 +583,14 @@ def configure(path=None, **config):

# Define units
if 'define' in units:
ureg.define(units['define'].strip())
UNITS.define(units['define'].strip())

# Add replacements
for old, new in units.get('replace', {}).items():
REPLACE_UNITS[old] = new

# Dimensions to be renamed
rename_dims.update(config.get('rename_dims', {}))
RENAME_DIMS.update(config.get('rename_dims', {}))


def _config_args(path, keys, sections={}):
Expand Down
38 changes: 21 additions & 17 deletions ixmp/reporting/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,28 @@

log = logging.getLogger(__name__)

ureg = pint.UnitRegistry()

# See also:
# - docstring of attrseries.AttrSeries.
# - test_report_size() for a test that shows how non-sparse xr.DataArray
# triggers MemoryError.
Quantity = AttrSeries
# Quantity = xr.DataArray

# Replacements to apply to quantity units before parsing by pint
#: Replacements to apply to quantity units before parsing by
#: :doc:`pint <pint:index>`. Mapping from original unit -> preferred unit.
REPLACE_UNITS = {
'%': 'percent',
}

#: Dimensions to rename when extracting raw data from Scenario objects.
#: Mapping from Scenario dimension name -> preferred dimension name.
RENAME_DIMS = {}

#: :doc:`pint <pint:index>` unit registry for processing quantity units.
#: All units handled by :mod:`imxp.reporting` must be either standard SI units,
#: or added to this registry.
UNITS = pint.UnitRegistry()


def clean_units(input_string):
"""Tolerate messy strings for units.
Expand All @@ -49,18 +57,14 @@ def collect_units(*args):
if '_unit' in arg.attrs:
# Convert units if necessary
if isinstance(arg.attrs['_unit'], str):
arg.attrs['_unit'] = ureg.parse_units(arg.attrs['_unit'])
arg.attrs['_unit'] = UNITS.parse_units(arg.attrs['_unit'])
else:
log.debug('assuming {} is unitless'.format(arg))
arg.attrs['_unit'] = ureg.parse_units('')
arg.attrs['_unit'] = UNITS.parse_units('')

return [arg.attrs['_unit'] for arg in args]


# Mapping from raw -> preferred dimension names
rename_dims = {}


def _find_dims(data):
"""Return the list of dimensions for *data*."""
if isinstance(data, pd.DataFrame):
Expand All @@ -77,7 +81,7 @@ def _find_dims(data):
continue

# Rename dimensions
return [rename_dims.get(d, d) for d in dims]
return [RENAME_DIMS.get(d, d) for d in dims]


def keys_for_quantity(ix_type, name, scenario):
Expand Down Expand Up @@ -124,16 +128,16 @@ def invalid(unit):
# Parse units
try:
unit = clean_units(unit[0])
unit = ureg.parse_units(unit)
unit = UNITS.parse_units(unit)
except IndexError:
# Quantity has no unit
unit = ureg.parse_units('')
unit = UNITS.parse_units('')
except pint.UndefinedUnitError:
# Unit(s) do not exist; define them in the UnitRegistry

# Split possible compound units
for u in unit.split('/'):
if u in dir(ureg):
if u in dir(UNITS):
# Unit already defined
continue

Expand All @@ -142,11 +146,11 @@ def invalid(unit):
log.info('Add unit definition: {}'.format(definition))

# This line will fail silently for units like 'G$'
ureg.define(definition)
UNITS.define(definition)

# Try to parse again
try:
unit = ureg.parse_units(unit)
unit = UNITS.parse_units(unit)
except pint.UndefinedUnitError:
# Handle the silent failure of define(), above
raise invalid(unit) from None
Expand Down Expand Up @@ -203,15 +207,15 @@ def data_for_quantity(ix_type, name, column, scenario, filters=None):
if 'mixed units' in e.args[0]:
# Discard mixed units
log.warn('{} discarded for {!r}'.format(e.args[0], name))
attrs = {'_unit': ureg.parse_units('')}
attrs = {'_unit': UNITS.parse_units('')}
else:
# Raise all other ValueErrors
raise

# Set index if 1 or more dimensions
if len(dims):
# First rename, then set index
data.rename(columns=rename_dims, inplace=True)
data.rename(columns=RENAME_DIMS, inplace=True)
data.set_index(dims, inplace=True)

# Check sparseness
Expand Down
11 changes: 6 additions & 5 deletions tests/test_reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
Reporter,
computations,
)
from ixmp.reporting.utils import ureg, Quantity
from ixmp.reporting import UNITS
from ixmp.reporting.utils import Quantity
from ixmp.testing import make_dantzig, assert_qty_allclose, assert_qty_equal


Expand Down Expand Up @@ -111,7 +112,7 @@ def test_reporter_from_dantzig(test_mp, test_data_path):
d_i = rep.get('d:i')

# Units pass through summation
assert d_i.attrs['_unit'] == ureg.parse_units('km')
assert d_i.attrs['_unit'] == UNITS.parse_units('km')

# Summation across all dimensions results a 1-element Quantity
d = rep.get('d:')
Expand Down Expand Up @@ -335,15 +336,15 @@ def test_reporting_units():

# Aggregation preserves units
r.add('energy', (computations.sum, 'energy:x', None, ['x']))
assert r.get('energy').attrs['_unit'] == ureg.parse_units('MJ')
assert r.get('energy').attrs['_unit'] == UNITS.parse_units('MJ')

# Units are derived for a ratio of two quantities
r.add('power', (computations.ratio, 'energy:x', 'time'))
assert r.get('power').attrs['_unit'] == ureg.parse_units('MJ/hour')
assert r.get('power').attrs['_unit'] == UNITS.parse_units('MJ/hour')

# Product of dimensioned and dimensionless quantities keeps the former
r.add('energy2', (computations.product, 'energy:x', 'efficiency'))
assert r.get('energy2').attrs['_unit'] == ureg.parse_units('MJ')
assert r.get('energy2').attrs['_unit'] == UNITS.parse_units('MJ')


def test_reporting_platform_units(test_mp, caplog):
Expand Down

0 comments on commit 8e0c2ff

Please sign in to comment.