From 1dd7bc385ab0184857e282af780a44902eeea346 Mon Sep 17 00:00:00 2001 From: bayliffe Date: Wed, 10 Nov 2021 15:02:16 +0000 Subject: [PATCH 01/18] Spot-extraction additional coordinates ordering fix (#1610) * Ensure additional coordinates are handled in the correct order. * Add an acceptance test. * Style fix. --- improver/spotdata/spot_extraction.py | 2 +- improver_tests/acceptance/SHA256SUMS | 2 ++ .../acceptance/test_spot_extract.py | 23 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/improver/spotdata/spot_extraction.py b/improver/spotdata/spot_extraction.py index d04188ecaa..8bb6868041 100644 --- a/improver/spotdata/spot_extraction.py +++ b/improver/spotdata/spot_extraction.py @@ -316,7 +316,7 @@ def process( additional_dims = None if len(spot_values.shape) > 1: - additional_dims = np.flip(diagnostic_cube.dim_coords)[2:] + additional_dims = diagnostic_cube.dim_coords[:-2] scalar_coords, nonscalar_coords = self.get_aux_coords( diagnostic_cube, x_indices, y_indices diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index 631e02b8d7..f6644ce4ce 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -462,6 +462,7 @@ b9d56c28ab78f296842865d3d6863d5017d6928e45632b00bf05f086fc651969 ./spot-extract b0cfd198d69d5d5f85775542c77dc4a2a0a3092d407e338d74c72618e5de7b1e ./spot-extract/inputs/enukx_temperature_percentiles.nc 7c1546a4a0eac805483d09a99268767ede51b5649eb545e002a1e02c9f3f43f2 ./spot-extract/inputs/enukx_temperature_realizations.nc f67aca830966488cb7ac76fbd91c50ccf2494898233557c3c4070244088a296c ./spot-extract/inputs/enukx_temperature_thresholds.nc +f84730d9815d1b488915eed704cadc311eb43135015395cb75cbd8cd4b9985cd ./spot-extract/inputs/enukx_temperature_thresholds_multi_time.nc b2332aeb77c317ab560156eff80df4323fad8162b64ce58d85e0b5581338f808 ./spot-extract/inputs/metadata.json c5b207e122acc748d63698fc17c3bf83e7630cd7ece95d65a98cf2d80ee3f083 ./spot-extract/inputs/nearest_uk.nc 4b24604d85c9127f74701f43532396126007a0bcff3b85706b2eb28f95deb9fc ./spot-extract/inputs/ukvx_lapse_rate.nc @@ -476,6 +477,7 @@ d79d5449451570f7e7f479b5ecf1e5524fbf2f6d5107550c0372dd744448c7a8 ./spot-extract 5904853a7ff3ccfe09fdcdfd79ed4edd5f1f5af12d3c6f9511a73bf0bb825068 ./spot-extract/outputs/lapse_rate_adjusted_uk_temperatures.nc 081a821471206a864d6723bedda6eb066af2514068160114eed16a156810d316 ./spot-extract/outputs/mindz_land_constraint_uk_temperatures.nc eff5c671c33c3520e9e7f0bcab4fe2a4eb5557fd84767da1a8c494f3ab7f3aa6 ./spot-extract/outputs/mindz_uk_temperatures.nc +15d7221f43a11a8bac467a03a6bf7bac1b5d713d6583767fcc9f11687c112300 ./spot-extract/outputs/multi_time_kgo.nc 522f5331735a778f65ad5ed6ca5ebca03bdcb8d4c7a783d6f7afe61b61efa0ab ./spot-extract/outputs/nearest_uk_temperatures.nc 522f5331735a778f65ad5ed6ca5ebca03bdcb8d4c7a783d6f7afe61b61efa0ab ./spot-extract/outputs/nearest_uk_temperatures_amended_metadata.nc 7f7734eb79abe416b0587cff2fbff286854e978037be3d63054c93d67b8bdc39 ./spot-extract/outputs/nearest_uk_temperatures_unique_ids.nc diff --git a/improver_tests/acceptance/test_spot_extract.py b/improver_tests/acceptance/test_spot_extract.py index 2753214c80..23e2b86970 100644 --- a/improver_tests/acceptance/test_spot_extract.py +++ b/improver_tests/acceptance/test_spot_extract.py @@ -522,3 +522,26 @@ def test_local_timezone_extraction(tmp_path): ] run_cli(args) acc.compare(output_path, kgo_path) + + +def test_multi_time_input(tmp_path): + """Test extracting from a cube with a time and threshold coordinate. Note + that utilities.load.load_cube reverses the order of the leading dimensions + on load. As such the KGO has the threshold and time coordinates in a + different order to the input, but this is unrelated to spot-extract.""" + + kgo_dir = acc.kgo_root() / "spot-extract" + neighbour_path = kgo_dir / "inputs/all_methods_uk.nc" + diag_path = kgo_dir / "inputs/enukx_temperature_thresholds_multi_time.nc" + kgo_path = kgo_dir / "outputs/multi_time_kgo.nc" + output_path = tmp_path / "output.nc" + args = [ + neighbour_path, + diag_path, + "--output", + output_path, + "--new-title", + UK_SPOT_TITLE, + ] + run_cli(args) + acc.compare(output_path, kgo_path) From 61f34ede99b3a826f93000044e22be518fe1e54a Mon Sep 17 00:00:00 2001 From: Belinda Trotta <73675905+btrotta-bom@users.noreply.github.com> Date: Fri, 12 Nov 2021 16:05:17 +1100 Subject: [PATCH 02/18] Speed up interpolation in ensemble_copula_coupling.ResamplePercentiles (#1548) * Speed up interpolations * Remove old code * Restore original code for _convert_probabilities_to_percentiles * Use numba * Update docstring and type hints * Fix licence text * Style * Style * Increase tolerance * Update warning message * Calculate in 64-bit and output in 32-bit * Add tests with known result * Style * Style * improver/ensemble_copula_coupling/utilities.py * Add input checking * Fix imports * Remove unused import * Update improver/ensemble_copula_coupling/numba_utilities.py Co-authored-by: bayliffe Co-authored-by: Belinda Trotta Co-authored-by: bayliffe --- codecov.yml | 2 + doc/source/conf.py | 2 + .../ensemble_copula_coupling.py | 17 ++-- .../numba_utilities.py | 79 +++++++++++++++++++ .../ensemble_copula_coupling/utilities.py | 31 +++++++- .../test_utilities.py | 78 ++++++++++++++++++ 6 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 improver/ensemble_copula_coupling/numba_utilities.py diff --git a/codecov.yml b/codecov.yml index 89dcdc672b..4ad0d5a0c9 100644 --- a/codecov.yml +++ b/codecov.yml @@ -8,3 +8,5 @@ coverage: informational: true ignore: - "improver/cli" # ignore folders and all its contents + - "improver/ensemble_copula_coupling/numba_utilities.py" # numba not installed in CI environment + \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py index 22c8ffd368..f74f3a5c04 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -129,6 +129,8 @@ # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ["modules.rst", "extended_documentation"] +autodoc_mock_imports = ["numba"] + # The reST default role (used for this markup: `text`) to use for all # documents. # diff --git a/improver/ensemble_copula_coupling/ensemble_copula_coupling.py b/improver/ensemble_copula_coupling/ensemble_copula_coupling.py index 4e708aa4fa..31e1cf7b22 100644 --- a/improver/ensemble_copula_coupling/ensemble_copula_coupling.py +++ b/improver/ensemble_copula_coupling/ensemble_copula_coupling.py @@ -51,6 +51,7 @@ create_cube_with_percentiles, get_bounds_of_distribution, insert_lower_and_upper_endpoint_to_1d_array, + interpolate_multiple_rows_same_x, restore_non_percentile_dimensions, ) from improver.metadata.probabilistic import ( @@ -284,16 +285,14 @@ def _interpolate_percentiles( original_percentiles, forecast_at_reshaped_percentiles, bounds_pairing ) - forecast_at_interpolated_percentiles = np.empty( - (len(desired_percentiles), forecast_at_reshaped_percentiles.shape[0]), - dtype=np.float32, + forecast_at_interpolated_percentiles = interpolate_multiple_rows_same_x( + np.array(desired_percentiles, dtype=np.float64), + original_percentiles.astype(np.float64), + forecast_at_reshaped_percentiles.astype(np.float64), + ) + forecast_at_interpolated_percentiles = np.transpose( + forecast_at_interpolated_percentiles ) - for index in range(forecast_at_reshaped_percentiles.shape[0]): - forecast_at_interpolated_percentiles[:, index] = np.interp( - desired_percentiles, - original_percentiles, - forecast_at_reshaped_percentiles[index, :], - ) # Reshape forecast_at_percentiles, so the percentiles dimension is # first, and any other dimension coordinates follow. diff --git a/improver/ensemble_copula_coupling/numba_utilities.py b/improver/ensemble_copula_coupling/numba_utilities.py new file mode 100644 index 0000000000..76ef20eb7b --- /dev/null +++ b/improver/ensemble_copula_coupling/numba_utilities.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# (C) British Crown Copyright 2017-2021 Met Office. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +""" +This module defines the optional numba utilities for Ensemble Copula Coupling +plugins. +""" + +import os + +import numpy as np +from numba import config, njit, prange, set_num_threads + +config.THREADING_LAYER = "omp" +if "OMP_NUM_THREADS" in os.environ: + set_num_threads(int(os.environ["OMP_NUM_THREADS"])) + + +@njit(parallel=True) +def fast_interp_same_x(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndarray: + """For each row i of fp, do the equivalent of np.interp(x, xp, fp[i, :]). + + Args: + x: 1-D array + xp: 1-D array, sorted in non-decreasing order + fp: 2-D array with len(xp) columns + Returns: + 2-D array with shape (len(fp), len(x)), with each row i equal to + np.interp(x, xp, fp[i, :]) + """ + # check inputs + if len(x.shape) != 1: + raise ValueError("x must be 1-dimensional.") + if len(xp.shape) != 1: + raise ValueError("xp must be 1-dimensional.") + if fp.shape[1] != len(xp): + raise ValueError("Dimension 1 of fp must be equal to length of xp.") + index = np.searchsorted(xp, x) + result = np.empty((fp.shape[0], len(x)), dtype=np.float32) + for row in prange(fp.shape[0]): + for i, ind in enumerate(index): + if ind == 0: + result[row, i] = fp[row, 0] + elif ind == len(xp): + result[row, i] = fp[row, -1] + elif xp[ind] - xp[ind - 1] >= 1e-15: + result[row, i] = fp[row, ind - 1] + (x[i] - xp[ind - 1]) / ( + xp[ind] - xp[ind - 1] + ) * (fp[row, ind] - fp[row, ind - 1]) + else: + result[row, i] = fp[row, ind - 1] + return result diff --git a/improver/ensemble_copula_coupling/utilities.py b/improver/ensemble_copula_coupling/utilities.py index 34a8508b9b..202a3b925e 100644 --- a/improver/ensemble_copula_coupling/utilities.py +++ b/improver/ensemble_copula_coupling/utilities.py @@ -33,7 +33,7 @@ plugins. """ - +import warnings from typing import List, Optional, Union import cf_units as unit @@ -289,3 +289,32 @@ def restore_non_percentile_dimensions( if n_percentiles > 1: shape_to_reshape_to = [n_percentiles] + shape_to_reshape_to return array_to_reshape.reshape(shape_to_reshape_to) + + +def slow_interp_same_x(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndarray: + """For each row i of fp, calculate np.interp(x, xp, fp[i, :]). + + Args: + x: 1-D array + xp: 1-D array, sorted in non-decreasing order + fp: 2-D array with len(xp) columns + Returns: + 2-D array with shape (len(fp), len(x)), with each row i equal to + np.interp(x, xp, fp[i, :]) + """ + + result = np.empty((fp.shape[0], len(x)), np.float32) + for i in range(fp.shape[0]): + result[i, :] = np.interp(x, xp, fp[i, :]) + return result + + +try: + import numba # noqa: F401 + + from improver.ensemble_copula_coupling.numba_utilities import fast_interp_same_x + + interpolate_multiple_rows_same_x = fast_interp_same_x +except ImportError: + warnings.warn("Module numba unavailable. ResamplePercentiles will be slower.") + interpolate_multiple_rows_same_x = slow_interp_same_x diff --git a/improver_tests/ensemble_copula_coupling/test_utilities.py b/improver_tests/ensemble_copula_coupling/test_utilities.py index 8697835c7f..bd265e278b 100644 --- a/improver_tests/ensemble_copula_coupling/test_utilities.py +++ b/improver_tests/ensemble_copula_coupling/test_utilities.py @@ -32,8 +32,10 @@ Unit tests for the `ensemble_copula_coupling.EnsembleCopulaCouplingUtilities` class. """ +import importlib import unittest from datetime import datetime +from unittest.case import skipIf import numpy as np from cf_units import Unit @@ -48,7 +50,9 @@ create_cube_with_percentiles, get_bounds_of_distribution, insert_lower_and_upper_endpoint_to_1d_array, + interpolate_multiple_rows_same_x, restore_non_percentile_dimensions, + slow_interp_same_x, ) from improver.synthetic_data.set_up_test_cubes import ( set_up_percentile_cube, @@ -398,5 +402,79 @@ def test_multiple_timesteps(self): self.assertArrayAlmostEqual(reshaped_array, expected) +numba_installed = True +try: + importlib.util.find_spec("numba") + from improver.ensemble_copula_coupling.numba_utilities import fast_interp_same_x +except ImportError: + numba_installed = False + + +class TestInterpolateMultipleRowsSameX(IrisTest): + + """Test interpolate_multiple_rows""" + + def setUp(self): + """Set up arrays.""" + np.random.seed(0) + self.x = np.arange(0, 1, 0.01) + self.xp = np.sort(np.random.random_sample(100)) + self.fp = np.random.random((100, 100)) + + def test_slow(self): + """Test slow interp against known result.""" + xp = np.array([0, 1, 2, 3, 4], dtype=np.float32) + fp = np.array([[0, 0.5, 1, 1.5, 2], [0, 2, 4, 6, 8]], dtype=np.float32) + x = np.array([-1, 0.5, 2], dtype=np.float32) + expected = np.array([[0, 0.25, 1], [0, 1, 4]], dtype=np.float32) + result = slow_interp_same_x(x, xp, fp) + np.testing.assert_allclose(result, expected) + + @skipIf(not (numba_installed), "numba not installed") + def test_fast(self): + """Test fast interp against known result.""" + xp = np.array([0, 1, 2, 3, 4], dtype=np.float32) + fp = np.array([[0, 0.5, 1, 1.5, 2], [0, 2, 4, 6, 8]], dtype=np.float32) + x = np.array([-1, 0.5, 2], dtype=np.float32) + expected = np.array([[0, 0.25, 1], [0, 1, 4]], dtype=np.float32) + result = fast_interp_same_x(x, xp, fp) + np.testing.assert_allclose(result, expected) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_fast(self): + """Test that slow and fast versions give same result.""" + result_slow = slow_interp_same_x(self.x, self.xp, self.fp) + result_fast = fast_interp_same_x(self.x, self.xp, self.fp) + np.testing.assert_allclose(result_slow, result_fast) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_fast_unordered(self): + """Test that slow and fast versions give same result + when x is not sorted.""" + shuffled_x = self.x.copy() + np.random.shuffle(shuffled_x) + result_slow = slow_interp_same_x(shuffled_x, self.xp, self.fp) + result_fast = fast_interp_same_x(shuffled_x, self.xp, self.fp) + np.testing.assert_allclose(result_slow, result_fast) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_fast_repeated(self): + """Test that slow and fast versions give same result when xp + contains repeats.""" + repeat_xp = self.xp.copy() + repeat_xp[51] = repeat_xp[50] + result_slow = slow_interp_same_x(self.x, repeat_xp, self.fp) + result_fast = fast_interp_same_x(self.x, repeat_xp, self.fp) + np.testing.assert_allclose(result_slow, result_fast) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_multi(self): + """Test that slow interp gives same result as + interpolate_multiple_rows_same_x.""" + result_slow = slow_interp_same_x(self.x, self.xp, self.fp) + result_multiple = interpolate_multiple_rows_same_x(self.x, self.xp, self.fp) + np.testing.assert_allclose(result_slow, result_multiple) + + if __name__ == "__main__": unittest.main() From 1622957a8e2a4449ce0febb3940ffb322458d85e Mon Sep 17 00:00:00 2001 From: Belinda Trotta <73675905+btrotta-bom@users.noreply.github.com> Date: Wed, 17 Nov 2021 09:20:31 +1100 Subject: [PATCH 03/18] Speed up interpolation in ensemble_copula_coupling.ConvertProbabilitiesToPercentiles (#1578) * Add fast interpolation method * Style * Add missing file * Isort * Style * Style * Add licence and docstring * Mock numba in sphinx autodoc * Tell codecov to ignore numba_utilities.py * Add more tests * Tell black to ignore import in test * isort * Add noqa for spurious black result * Remove unused code * Fix bug * Remove warning test * Remove unused code * Remove unused import * Fix imports * Simplify code * Add a test * Change method names to distinguish from functionality for other open PR * Update type hints * Add test * Style * Calculate in 64-bit, output in 32-bit * Sort x * Add simple tests with known result * Style * Restore old code for handling unordered x * Add input checking * Test correct version is used depending on numba * Style * Fix imports * Style * Fix mocking * Cast result to float32 * MAINT: Proposed review suggestions * Fix merge * Fix interpolate_multiple_rows_same_x to use same approach as interpolate_multiple_rows_same_y * Style * Isort * Fix comment Co-authored-by: Belinda Trotta Co-authored-by: cpelley --- codecov.yml | 1 - .../ensemble_copula_coupling.py | 14 ++- .../numba_utilities.py | 65 ++++++++++++- .../ensemble_copula_coupling/utilities.py | 73 ++++++++++++-- .../test_utilities.py | 97 ++++++++++++++++++- 5 files changed, 231 insertions(+), 19 deletions(-) diff --git a/codecov.yml b/codecov.yml index 4ad0d5a0c9..2762647a4e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -9,4 +9,3 @@ coverage: ignore: - "improver/cli" # ignore folders and all its contents - "improver/ensemble_copula_coupling/numba_utilities.py" # numba not installed in CI environment - \ No newline at end of file diff --git a/improver/ensemble_copula_coupling/ensemble_copula_coupling.py b/improver/ensemble_copula_coupling/ensemble_copula_coupling.py index 31e1cf7b22..247f1160d3 100644 --- a/improver/ensemble_copula_coupling/ensemble_copula_coupling.py +++ b/improver/ensemble_copula_coupling/ensemble_copula_coupling.py @@ -52,6 +52,7 @@ get_bounds_of_distribution, insert_lower_and_upper_endpoint_to_1d_array, interpolate_multiple_rows_same_x, + interpolate_multiple_rows_same_y, restore_non_percentile_dimensions, ) from improver.metadata.probabilistic import ( @@ -560,15 +561,12 @@ def _probabilities_to_percentiles( [x / 100.0 for x in percentiles], dtype=np.float32 ) - forecast_at_percentiles = np.empty( - (len(percentiles), probabilities_for_cdf.shape[0]), dtype=np.float32 + forecast_at_percentiles = interpolate_multiple_rows_same_y( + percentiles_as_fractions.astype(np.float64), + probabilities_for_cdf.astype(np.float64), + threshold_points.astype(np.float64), ) - for index in range(probabilities_for_cdf.shape[0]): - forecast_at_percentiles[:, index] = np.interp( - percentiles_as_fractions, - probabilities_for_cdf[index, :], - threshold_points, - ) + forecast_at_percentiles = forecast_at_percentiles.transpose() # Reshape forecast_at_percentiles, so the percentiles dimension is # first, and any other dimension coordinates follow. diff --git a/improver/ensemble_copula_coupling/numba_utilities.py b/improver/ensemble_copula_coupling/numba_utilities.py index 76ef20eb7b..708f6fd8e6 100644 --- a/improver/ensemble_copula_coupling/numba_utilities.py +++ b/improver/ensemble_copula_coupling/numba_utilities.py @@ -46,7 +46,6 @@ @njit(parallel=True) def fast_interp_same_x(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndarray: """For each row i of fp, do the equivalent of np.interp(x, xp, fp[i, :]). - Args: x: 1-D array xp: 1-D array, sorted in non-decreasing order @@ -77,3 +76,67 @@ def fast_interp_same_x(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndar else: result[row, i] = fp[row, ind - 1] return result + + +@njit(parallel=True) +def fast_interp_same_y(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndarray: + """For each row i of xp, do the equivalent of np.interp(x, xp[i], fp). + Args: + x: 1-d array + xp: n * m array, each row must be in non-decreasing order + fp: 1-d array with length m + Returns: + n * len(x) array where each row i is equal to np.interp(x, xp[i], fp) + """ + # check inputs + if len(x.shape) != 1: + raise ValueError("x must be 1-dimensional.") + if len(fp.shape) != 1: + raise ValueError("fp must be 1-dimensional.") + if xp.shape[1] != len(fp): + raise ValueError("Dimension 1 of xp must be equal to length of fp.") + # check whether x is non-decreasing + x_ordered = True + for i in range(1, len(x)): + if x[i] < x[i - 1]: + x_ordered = False + break + max_ind = xp.shape[1] + min_val = fp[0] + max_val = fp[-1] + result = np.empty((xp.shape[0], len(x)), dtype=np.float32) + for i in prange(xp.shape[0]): + ind = 0 + intercept = 0 + slope = 0 + x_lower = 0 + for j in range(len(x)): + recalculate = False + curr_x = x[j] + # Find the indices of xp[i] to interpolate between. We need the + # smallest index ind of xp[i] for which xp[i, ind] >= curr_x. + if x_ordered: + # Since x and x[i] are non-decreasing, ind for current j must be + # greater than equal to ind for previous j. + while (ind < max_ind) and (xp[i, ind] < curr_x): + ind = ind + 1 + recalculate = True + else: + ind = np.searchsorted(xp[i], curr_x) + # linear interpolation + if ind == 0: + result[i, j] = min_val + elif ind == max_ind: + result[i, j] = max_val + else: + if recalculate or not (x_ordered): + intercept = fp[ind - 1] + x_lower = xp[i, ind - 1] + h_diff = xp[i, ind] - x_lower + if h_diff < 1e-15: + # avoid division by very small values for numerical stability + slope = 0 + else: + slope = (fp[ind] - intercept) / h_diff + result[i, j] = intercept + (curr_x - x_lower) * slope + return result diff --git a/improver/ensemble_copula_coupling/utilities.py b/improver/ensemble_copula_coupling/utilities.py index 202a3b925e..9c303846f1 100644 --- a/improver/ensemble_copula_coupling/utilities.py +++ b/improver/ensemble_copula_coupling/utilities.py @@ -293,7 +293,6 @@ def restore_non_percentile_dimensions( def slow_interp_same_x(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndarray: """For each row i of fp, calculate np.interp(x, xp, fp[i, :]). - Args: x: 1-D array xp: 1-D array, sorted in non-decreasing order @@ -309,12 +308,70 @@ def slow_interp_same_x(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndar return result -try: - import numba # noqa: F401 +def interpolate_multiple_rows_same_x(*args): + """For each row i of fp, do the equivalent of np.interp(x, xp, fp[i, :]). + + Calls a fast numba implementation where numba is available (see + `improver.ensemble_copula_coupling.numba_utilities.fast_interp_same_y`) and calls a + the native python implementation otherwise (see :func:`slow_interp_same_y`). + + Args: + x: 1-D array + xp: 1-D array, sorted in non-decreasing order + fp: 2-D array with len(xp) columns + Returns: + 2-D array with shape (len(fp), len(x)), with each row i equal to + np.interp(x, xp, fp[i, :]) + """ + try: + import numba # noqa: F401 + + from improver.ensemble_copula_coupling.numba_utilities import fast_interp_same_x + + return fast_interp_same_x(*args) + except ImportError: + warnings.warn("Module numba unavailable. ResamplePercentiles will be slower.") + return slow_interp_same_x(*args) + + +def slow_interp_same_y(x: np.ndarray, xp: np.ndarray, fp: np.ndarray) -> np.ndarray: + """For each row i of xp, do the equivalent of np.interp(x, xp[i], fp). + + Args: + x: 1-d array + xp: n * m array, each row must be in non-decreasing order + fp: 1-d array with length m + Returns: + n * len(x) array where each row i is equal to np.interp(x, xp[i], fp) + """ + result = np.empty((xp.shape[0], len(x)), dtype=np.float32) + for i in range(xp.shape[0]): + result[i] = np.interp(x, xp[i, :], fp) + return result + + +def interpolate_multiple_rows_same_y(*args): + """For each row i of xp, do the equivalent of np.interp(x, xp[i], fp). - from improver.ensemble_copula_coupling.numba_utilities import fast_interp_same_x + Calls a fast numba implementation where numba is available (see + `improver.ensemble_copula_coupling.numba_utilities.fast_interp_same_y`) and calls a + the native python implementation otherwise (see :func:`slow_interp_same_y`). - interpolate_multiple_rows_same_x = fast_interp_same_x -except ImportError: - warnings.warn("Module numba unavailable. ResamplePercentiles will be slower.") - interpolate_multiple_rows_same_x = slow_interp_same_x + Args: + x: 1-d array + xp: n * m array, each row must be in non-decreasing order + fp: 1-d array with length m + Returns: + n * len(x) array where each row i is equal to np.interp(x, xp[i], fp) + """ + try: + import numba # noqa: F401 + + from improver.ensemble_copula_coupling.numba_utilities import fast_interp_same_y + + return fast_interp_same_y(*args) + except ImportError: + warnings.warn( + "Module numba unavailable. ConvertProbabilitiesToPercentiles will be slower." + ) + return slow_interp_same_y(*args) diff --git a/improver_tests/ensemble_copula_coupling/test_utilities.py b/improver_tests/ensemble_copula_coupling/test_utilities.py index bd265e278b..ed60141451 100644 --- a/improver_tests/ensemble_copula_coupling/test_utilities.py +++ b/improver_tests/ensemble_copula_coupling/test_utilities.py @@ -34,8 +34,10 @@ """ import importlib import unittest +import unittest.mock as mock from datetime import datetime from unittest.case import skipIf +from unittest.mock import patch import numpy as np from cf_units import Unit @@ -51,8 +53,10 @@ get_bounds_of_distribution, insert_lower_and_upper_endpoint_to_1d_array, interpolate_multiple_rows_same_x, + interpolate_multiple_rows_same_y, restore_non_percentile_dimensions, slow_interp_same_x, + slow_interp_same_y, ) from improver.synthetic_data.set_up_test_cubes import ( set_up_percentile_cube, @@ -405,11 +409,102 @@ def test_multiple_timesteps(self): numba_installed = True try: importlib.util.find_spec("numba") - from improver.ensemble_copula_coupling.numba_utilities import fast_interp_same_x + from improver.ensemble_copula_coupling.numba_utilities import ( + fast_interp_same_x, + fast_interp_same_y, + ) except ImportError: numba_installed = False +class Test_interpolate_multiple_rows_same_y(IrisTest): + + """Test interpolate_multiple_rows_same_y""" + + def setUp(self): + """Set up arrays.""" + np.random.seed(0) + self.x = np.arange(0, 1, 0.01) + self.xp = np.sort(np.random.random_sample((100, 100)), axis=1) + self.fp = np.arange(0, 100, 1).astype(float) + + def test_slow(self): + """Test slow interp against known result.""" + xp = np.array([[0, 1, 2, 3, 4], [-4, -3, -2, -1, 0]], dtype=np.float32) + fp = np.array([0, 2, 4, 6, 8], dtype=np.float32) + x = np.array([-1, 0.5, 2], dtype=np.float32) + expected = np.array([[0, 1, 4], [6, 8, 8]], dtype=np.float32) + result = slow_interp_same_y(x, xp, fp) + np.testing.assert_allclose(result, expected) + + @patch.dict("sys.modules", numba=None) + @patch("improver.ensemble_copula_coupling.utilities.slow_interp_same_y") + def test_slow_interp_same_y_called(self, interp_imp): + """Test that slow_interp_same_y is called if numba is not installed.""" + interpolate_multiple_rows_same_y( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + interp_imp.assert_called_once_with( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + + @skipIf(not (numba_installed), "numba not installed") + @patch("improver.ensemble_copula_coupling.numba_utilities.fast_interp_same_y") + def test_fast_interp_same_y_called(self, interp_imp): + """Test that fast_interp_same_y is called if numba is installed.""" + interpolate_multiple_rows_same_y( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + interp_imp.assert_called_once_with( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + + @skipIf(not (numba_installed), "numba not installed") + def test_fast(self): + """Test fast interp against known result.""" + xp = np.array([[0, 1, 2, 3, 4], [-4, -3, -2, -1, 0]], dtype=np.float32) + fp = np.array([0, 2, 4, 6, 8], dtype=np.float32) + x = np.array([-1, 0.5, 2], dtype=np.float32) + expected = np.array([[0, 1, 4], [6, 8, 8]], dtype=np.float32) + result = fast_interp_same_y(x, xp, fp) + np.testing.assert_allclose(result, expected) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_fast(self): + """Test that slow and fast versions give same result.""" + result_slow = slow_interp_same_y(self.x, self.xp, self.fp) + result_fast = fast_interp_same_y(self.x, self.xp, self.fp) + np.testing.assert_allclose(result_slow, result_fast) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_fast_unordered(self): + """Test that slow and fast versions give same result + when x is not sorted.""" + shuffled_x = self.x.copy() + np.random.shuffle(shuffled_x) + result_slow = slow_interp_same_y(shuffled_x, self.xp, self.fp) + result_fast = fast_interp_same_y(shuffled_x, self.xp, self.fp) + np.testing.assert_allclose(result_slow, result_fast) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_fast_repeated(self): + """Test that slow and fast versions give same result when + rows of xp contain repeats.""" + xp_repeat = self.xp.copy() + xp_repeat[:, 51] = xp_repeat[:, 50] + result_slow = slow_interp_same_y(self.x, xp_repeat, self.fp) + result_fast = fast_interp_same_y(self.x, xp_repeat, self.fp) + np.testing.assert_allclose(result_slow, result_fast) + + @skipIf(not (numba_installed), "numba not installed") + def test_slow_vs_multi(self): + """Test that slow interp gives same result as + interpolate_multiple_rows_same_y.""" + result_slow = slow_interp_same_y(self.x, self.xp, self.fp) + result_multiple = interpolate_multiple_rows_same_y(self.x, self.xp, self.fp) + np.testing.assert_allclose(result_slow, result_multiple) + + class TestInterpolateMultipleRowsSameX(IrisTest): """Test interpolate_multiple_rows""" From 52cad26de90d8e991e2373990eb356c883c67bdd Mon Sep 17 00:00:00 2001 From: "Zhiliang(Julian) Fan" <71359633+zfan001@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:01:06 +1100 Subject: [PATCH 04/18] Enhancements on new regridding code (#1560) * target domain bigger than source domain * add new unit test * add more unit tests * more fixing * minor fixing * Combine tests using pytest parametrize * minor fixing * more test * changes suggested from review * add check data type * minor changes Co-authored-by: Tom Gale --- improver/regrid/bilinear.py | 45 ++++-- improver/regrid/grid.py | 100 ++++++++++++- improver/regrid/landsea2.py | 38 ++++- .../regrid/test_RegridWithLandSeaMask.py | 140 ++++++++++-------- improver_tests/regrid/test_grid.py | 18 +++ 5 files changed, 264 insertions(+), 77 deletions(-) diff --git a/improver/regrid/bilinear.py b/improver/regrid/bilinear.py index 685e182953..074e32eef7 100644 --- a/improver/regrid/bilinear.py +++ b/improver/regrid/bilinear.py @@ -34,10 +34,11 @@ bilinear_land_sea.rst """ -from typing import Tuple +from typing import Tuple, Union import numpy as np from numpy import ndarray +from numpy.ma.core import MaskedArray from improver.regrid.grid import similar_surface_classify from improver.regrid.idw import ( @@ -49,7 +50,9 @@ NUM_NEIGHBOURS = 4 -def apply_weights(indexes: ndarray, in_values: ndarray, weights: ndarray) -> ndarray: +def apply_weights( + indexes: ndarray, in_values: Union[ndarray, MaskedArray], weights: ndarray +) -> Union[ndarray, MaskedArray]: """ Apply bilinear weight of source points for target value. @@ -64,11 +67,19 @@ def apply_weights(indexes: ndarray, in_values: ndarray, weights: ndarray) -> nda Returns: Regridded values for target points. """ - in_values_expanded = (np.ma.filled(in_values, np.nan))[indexes] + input_array_masked = False + if isinstance(in_values, MaskedArray): + input_array_masked = True + in_values = np.ma.filled(in_values, np.nan) + in_values_expanded = in_values[indexes] + weighted = np.transpose( np.multiply(np.transpose(weights), np.transpose(in_values_expanded)) ) out_values = np.sum(weighted, axis=1) + if input_array_masked: + out_values = np.ma.masked_invalid(out_values) + return out_values @@ -118,8 +129,10 @@ def basic_indexes( # needs change at relevant boundary lat_max_in, lon_max_in = in_latlons.max(axis=0) lat_max_out, lon_max_out = out_latlons.max(axis=0) + lat_max_equal = np.isclose(lat_max_in, lat_max_out) lon_max_equal = np.isclose(lon_max_in, lon_max_out) + if lat_max_equal or lon_max_equal: indexes = adjust_boundary_indexes( in_lons_size, @@ -130,6 +143,7 @@ def basic_indexes( out_latlons, indexes, ) + return indexes @@ -177,17 +191,20 @@ def adjust_boundary_indexes( point_lat_lon_max_index = np.where( np.isclose(out_latlons[point_lat_max, 1], lon_max_in) )[0] - point_lat_lon_max = point_lat_max[point_lat_lon_max_index[0]] - point_lat_max = np.delete( - point_lat_max, np.where(point_lat_max == point_lat_lon_max)[0] - ) - point_lon_max = np.delete( - point_lon_max, np.where(point_lon_max == point_lat_lon_max)[0] - ) - indexes[point_lat_lon_max, 2] = indexes[point_lat_lon_max, 0] - indexes[point_lat_lon_max, 1] = indexes[point_lat_lon_max, 2] - 1 - indexes[point_lat_lon_max, 0] = indexes[point_lat_lon_max, 1] - in_lons_size - indexes[point_lat_lon_max, 3] = indexes[point_lat_lon_max, 0] + 1 + + # if point_lat_lon_max_index exists, handle it. + if point_lat_lon_max_index: + point_lat_lon_max = point_lat_max[point_lat_lon_max_index[0]] + point_lat_max = np.delete( + point_lat_max, np.where(point_lat_max == point_lat_lon_max)[0] + ) + point_lon_max = np.delete( + point_lon_max, np.where(point_lon_max == point_lat_lon_max)[0] + ) + indexes[point_lat_lon_max, 2] = indexes[point_lat_lon_max, 0] + indexes[point_lat_lon_max, 1] = indexes[point_lat_lon_max, 2] - 1 + indexes[point_lat_lon_max, 0] = indexes[point_lat_lon_max, 1] - in_lons_size + indexes[point_lat_lon_max, 3] = indexes[point_lat_lon_max, 0] + 1 if lat_max_equal: indexes[point_lat_max, 1] = indexes[point_lat_max, 0] diff --git a/improver/regrid/grid.py b/improver/regrid/grid.py index 76e1be19e0..5607317989 100644 --- a/improver/regrid/grid.py +++ b/improver/regrid/grid.py @@ -40,9 +40,27 @@ from numpy.ma.core import MaskedArray from scipy.interpolate import RegularGridInterpolator +from improver.utilities.cube_manipulation import sort_coord_in_cube from improver.utilities.spatial import calculate_grid_spacing, lat_lon_determine +def ensure_ascending_coord(cube: Cube) -> Cube: + """ + Check if cube coordinates ascending. if not, make it ascending + + Args: + cube: + Input source cube. + + Returns: + Cube with ascending coordinates + """ + for ax in ("x", "y"): + if cube.coord(axis=ax).points[0] > cube.coord(axis=ax).points[-1]: + cube = sort_coord_in_cube(cube, cube.coord(axis=ax).standard_name) + return cube + + def calculate_input_grid_spacing(cube_in: Cube) -> Tuple[float, float]: """ Calculate grid spacing in latitude and logitude. @@ -357,7 +375,7 @@ def create_regrid_cube(cube_array: ndarray, cube_in: Cube, cube_out: Cube) -> Cu target cube (for target grid information) Returns: - Regridded result cube + Regridded result cube """ # generate a cube based on new data and cube_in cube_v = Cube( @@ -384,8 +402,84 @@ def create_regrid_cube(cube_array: ndarray, cube_in: Cube, cube_out: Cube) -> Cu cube_v.add_dim_coord(cube_out.coord(cord_2), ndim + 1) # add all aus_coords from cube_in + for coord in cube_in.aux_coords: - dims = np.array(cube_in.coord_dims(coord)) + 1 - cube_v.add_aux_coord(coord.copy(), dims) + cube_v.add_aux_coord(coord.copy(), cube_in.coord_dims(coord)) return cube_v + + +def group_target_points_with_source_domain( + cube_in: Cube, out_latlons: ndarray +) -> Tuple[ndarray, ndarray]: + """ + Group cube_out's grid points into outside or inside cube_in's domain. + + Args: + cube_in: + Source cube. + out_latlons: + Target points's latitude-longitudes. + + Returns: + - Index array of target points outside input domain. + - Index array of target points inside input domain. + """ + + # get latitude and longitude coordinates of cube_in + lat_coord = cube_in.coord(axis="y").points + lon_coord = cube_in.coord(axis="x").points + + in_lat_max, in_lat_min = np.max(lat_coord), np.min(lat_coord) + in_lon_max, in_lon_min = np.max(lon_coord), np.min(lon_coord) + + lat = out_latlons[:, 0] + lon = out_latlons[:, 1] + + # check target point coordinates inside/outside input source domain + in_domain_lat = np.logical_and(lat >= in_lat_min, lat <= in_lat_max) + in_domain_lon = np.logical_and(lon >= in_lon_min, lon <= in_lon_max) + in_domain = np.logical_and(in_domain_lat, in_domain_lon) + + outside_input_domain_index = np.where(np.logical_not(in_domain))[0] + inside_input_domain_index = np.where(in_domain)[0] + + return outside_input_domain_index, inside_input_domain_index + + +def mask_target_points_outside_source_domain( + total_out_point_num: int, + outside_input_domain_index: ndarray, + inside_input_domain_index: ndarray, + regrid_result: Union[ndarray, MaskedArray], +) -> Union[ndarray, MaskedArray]: + """ + Mask target points outside cube_in's domain. + + Args: + total_out_point_num: + Total number of target points + outside_input_domain_index: + Index array of target points outside input domain. + inside_input_domain_index: + Index array of target points inside input domain. + regrid_result: + Array of regridded result in (lat*lon,....) or (projy*projx,...). + + Returns: + Array of regridded result in (lat*lon,....) or (projy*projx,...). + """ + # masked cube_out grid points which are out of cube_in range + + output_shape = [total_out_point_num] + list(regrid_result.shape[1:]) + if isinstance(regrid_result, np.ma.MaskedArray): + output = np.ma.zeros(output_shape, dtype=np.float32) + output.mask = np.full(output_shape, True, dtype=bool) + output.mask[inside_input_domain_index] = regrid_result.mask + output.data[inside_input_domain_index] = regrid_result.data + else: + output = np.zeros(output_shape, dtype=np.float32) + output[inside_input_domain_index] = regrid_result + output[outside_input_domain_index] = np.nan + + return output diff --git a/improver/regrid/landsea2.py b/improver/regrid/landsea2.py index 8dd57074fa..776b6fa099 100644 --- a/improver/regrid/landsea2.py +++ b/improver/regrid/landsea2.py @@ -48,8 +48,11 @@ classify_input_surface_type, classify_output_surface_type, create_regrid_cube, + ensure_ascending_coord, flatten_spatial_dimensions, + group_target_points_with_source_domain, latlon_from_cube, + mask_target_points_outside_source_domain, similar_surface_classify, slice_cube_by_domain, slice_mask_cube_by_domain, @@ -99,8 +102,9 @@ def process(self, cube_in: Cube, cube_in_mask: Cube, cube_out_mask: Cube) -> Cub """ Regridding considering land_sea mask. please note cube_in must use lats/lons rectlinear system(GeogCS). cube_in_mask and cube_in could be - different resolution. cube_our could be either in lats/lons rectlinear - system or LambertAzimuthalEqualArea system. + different resolution. cube_out could be either in lats/lons rectlinear + system or LambertAzimuthalEqualArea system. Grid points in cube_out + domain but not in cube_in domain will be masked. Args: cube_in: @@ -114,6 +118,12 @@ def process(self, cube_in: Cube, cube_in_mask: Cube, cube_out_mask: Cube) -> Cub Returns: Regridded result cube. """ + # if cube_in's coordinate descending, make it assending. + # if mask considered, reverse mask cube's coordinate if descending + cube_in = ensure_ascending_coord(cube_in) + if WITH_MASK in self.regrid_mode: + cube_in_mask = ensure_ascending_coord(cube_in_mask) + # check if input source grid is on even-spacing, ascending lat/lon system # return grid spacing for latitude and logitude lat_spacing, lon_spacing = calculate_input_grid_spacing(cube_in) @@ -131,6 +141,7 @@ def process(self, cube_in: Cube, cube_in_mask: Cube, cube_out_mask: Cube) -> Cub # Subset the input cube so that extra spatial area beyond the output is removed # This is a performance optimisation to reduce the size of the dataset being processed + total_out_point_num = out_latlons.shape[0] lat_max, lon_max = out_latlons.max(axis=0) lat_min, lon_min = out_latlons.min(axis=0) if WITH_MASK in self.regrid_mode: @@ -142,6 +153,16 @@ def process(self, cube_in: Cube, cube_in_mask: Cube, cube_out_mask: Cube) -> Cub cube_in, (lat_max, lon_max, lat_min, lon_min) ) + # group cube_out's grid points into outside or inside cube_in's domain + ( + outside_input_domain_index, + inside_input_domain_index, + ) = group_target_points_with_source_domain(cube_in, out_latlons) + + # exclude out-of-input-domain target point here + if len(outside_input_domain_index) > 0: + out_latlons = out_latlons[inside_input_domain_index] + # Gather input latitude/longitudes from input cube in_latlons = latlon_from_cube(cube_in) # Number of grid points in X dimension is used to work out length of flattened array @@ -158,7 +179,12 @@ def process(self, cube_in: Cube, cube_in_mask: Cube, cube_out_mask: Cube) -> Cub if WITH_MASK in self.regrid_mode: in_classified = classify_input_surface_type(cube_in_mask, in_latlons) + out_classified = classify_output_surface_type(cube_out_mask) + + if len(outside_input_domain_index) > 0: + out_classified = out_classified[inside_input_domain_index] + # Identify mismatched surface types from input and output classifications surface_type_mask = similar_surface_classify( in_classified, out_classified, indexes @@ -220,6 +246,14 @@ def process(self, cube_in: Cube, cube_in_mask: Cube, cube_out_mask: Cube) -> Cub # apply bilinear rule output_flat = apply_weights(indexes, in_values, weights) + # check if we need mask cube_out grid points which are out of cube_in range + if len(outside_input_domain_index) > 0: + output_flat = mask_target_points_outside_source_domain( + total_out_point_num, + outside_input_domain_index, + inside_input_domain_index, + output_flat, + ) # Un-flatten spatial dimensions and put into output cube output_array = unflatten_spatial_dimensions( output_flat, cube_out_mask, in_values, lats_index, lons_index diff --git a/improver_tests/regrid/test_RegridWithLandSeaMask.py b/improver_tests/regrid/test_RegridWithLandSeaMask.py index 91d43878e8..996ee2638c 100644 --- a/improver_tests/regrid/test_RegridWithLandSeaMask.py +++ b/improver_tests/regrid/test_RegridWithLandSeaMask.py @@ -36,11 +36,13 @@ # not using "set_up_variable_cube" because of different spacing at lat/lon import numpy as np +import pytest from improver.regrid.bilinear import basic_indexes from improver.regrid.grid import calculate_input_grid_spacing, latlon_from_cube from improver.regrid.landsea import RegridLandSea from improver.synthetic_data.set_up_test_cubes import set_up_variable_cube +from improver.utilities.pad_spatial import pad_cube_with_halo def modify_cube_coordinate_value(cube, coord_x, coord_y): @@ -57,12 +59,12 @@ def modify_cube_coordinate_value(cube, coord_x, coord_y): def define_source_target_grid_data(): """ define cube_in, cube_in_mask,cube_out_mask using assumed data """ # source (input) grid - in_lats = np.linspace(0, 15, 4) - in_lons = np.linspace(0, 40, 5) + in_lats = np.linspace(0, 15, 4, dtype=np.float32) + in_lons = np.linspace(0, 40, 5, dtype=np.float32) # target (output) grid - out_lats = np.linspace(0, 14, 8) - out_lons = np.linspace(5, 35, 11) + out_lats = np.linspace(0, 14, 8, dtype=np.float32) + out_lons = np.linspace(5, 35, 11, dtype=np.float32) # assume a set of nwp data data = np.arange(20).reshape(4, 5).astype(np.float32) @@ -101,12 +103,12 @@ def define_source_target_grid_data(): def define_source_target_grid_data_same_domain(): """ define cube_in, cube_in_mask,cube_out_mask, assume the same domain """ # source (input) grid - in_lats = np.linspace(0, 15, 4) - in_lons = np.linspace(0, 40, 5) + in_lats = np.linspace(0, 15, 4, dtype=np.float32) + in_lons = np.linspace(0, 40, 5, dtype=np.float32) # target (output) grid - out_lats = np.linspace(0, 15, 7) - out_lons = np.linspace(5, 40, 9) + out_lats = np.linspace(0, 15, 7, dtype=np.float32) + out_lons = np.linspace(0, 40, 9, dtype=np.float32) # assume a set of nwp data data = np.arange(20).reshape(4, 5).astype(np.float32) @@ -220,7 +222,7 @@ def test_regrid_nearest_with_mask_2(): [0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3], [0, 1, 1, 1, 7, 2, 7, 3, 3, 3, 3], [5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8], - [5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9], + [5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8], [10, 11, 11, 11, 7, 7, 7, 8, 8, 8, 14], [10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14], [10, 11, 11, 11, 12, 12, 7, 13, 13, 13, 14], @@ -258,62 +260,26 @@ def test_regrid_bilinear_with_mask_2(): expected_results = np.array( [ - [0.5, 0.8, 1.40096, 3.2916, 2.0, 2.0, 2.0, 4.94333, 3.25586, 3.2, 3.5], - [2.5, 2.8, 3.1, 3.4, 5.48911, 2.76267, 6.32926, 4.6, 4.9, 5.2, 5.5], - [4.5, 4.8, 5.1, 5.4, 5.7, 7.0154, 6.3, 6.6, 6.9, 7.2, 7.5], - [6.5, 6.8, 7.1, 7.4, 7.7, 7.0, 7.19033, 7.6681, 7.6618, 9.2, 9.5], - [ - 8.5, - 8.8, - 9.1, - 9.4, - 8.10633, - 7.0, - 7.0, - 7.62915, - 7.21672, - 9.11434, - 10.52363, - ], - [ - 10.5, - 10.8, - 11.00012, - 11.01183, - 13.15439, - 12.0, - 12.3, - 12.6, - 12.9, - 13.71286, - 15.74504, - ], + [0.5, 0.8, 1.401, 3.292, 2.0, 2.0, 2.0, 4.943, 3.256, 3.2, 3.5], + [2.5, 2.8, 3.1, 3.4, 5.489, 2.763, 6.329, 4.6, 4.9, 5.2, 5.5], + [4.5, 4.8, 5.1, 5.4, 5.7, 6.985, 6.3, 6.6, 6.9, 7.2, 7.5], + [6.5, 6.8, 7.1, 7.4, 7.7, 7.0, 7.19, 7.668, 7.662, 9.2, 9.5], + [8.5, 8.8, 9.1, 9.4, 8.106, 7.0, 7.0, 7.629, 7.217, 9.114, 10.524], + [10.5, 10.8, 11.0, 11.012, 13.154, 12.0, 12.3, 12.6, 12.9, 13.713, 15.745], [ 12.5, 12.8, - 12.23411, - 13.25881, - 14.14155, + 12.234, + 13.259, + 14.142, 14.0, - 8.07328, + 8.073, 14.6, 14.9, - 14.96332, - 16.3334, - ], - [ - 14.5, - 14.8, - 15.0997, - 14.22659, - 15.50905, - 16.0, - 9.8733, - 16.6, - 16.9, - 16.91114, - 17.03773, + 14.963, + 16.333, ], + [14.5, 14.8, 15.1, 14.227, 15.509, 16.0, 9.873, 16.6, 16.9, 16.911, 17.038], ] ) @@ -334,3 +300,61 @@ def test_regrid_bilinear_with_mask_2(): np.testing.assert_allclose( regrid_bilinear_with_mask.data, expected_results, atol=1e-3 ) + + +@pytest.mark.parametrize("regridder", ("nearest", "bilinear")) +@pytest.mark.parametrize("landmask", (True, False)) +@pytest.mark.parametrize("maskedinput", (True, False)) +def test_target_domain_bigger_than_source_domain(regridder, landmask, maskedinput): + """Test regridding when target domain is bigger than source domain""" + + # set up source cube, target cube and land-sea mask cube + cube_in, cube_out_mask, cube_in_mask = define_source_target_grid_data_same_domain() + + # add a circle of grid points so that output domain is much bigger than input domain + width_x, width_y = 2, 4 # lon,lat + cube_out_mask_pad = pad_cube_with_halo(cube_out_mask, width_x, width_y) + + if landmask: + with_mask = "-with-mask" + else: + with_mask = "" + cube_in_mask = None + regrid_mode = f"{regridder}{with_mask}-2" + + if maskedinput: + # convert the input data to a masked array with no values covered by the mask + cube_in_masked_data = np.ma.masked_array(cube_in.data, mask=False) + cube_in.data = cube_in_masked_data + + # run the regridding + regridderLandSea = RegridLandSea( + regrid_mode=regrid_mode, landmask=cube_in_mask, landmask_vicinity=250000000, + ) + regrid_out = regridderLandSea(cube_in, cube_out_mask) + regrid_out_pad = regridderLandSea(cube_in, cube_out_mask_pad) + + # check that results inside the padding matches the same regridding without padding + np.testing.assert_allclose( + regrid_out.data, regrid_out_pad.data[width_y:-width_y, width_x:-width_x], + ) + + # check results in the padded area + if maskedinput: + # masked array input should result in masked array output + assert hasattr(regrid_out_pad.data, "mask") + assert regrid_out_pad.dtype == np.float32 + regrid_out_pad.data.mask[width_y:-width_y, width_x:-width_x] = True + np.testing.assert_array_equal( + regrid_out_pad.data.mask, + np.full_like(regrid_out_pad.data, True, dtype=np.bool), + ) + else: + assert not hasattr(regrid_out_pad.data, "mask") + assert regrid_out_pad.dtype == np.float32 + # fill the area inside the padding with NaNs + regrid_out_pad.data[width_y:-width_y, width_x:-width_x] = np.nan + # this should result in the whole grid being NaN + np.testing.assert_array_equal( + regrid_out_pad.data, np.full_like(regrid_out_pad.data, np.nan) + ) diff --git a/improver_tests/regrid/test_grid.py b/improver_tests/regrid/test_grid.py index 1eea452b6d..7eee3f2ba9 100644 --- a/improver_tests/regrid/test_grid.py +++ b/improver_tests/regrid/test_grid.py @@ -40,6 +40,7 @@ from improver.regrid.grid import ( calculate_input_grid_spacing, create_regrid_cube, + ensure_ascending_coord, flatten_spatial_dimensions, get_cube_coord_names, latlon_from_cube, @@ -146,6 +147,23 @@ def test_flatten_spatial_dimensions(request, fixture_name): np.testing.assert_equal(flat[0:2, :], [[1, 21, 41], [2, 22, 42]]) +@pytest.mark.parametrize("flip", (True, False)) +def test_ensure_ascending_coord(flip): + """Test the ensure_ascending_coord function""" + + """Set up a lat/lon cube""" + lat_lon_cube = set_up_variable_cube(np.ones((5, 5), dtype=np.float32)) + lon_coord = lat_lon_cube.coord("longitude").points + lat_coord = lat_lon_cube.coord("latitude").points + if flip: + lat_lon_cube.coord("longitude").points = lon_coord[::-1] + lat_lon_cube.coord("latitude").points = lat_coord[::-1] + lat_lon_cube = ensure_ascending_coord(lat_lon_cube) + + np.testing.assert_allclose(lat_lon_cube.coord("latitude").points, lat_coord) + np.testing.assert_allclose(lat_lon_cube.coord("longitude").points, lon_coord) + + class Test_calculate_input_grid_spacing(IrisTest): """Test the calculate_input_grid_spacing function""" From fab09e63a901dd0b906682fd3f38bf623535e89e Mon Sep 17 00:00:00 2001 From: Belinda Trotta <73675905+btrotta-bom@users.noreply.github.com> Date: Tue, 30 Nov 2021 09:31:48 +1100 Subject: [PATCH 05/18] Add tests (#1626) Co-authored-by: Belinda Trotta --- .../test_utilities.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/improver_tests/ensemble_copula_coupling/test_utilities.py b/improver_tests/ensemble_copula_coupling/test_utilities.py index ed60141451..ffe9731ee0 100644 --- a/improver_tests/ensemble_copula_coupling/test_utilities.py +++ b/improver_tests/ensemble_copula_coupling/test_utilities.py @@ -570,6 +570,28 @@ def test_slow_vs_multi(self): result_multiple = interpolate_multiple_rows_same_x(self.x, self.xp, self.fp) np.testing.assert_allclose(result_slow, result_multiple) + @patch.dict("sys.modules", numba=None) + @patch("improver.ensemble_copula_coupling.utilities.slow_interp_same_x") + def test_slow_interp_same_x_called(self, interp_imp): + """Test that slow_interp_same_x is called if numba is not installed.""" + interpolate_multiple_rows_same_x( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + interp_imp.assert_called_once_with( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + + @skipIf(not (numba_installed), "numba not installed") + @patch("improver.ensemble_copula_coupling.numba_utilities.fast_interp_same_x") + def test_fast_interp_same_x_called(self, interp_imp): + """Test that fast_interp_same_x is called if numba is installed.""" + interpolate_multiple_rows_same_x( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + interp_imp.assert_called_once_with( + mock.sentinel.x, mock.sentinel.xp, mock.sentinel.fp + ) + if __name__ == "__main__": unittest.main() From cf96e414e05da11ef511fa1b92c64f88345ecf86 Mon Sep 17 00:00:00 2001 From: Tom Gale Date: Wed, 1 Dec 2021 11:37:02 +1100 Subject: [PATCH 06/18] Better handling and documentation of dependencies (#1589) * Rename and add environments * Thin out requirements in setup.cfg, add note * Remove improver_tests tests from built package The previous exclude was not effective - it excluded the top level directory, but not the subdirectories containing all the test source code files. * Skip checksum sorted test if file not available * Pytest skips for stratify * Move stratify import inside function * Remove duplicated statsmodels * Better explanation for latest environment * Add sphinx typehints to conda-forge tests section * Run security checks on all environments * Separate coverage and no-coverage environments * Add detailed pinning to A/B environments * Pin numpy/cartopy in latest environment Unit test failures occur with older versions * Start documentation * Remove dateutil dependency * More documentation * Remove timezone database dependence in unit test * Fix duplicated name in conda-forge environment * Add note that not all CLIs are available with environment_b * Remove leftover expected data from test_process * Fix flake8 * Remove not-really-working install_requires section * Clarify conda-forge environment comment * Move latest environment tests to schedule * Fix actions YAML * Reschedule * Fix scheduling, add manual running, metoppv-only * Add timezone mask test on europe lat/lon domain --- .github/workflows/{tests.yml => ci.yml} | 105 +++++------- .github/workflows/scheduled.yml | 137 +++++++++++++++ doc/source/Dependencies.rst | 162 ++++++++++++++++++ doc/source/Running-at-your-site.rst | 11 +- doc/source/index.rst | 1 + envs/conda-forge.yml | 28 +++ ...ment_py37_iris30.yml => environment_a.yml} | 37 ++-- envs/environment_b.yml | 41 +++++ envs/environment_py38_iris30.yml | 40 ----- envs/latest.yml | 44 +++++ improver/metadata/amend.py | 8 +- .../psychrometric_calculations.py | 3 +- improver_tests/acceptance/test_checksums.py | 5 +- .../test_interpolate_using_difference.py | 1 + .../acceptance/test_phase_change_level.py | 1 + .../test_GenerateTimezoneMask.py | 137 +++++++++++---- .../test_PhaseChangeLevel.py | 4 +- setup.cfg | 42 +---- 18 files changed, 600 insertions(+), 207 deletions(-) rename .github/workflows/{tests.yml => ci.yml} (53%) create mode 100644 .github/workflows/scheduled.yml create mode 100644 doc/source/Dependencies.rst create mode 100644 envs/conda-forge.yml rename envs/{environment_py37_iris30.yml => environment_a.yml} (51%) create mode 100644 envs/environment_b.yml delete mode 100644 envs/environment_py38_iris30.yml create mode 100644 envs/latest.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml similarity index 53% rename from .github/workflows/tests.yml rename to .github/workflows/ci.yml index 1e0de81ce2..cbedcb778f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,16 @@ -name: Tests +name: CI Tests -on: [pull_request, push] +on: + pull_request: + push: + workflow_dispatch: jobs: Sphinx-Pytest-Coverage: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - env: [py37_iris30, py38_iris30] + env: [environment_a, environment_b, conda-forge] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -16,45 +19,54 @@ jobs: # Increase this value to reset cache CACHE_NUMBER: 2 with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} conda info conda list - - name: sphinx-build + - name: sphinx documentation run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} make -C doc html SPHINXOPTS="-W --keep-going" - - name: pytest unit-tests & cov-report + - name: pytest without coverage + if: matrix.env != 'environment_a' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} + pytest + - name: pytest with coverage + if: matrix.env == 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} pytest --cov=improver --cov-report xml:coverage.xml - - name: codacy-coverage - if: env.CODACY_PROJECT_TOKEN + - name: codacy upload + if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} python-codacy-coverage -v -r coverage.xml env: CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} - - uses: codecov/codecov-action@v1 + - name: codecov upload + uses: codecov/codecov-action@v1 + if: matrix.env == 'environment_a' Codestyle-and-flake8: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - env: [py37_iris30] + env: [environment_a] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -63,41 +75,41 @@ jobs: # Increase this value to reset cache CACHE_NUMBER: 2 with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} conda info conda list - name: isort run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} isort --check-only . - name: black run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} black --check . - name: flake8 run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} flake8 improver improver_tests Safety-Bandit: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - env: [py37_iris30, py38_iris30] + env: [environment_a, environment_b, conda-forge] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -106,60 +118,27 @@ jobs: # Increase this value to reset cache CACHE_NUMBER: 2 with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} conda info conda list - name: safety run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} safety check || true - name: bandit run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} bandit -r improver - Type-checking: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - env: [py37_iris30] - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - id: cache - env: - # Increase this value to reset cache - CACHE_NUMBER: 2 - with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} - - name: conda env update - if: steps.cache.outputs.cache-hit != 'true' - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} - - name: conda info - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} - conda info - conda list - - name: mypy - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} - mypy improver || true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml new file mode 100644 index 0000000000..dcd2c412ca --- /dev/null +++ b/.github/workflows/scheduled.yml @@ -0,0 +1,137 @@ +name: Scheduled Tests + +on: + schedule: + - cron: '7 4 * * *' + workflow_dispatch: +jobs: + Sphinx-Pytest-Coverage: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: [latest] + if: github.repository_owner == 'metoppv' + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache + env: + # Increase this value to reset cache + CACHE_NUMBER: 2 + with: + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update + if: steps.cache.outputs.cache-hit != 'true' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda install -c conda-forge mamba + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + conda info + conda list + - name: sphinx documentation + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + make -C doc html SPHINXOPTS="-W --keep-going" + - name: pytest without coverage + if: matrix.env != 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + pytest + - name: pytest with coverage + if: matrix.env == 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + pytest --cov=improver --cov-report xml:coverage.xml + - name: codacy upload + if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + python-codacy-coverage -v -r coverage.xml + env: + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + - name: codecov upload + uses: codecov/codecov-action@v1 + if: matrix.env == 'environment_a' + Safety-Bandit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: [latest] + if: github.repository_owner == 'metoppv' + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache + env: + # Increase this value to reset cache + CACHE_NUMBER: 2 + with: + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update + if: steps.cache.outputs.cache-hit != 'true' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda install -c conda-forge mamba + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + conda info + conda list + - name: safety + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + safety check || true + - name: bandit + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + bandit -r improver + Type-checking: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: [latest] + if: github.repository_owner == 'metoppv' + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache + env: + # Increase this value to reset cache + CACHE_NUMBER: 2 + with: + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update + if: steps.cache.outputs.cache-hit != 'true' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda install -c conda-forge mamba + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + conda info + conda list + - name: mypy + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + mypy improver || true diff --git a/doc/source/Dependencies.rst b/doc/source/Dependencies.rst new file mode 100644 index 0000000000..9a5a63d4c6 --- /dev/null +++ b/doc/source/Dependencies.rst @@ -0,0 +1,162 @@ +Dependencies +================ + +.. contents:: Contents + :depth: 2 + + +IMPROVER builds on the functionality provided by a range of open source +libraries. + +Some of these libraries are widely used throughout the IMPROVER code, so are +considered as required for IMPROVER as a whole. + +Other libraries are only used in specific parts of IMPROVER. +These libraries are optional for IMPROVER as a whole, but are required to use +related parts of IMPROVER. +Optional installation of these libraries allows for smaller sized installations +of IMPROVER and reduces conda environment dependency solving difficulties. + +Required +----------------- + +cartopy +~~~~~~~~~~~~~~~~~ +Cartopy is used for grid projections and coordinate transformations. + +https://scitools.org.uk/cartopy/docs/stable/ + + +cftime +~~~~~~~~~~~~~~~~~ +cftime provides functions for handling time in NetCDF files according to the +Climate and Forecast (CF) conventions. + +https://unidata.github.io/cftime/ + + +cf-units +~~~~~~~~~~~~~~~~~ +cf-units provides units conversion following the Climate and Forecast (CF) +conventions. + +https://cf-units.readthedocs.io/en/stable/ + + +Clize +~~~~~~~~~~~~~~~~~ +Clize automatically generates command line interfaces (CLI) from Python function +signatures. + +https://clize.readthedocs.io/en/stable/ + + +Dask +~~~~~~~~~~~~~~~~~ +Dask lazy-loaded arrays are used (often via Iris) to reduce the amount of data +loaded into memory at once. + +https://dask.org/ + + +Iris +~~~~~~~~~~~~~~~~~ +Iris cubes are used as a primary data structure throughout the IMPROVER code. + +https://scitools-iris.readthedocs.io/en/stable/ + + +NetCDF4 +~~~~~~~~~~~~~~~~~ +Python library for reading and writing NetCDF data files via Iris. + +https://unidata.github.io/netcdf4-python/ + + +Numpy +~~~~~~~~~~~~~~~~~ +Multidimensional numerical array library, used as the basis for Iris cubes and +dask arrays. + +https://numpy.org/doc/stable/ + + +Scipy +~~~~~~~~~~~~~~~~~ +Scientific python library, used for a variety of statistical, image processing, +interpolation and spatial functions. + +https://docs.scipy.org/doc/scipy/reference/ + + +Sigtools +~~~~~~~~~~~~~~~~~ +Sigtools provides introspection tools for function signatures. +Sigtools is required by clize, so this dependency is needed anyway +despite minor usage in IMPROVER. + +https://sigtools.readthedocs.io/en/stable/ + + +Sphinx +~~~~~~~~~~~~~~~~~ +Sphinx is a documentation library for Python. IMPROVER requires it at runtime +to generate help strings from CLI function docstrings via clize. + +https://www.sphinx-doc.org/en/master/ + + +Optional dependencies +--------------------- + +python-stratify +~~~~~~~~~~~~~~~~~~ +Vectorised (cython) interpolation, particularly for vertical levels of the +atmosphere. + +https://github.com/SciTools/python-stratify + +Required for CLIs: ``interpolate-using-difference``, ``phase-change-level`` + +statsmodels +~~~~~~~~~~~~~~~~~~ +Estimation of statistical models, used for +:doc:`EMOS `. + +https://www.statsmodels.org/stable/ + +Required for CLIs: ``estimate-emos-coefficients`` + +numba +~~~~~~~~~~~~~~~~~~ +JIT compiler for numerical Python code, used for better computational performance. + +https://numba.readthedocs.io/en/stable/ + +Required for CLIs: ``generate-timezone-mask-ancillary`` + +PySTEPS +~~~~~~~~~~~~~~~~~~ +Probabilistic nowcasting of radar precipitation fields, used for nowcasting. + +https://pysteps.github.io/ + +Required for CLIs: ``nowcast-accumulate``, ``nowcast-extrapolate``, +``nowcast-optical-flow-from-winds`` + +pytz +~~~~~~~~~~~~~~~~~ +Timezone database for Python. + +https://pythonhosted.org/pytz/ + +Required for CLIs: ``generate-timezone-mask-ancillary`` + +timezonefinder +~~~~~~~~~~~~~~~~~~ +Lookup of timezone using geographic coordinates, used to generate timezone +grids. + +https://timezonefinder.readthedocs.io/en/stable/ + +Required for CLIs: ``generate-timezone-mask-ancillary`` diff --git a/doc/source/Running-at-your-site.rst b/doc/source/Running-at-your-site.rst index 91b8c6fd50..ac7cea27a1 100644 --- a/doc/source/Running-at-your-site.rst +++ b/doc/source/Running-at-your-site.rst @@ -22,13 +22,10 @@ or ``python setup.py install`` (``python setup.py develop``). Note that ``pip install`` will not work in an empty environment due to problems with installation of the dependency ``iris`` via pip. -Required dependencies are listed in -`environment_py37_iris30.yml `_, -and -`environment_py38_iris30.yml `_, -depending on your version requirements. Both environment files are used -to run the test suite on Github actions, so these conda environments -file should stay up to date with any dependency changes. +Example environments are included in the repository ``envs`` directory. +These environment files are used to run the test suite on Github actions, +so they should stay up to date with any dependency changes. See also +documentation about :doc:`use of dependencies in IMPROVER `. Alternatively, you can manually 'install' by downloading the code and putting the IMPROVER ``bin/`` directory in your PATH. diff --git a/doc/source/index.rst b/doc/source/index.rst index a09e7f34cc..85e24660ab 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -15,6 +15,7 @@ Guidance-for-Reviewing-Code How-to-implement-a-command-line-utility Running-at-your-site + Dependencies Ticket-Creation-and-Definition-of-Ready .. toctree:: diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml new file mode 100644 index 0000000000..e1d8793fed --- /dev/null +++ b/envs/conda-forge.yml @@ -0,0 +1,28 @@ +# This environment is intended to be used for the conda-forge improver-feedstock +# dependencies in https://github.com/conda-forge/improver-feedstock +# If this file is changed, recipe/meta.yaml in the improver-feedstock repository +# should also be updated. +# This environment should not include optional dependencies. +# This environment should pin versions so that all unit tests pass. +name: improver_conda_forge +channels: + - conda-forge +dependencies: + - python>=3.6 + # Included in improver-feedstock requirements + - cartopy<0.20 + - cftime<1.5 + - cf-units=2.1.5 + - clize + - dask + - iris>=3.0,<3.1 + - netCDF4 + - numpy<1.21 + - scipy + - sigtools + - sphinx + # Additional libraries to run tests, not included in improver-feedstock + - bandit + - filelock + - pytest + - sphinx-autodoc-typehints diff --git a/envs/environment_py37_iris30.yml b/envs/environment_a.yml similarity index 51% rename from envs/environment_py37_iris30.yml rename to envs/environment_a.yml index e6f0e0a5d4..fff6eaf147 100644 --- a/envs/environment_py37_iris30.yml +++ b/envs/environment_a.yml @@ -1,32 +1,29 @@ -# Use this file to create a conda environment using: -# conda env create --file envs/environment_py37_iris30.yml - -name: improver_py37_iris30 +# This environment is a representative example for usage of IMPROVER +name: improver_a channels: - conda-forge dependencies: - python=3.7 # Required - cartopy=0.19 - - cftime<1.5 + - cftime=1.2.1 - cf-units=2.1.5 - - clize - - dask - - iris=3.0 - - netCDF4 - - numpy<1.21 - - pandas - - python-dateutil - - python-stratify + - clize=4.1.1 + - dask=2021.8.1 + - iris=3.0.3 + - netCDF4=1.4.1 + - numpy=1.20.2 + - pandas=1.2.4 - pytz=2020.5 - - scipy=1.6 - - sigtools - - sphinx + - scipy=1.6.2 + - sigtools=2.0.2 + - sphinx=4.0.1 # Optional - - fastparquet - - numba + - fastparquet=0.7.1 + - statsmodels=0.12.2 + - numba=0.53.1 - pysteps=1.4.1 - - statsmodels + - python-stratify=0.1.1 - timezonefinder=4.1.0 # Development - astroid @@ -35,7 +32,7 @@ dependencies: - codacy-coverage - filelock - flake8 - - isort=5.* + - isort=5.7.0 - mock - mypy - pytest diff --git a/envs/environment_b.yml b/envs/environment_b.yml new file mode 100644 index 0000000000..82057101bf --- /dev/null +++ b/envs/environment_b.yml @@ -0,0 +1,41 @@ +# This environment is a representative example for usage of IMPROVER +# It does not include dependencies for the following IMPROVER CLIs: +# estimate-emos-coefficients, generate-timezone-mask-ancillary, +# nowcast-accumulate, nowcast-extrapolate and nowcast-optical-flow-from-winds +# See doc/source/Dependencies.rst for more information +name: improver_b +channels: + - conda-forge +dependencies: + - python=3.8 + # Required + - cartopy=0.19 + - cftime=1.2.1 + - cf-units=2.1.5 + - clize=4.2.0 + - dask=2021.8.1 + - iris=3.0.3 + - netCDF4=1.5.7 + - numpy=1.20 + - pandas=1.3.4 + - pytz=2020.5 + - scipy=1.6.2 + - sigtools=2.0.3 + - sphinx=4.0.2 + # Optional + - numba=0.53.1 + - python-stratify=0.2.post0 + # Development + - astroid + - bandit + - black=19.10b0 + - filelock + - flake8 + - isort=5 + - mock + - mypy + - pytest + - pytest-cov + - pytest-xdist + - safety + - sphinx-autodoc-typehints diff --git a/envs/environment_py38_iris30.yml b/envs/environment_py38_iris30.yml deleted file mode 100644 index 413f894951..0000000000 --- a/envs/environment_py38_iris30.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Use this file to create a conda environment using: -# conda env create --file envs/environment_py38_iris30.yml - -name: improver_py38_iris30 -channels: - - conda-forge -dependencies: - - python=3.8 - # Required - - cartopy=0.19 - - cftime<1.5 - - cf-units=2.1.5 - - clize - - dask - - iris=3.0 - - netCDF4 - - numpy<1.21 - - pandas - - python-dateutil - - python-stratify - - pytz=2020.5 - - scipy=1.6 - - sigtools - - sphinx - # Optional - # Development - - astroid - - bandit - - black=19.10b0 - - codacy-coverage - - filelock - - flake8 - - isort=5.* - - mock - - mypy - - pytest - - pytest-cov - - pytest-xdist - - safety - - sphinx-autodoc-typehints diff --git a/envs/latest.yml b/envs/latest.yml new file mode 100644 index 0000000000..ae82212e3f --- /dev/null +++ b/envs/latest.yml @@ -0,0 +1,44 @@ +# This environment is intended to provide all optional dependencies and +# use as-recent-as-possible versions with minimal pinning. +# Acceptance test failures due to different output values are OK. +# Unit test failures are not OK - pinning should be used where needed to +# make the unit tests pass. +name: improver_latest +channels: + - conda-forge +dependencies: + - python=3 + # Required + - cartopy<0.20 + - cftime<1.5 + - cf-units=2.1.5 + - clize + - dask + - iris>=3.0 + - netCDF4 + - numpy<1.21 + - pytz + - scipy + - sigtools + - sphinx + # Optional + - python-stratify + - statsmodels + - numba + - pysteps + - timezonefinder + # Development + - astroid + - bandit + - black + - codacy-coverage + - filelock + - flake8 + - isort + - mock + - mypy + - pytest + - pytest-cov + - pytest-xdist + - safety + - sphinx-autodoc-typehints diff --git a/improver/metadata/amend.py b/improver/metadata/amend.py index 9d6a916187..28a3a30a9e 100644 --- a/improver/metadata/amend.py +++ b/improver/metadata/amend.py @@ -29,10 +29,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """Module containing utilities for modifying cube metadata""" -from datetime import datetime +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Union -from dateutil import tz from iris.cube import Cube, CubeList from improver.metadata.constants.mo_attributes import ( @@ -94,8 +93,9 @@ def set_history_attribute(cube: Cube, value: str, append: bool = False) -> None: If True, add to the existing history rather than replacing the existing attribute. Default is False. """ - tzinfo = tz.tzoffset("Z", 0) - timestamp = datetime.strftime(datetime.now(tzinfo), "%Y-%m-%dT%H:%M:%S%Z") + timestamp = datetime.strftime( + datetime.now(timezone(timedelta(0), name="Z")), "%Y-%m-%dT%H:%M:%S%Z" + ) new_history = "{}: {}".format(timestamp, value) if append and "history" in cube.attributes.keys(): cube.attributes["history"] += "; {}".format(new_history) diff --git a/improver/psychrometric_calculations/psychrometric_calculations.py b/improver/psychrometric_calculations/psychrometric_calculations.py index e7ada36ec6..99d221c101 100644 --- a/improver/psychrometric_calculations/psychrometric_calculations.py +++ b/improver/psychrometric_calculations/psychrometric_calculations.py @@ -38,7 +38,6 @@ from cf_units import Unit from iris.cube import Cube, CubeList from numpy import ndarray -from stratify import interpolate import improver.constants as consts from improver import BasePlugin @@ -587,6 +586,8 @@ def find_falling_level( Returns: Phase change level data asl. """ + from stratify import interpolate + # Create cube of heights above sea level for each height in # the wet bulb integral cube. asl = wb_int_data.copy() diff --git a/improver_tests/acceptance/test_checksums.py b/improver_tests/acceptance/test_checksums.py index 8cd01038c9..d08e8603cd 100644 --- a/improver_tests/acceptance/test_checksums.py +++ b/improver_tests/acceptance/test_checksums.py @@ -131,7 +131,10 @@ def test_checksums_sorted(): This test doesn't depend on having the acceptance test data available, so can run with the unit tests. """ - csum_paths = [str(path) for path in acc.acceptance_checksums().keys()] + try: + csum_paths = [str(path) for path in acc.acceptance_checksums().keys()] + except FileNotFoundError: + pytest.skip("no checksum file, likely due to package being installed") with temporary_sort_locale("C") as strcoll: csum_paths_sorted = sorted(csum_paths, key=functools.cmp_to_key(strcoll)) assert csum_paths == csum_paths_sorted diff --git a/improver_tests/acceptance/test_interpolate_using_difference.py b/improver_tests/acceptance/test_interpolate_using_difference.py index 2cea0b8ea3..fd51706ec2 100644 --- a/improver_tests/acceptance/test_interpolate_using_difference.py +++ b/improver_tests/acceptance/test_interpolate_using_difference.py @@ -85,6 +85,7 @@ def test_filling_with_nearest_use(tmp_path): """Test filling masked areas using difference interpolation, in this case with a hole in the corner of the data that requires use of nearest neighbour interpolation.""" + pytest.importorskip("stratify") kgo_dir = acc.kgo_root() / f"{CLI}/basic" kgo_path = kgo_dir / "sleet_rain_nearest_filled_kgo.nc" output_path = tmp_path / "output.nc" diff --git a/improver_tests/acceptance/test_phase_change_level.py b/improver_tests/acceptance/test_phase_change_level.py index 13a986f08d..035c721282 100644 --- a/improver_tests/acceptance/test_phase_change_level.py +++ b/improver_tests/acceptance/test_phase_change_level.py @@ -55,6 +55,7 @@ def test_phase_change(tmp_path, phase_type, kgo_name, horiz_interp): snow/sleet level sleet/rain level leaving below orography points unfilled. """ + pytest.importorskip("stratify") kgo_dir = acc.kgo_root() / f"{CLI}/basic" kgo_name = "{}_kgo.nc".format(kgo_name) kgo_path = kgo_dir / kgo_name diff --git a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py index 2a02423fa4..14aabf638c 100644 --- a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py +++ b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py @@ -89,11 +89,24 @@ def global_grid_fixture_360() -> Cube: return cube +@pytest.fixture(name="europe_grid") +def europe_grid_fixture() -> Cube: + data = np.zeros((10, 10), dtype=np.float32) + cube = set_up_variable_cube( + data, + name="template", + grid_spacing=1, + domain_corner=(45, -5), + attributes=GLOBAL_ATTRIBUTES, + ) + return cube + + @pytest.fixture(name="uk_grid") def uk_grid_fixture() -> Cube: """UK grid template""" - data = np.zeros((21, 22), dtype=np.float32) + data = np.zeros((11, 11), dtype=np.float32) cube = set_up_variable_cube( data, name="template", @@ -160,7 +173,7 @@ def test__get_coordinate_pairs(request, grid_fixture): expected_data = { "global_grid": [[-90.0, -180.0], [-90.0, -80.0], [90.0, 180.0]], "global_grid_360": [[-90.0, -180.0], [-90.0, -80.0], [90.0, 180.0]], - "uk_grid": [[44.517, -17.117], [45.548, -4.913], [62.026, 14.410]], + "uk_grid": [[44.517, -17.117], [45.548, -4.913], [54.263, -5.401]], } grid = request.getfixturevalue(grid_fixture) @@ -245,7 +258,7 @@ def test__create_template_cube(request, grid_fixture, include_dst): expected = { "global_grid": {"shape": (19, 37), "attributes": GLOBAL_ATTRIBUTES}, - "uk_grid": {"shape": (21, 22), "attributes": UK_ATTRIBUTES}, + "uk_grid": {"shape": (11, 11), "attributes": UK_ATTRIBUTES}, } # Set expected includes_daylight_savings attribute @@ -309,50 +322,98 @@ def test__group_timezones_empty_group(timezone_mask): # Expected data for process tests - -# ungrouped -GLOBAL_GRID = {"shape": (27, 19, 37), "min": -12 * 3600, "max": 14 * 3600} -# grouped -GLOBAL_GRID_GR = {"shape": (2, 19, 37), "min": -6 * 3600, "max": 6 * 3600} -UK_GRID_GR = {"shape": (2, 21, 22), "min": -6 * 3600, "max": 6 * 3600} - EXPECTED_TIME = {None: 1510286400, "20200716T1500Z": 1594911600} +GROUPED_MIN_MAX = {"min": -6 * 3600, "max": 6 * 3600} EXPECTED = { "ungrouped": { - "global": { - None: {**GLOBAL_GRID, "data": np.array([1, 1, 1, 1, 1, 0, 1, 1, 1, 1])}, - "20200716T1500Z": {**GLOBAL_GRID, "data": np.ones([10])}, - "indices": (12, 2), - }, "uk": { None: { - "shape": (4, 21, 22), - "min": -2 * 3600, + "shape": (3, 11, 11), + "min": -1 * 3600, + "max": 1 * 3600, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ), + }, + "20200716T1500Z": { + "shape": (4, 11, 11), + "min": -1 * 3600, + "max": 2 * 3600, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ], + ), + }, + }, + "europe": { + None: { + "shape": (2, 10, 10), + "min": 0, "max": 1 * 3600, - "data": np.array([1, 1, 0, 0, 0, 1]), + "data": np.array( + [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] + ), }, "20200716T1500Z": { - "shape": (5, 21, 22), - "min": -2 * 3600, + "shape": (3, 10, 10), + "min": 0, "max": 2 * 3600, - "data": np.array([1, 1, 1, 1, 0, 1]), + "data": np.array( + [ + [1, 0, 1, 1, 1, 1, 0, 0, 0, 0], + [0, 1, 0, 0, 0, 0, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ), }, - "indices": (2, 10), }, }, "grouped": { - "global": { - None: {**GLOBAL_GRID_GR, "data": np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1])}, + "uk": { + None: { + "shape": (2, 11, 11), + **GROUPED_MIN_MAX, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ), + }, "20200716T1500Z": { - **GLOBAL_GRID_GR, - "data": np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]), + "shape": (2, 11, 11), + **GROUPED_MIN_MAX, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1], + ], + ), }, - "indices": (0, 2), }, - "uk": { - None: {**UK_GRID_GR, "data": np.array([0, 0, 0, 0, 0, 1])}, - "20200716T1500Z": {**UK_GRID_GR, "data": np.array([0, 0, 1, 1, 0, 1])}, - "indices": (0, 9), + "europe": { + None: { + "shape": (2, 10, 10), + **GROUPED_MIN_MAX, + "data": np.array( + [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] + ), + }, + "20200716T1500Z": { + "shape": (2, 10, 10), + **GROUPED_MIN_MAX, + "data": np.array( + [[1, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 1, 1, 1, 1]] + ), + }, }, }, } @@ -360,7 +421,7 @@ def test__group_timezones_empty_group(timezone_mask): @pytest.mark.parametrize("grouping", ["ungrouped", "grouped"]) @pytest.mark.parametrize("time", [None, "20200716T1500Z"]) -@pytest.mark.parametrize("grid_fixture", ["global_grid", "global_grid_360", "uk_grid"]) +@pytest.mark.parametrize("grid_fixture", ["uk_grid", "europe_grid"]) def test_process(request, grid_fixture, time, grouping): """Test that the process method returns cubes that take the expected form for different grids and different dates. @@ -376,7 +437,6 @@ def test_process(request, grid_fixture, time, grouping): expected = EXPECTED[grouping][domain][time] expected_time = EXPECTED_TIME[time] - index = EXPECTED[grouping][domain]["indices"] grid = request.getfixturevalue(grid_fixture) @@ -393,5 +453,14 @@ def test_process(request, grid_fixture, time, grouping): assert ( result.coord("UTC_offset").bounds.dtype == TIME_COORDS["UTC_offset"].dtype ) + # slice the first spatial dimension to moderate size of expected arrays + assert_array_equal(result.data[:, 9, :], expected["data"]) - assert_array_equal(result.data[index][::4], expected["data"]) + # check each spatial location in the UTC_offset dimension + zone_count = np.count_nonzero(result.data, axis=0) + if grouping == "grouped": + # grouped outputs have a single UTC_offset with a non-zero entry + assert_array_equal(zone_count, 1) + else: + # ungrouped outputs have a single UTC_offset with a zero entry + assert_array_equal(zone_count, expected["shape"][0] - 1) diff --git a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py index 237e4b89b7..e061baafb3 100644 --- a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py +++ b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py @@ -34,6 +34,7 @@ import iris import numpy as np +import pytest from cf_units import Unit from iris.cube import CubeList from iris.tests import IrisTest @@ -109,6 +110,7 @@ class Test_find_falling_level(IrisTest): def setUp(self): """Set up arrays.""" + pytest.importorskip("stratify") self.wb_int_data = np.array( [ [[80.0, 80.0], [70.0, 50.0]], @@ -449,7 +451,7 @@ def setUp(self): """Set up orography and land-sea mask cubes. Also create temperature, pressure, and relative humidity cubes that contain multiple height levels.""" - + pytest.importorskip("stratify") self.setup_cubes_for_process() def setup_cubes_for_process(self, spatial_grid="equalarea"): diff --git a/setup.cfg b/setup.cfg index 73a8026068..74a7564144 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,45 +18,15 @@ packages = find: setup_requires = setuptools >= 38.3.0 setuptools_scm -install_requires = - cartopy - cftime == 1.0.1 - cf_units - clize - dask - netCDF4 - numpy - python-dateutil - pytz - scipy >= 1.3.0, < 1.4.0 - scitools-iris >= 3.0 - sigtools - sphinx - statsmodels - stratify +# Note: no install_requires run-time requirements are included here. +# Requirements are expected to be provided through another method such as conda. +# See envs directory at top level of repository. scripts = bin/improver [options.packages.find] -exclude = improver_tests - -[options.extras_require] -dev = - astroid - bandit - black == 19.10b0 - codacy-coverage - filelock - isort == 5.* - mock - mypy - pytest - pytest-cov - safety - sphinx-autodoc-typehints -full = - numba - pysteps == 1.3.2 - timezonefinder +exclude = + improver_tests + improver_tests.* [flake8] max-line-length = 100 From 75e9bd68002ffb28855e76ca17ebe19151fb2888 Mon Sep 17 00:00:00 2001 From: Kat <67630960+Kat-90@users.noreply.github.com> Date: Fri, 17 Dec 2021 08:34:23 +0000 Subject: [PATCH 07/18] Expanding on triangle time blending doc strings. (#1630) * Expanding on triangle blending doc strings. * Update improver/blending/blend_across_adjacent_points.py Co-authored-by: Stephen Moseley * Updated weights.py doc string. Co-authored-by: Stephen Moseley --- .../blending/blend_across_adjacent_points.py | 40 +++++++++++-------- improver/blending/weights.py | 11 +++-- improver/cli/blend_adjacent_points.py | 20 ++++++---- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/improver/blending/blend_across_adjacent_points.py b/improver/blending/blend_across_adjacent_points.py index a6e35d9577..076cc90ec7 100644 --- a/improver/blending/blend_across_adjacent_points.py +++ b/improver/blending/blend_across_adjacent_points.py @@ -44,11 +44,14 @@ class TriangularWeightedBlendAcrossAdjacentPoints(PostProcessingPlugin): """ - Apply a Weighted blend to a coordinate, using triangular weights at each - point in the coordinate. Returns a cube with the same coordinates as the - input cube, with each point in the coordinate of interest having been - blended with the adjacent points according to a triangular weighting - function of a specified width. + Applies a weighted blend to the data using a triangular weighting function + at each point in the specified dimension. The maximum weighting is applied + to the specified point, and weighting decreases linearly for neighbouring + points to zero at the specified triangle width. + + Returns a cube with the same coordinates as the input cube, with each point + in the dimension having been blended with the adjacent points according to + a triangular weighting function of a specified width. """ def __init__( @@ -62,20 +65,24 @@ def __init__( Args: coord: - The name of a coordinate dimension in the cube that we - will blend over. + The name of a coordinate dimension in the cube to be blended + over. central_point: Central point at which the output from the triangular weighted - blending will be calculated. + blending will be calculated. This should be in the units of the + units argument that is passed in. This value should be a point + on the coordinate for blending over. parameter_units: The units of the width of the triangular weighting function and the units of the central_point. This does not need to be the same as the units of the - coordinate we are blending over, but it should be possible to + coordinate being blending over, but it should be possible to convert between them. width: - The width of the triangular weighting function we will use - to blend. + The width from the triangle’s centre point, in units of the units + argument, which will determine the triangular weighting function + used to blend that specified point with its adjacent points. Beyond + this width the weighting drops to zero. """ self.coord = coord self.central_point = central_point @@ -142,17 +149,18 @@ def _find_central_point(self, cube: Cube) -> Cube: def process(self, cube: Cube) -> Cube: """ - Apply the weighted blend for each point in the given coordinate. + Apply the weighted blend for each point in the given dimension. Args: cube: Cube containing input for blending. Returns: - The processed cube, with the same coordinates as the input - central_cube. The points in one coordinate will be blended - with the adjacent points based on a triangular weighting - function of the specified width. + A processed cube, with the same coordinates as the input + central_cube. The points in one dimension corresponding to + the specified coordinate will be blended with the adjacent + points based on a triangular weighting function of the + specified width. """ # Extract the central point from the input cube. central_point_cube = self._find_central_point(cube) diff --git a/improver/blending/weights.py b/improver/blending/weights.py index a455ecf950..a8b714b77f 100644 --- a/improver/blending/weights.py +++ b/improver/blending/weights.py @@ -746,7 +746,7 @@ def __repr__(self) -> str: def triangular_weights( coord_vals: ndarray, midpoint: float, width: float ) -> ndarray: - """Create triangular weights. + """Calculate triangular weights. Args: coord_vals: @@ -755,10 +755,13 @@ def triangular_weights( midpoint: The centre point of the triangular function. width: - The width of the triangular function from the centre point. + The width from the triangle’s centre point, in units of the plugin's + units argument, which will determine the triangular weighting function + used to blend that specified point with its adjacent points. Beyond + this width the weighting drops to zero. Returns: - array of weights, sum of all weights should equal 1.0. + An array of weights, the sum of which should equal 1.0. """ def calculate_weight(point: float, slope: float) -> float: @@ -812,7 +815,7 @@ def process(self, cube: Cube, coord_name: str, midpoint: float) -> Cube: Returns: 1D cube of normalised (sum = 1.0) weights matching length - of input dimension to be blended + of input dimension to be blended. Raises: TypeError : input is not a cube diff --git a/improver/cli/blend_adjacent_points.py b/improver/cli/blend_adjacent_points.py index 227b31c40a..556e4ce8cc 100755 --- a/improver/cli/blend_adjacent_points.py +++ b/improver/cli/blend_adjacent_points.py @@ -49,9 +49,9 @@ def process( """Runs weighted blending across adjacent points. Uses the TriangularWeightedBlendAcrossAdjacentPoints to blend across - a particular coordinate. It does not collapse the coordinate, but - instead blends across adjacent points and puts the blended values back - in the original coordinate, with adjusted bounds. + the dimension of a particular coordinate. It does not collapse the + coordinate, but instead blends across adjacent points and puts the + blended values back in the original coordinate, with adjusted bounds. Args: cubes (list of iris.cube.Cube): @@ -64,10 +64,12 @@ def process( units argument that is passed in. This value should be a point on the coordinate for blending over. units (str): - Units of the central_point and width + Units of the central_point and width. width (float): - Width of the triangular weighting function used in the blending, - in the units of the units argument. + The width from the triangle’s centre point, in units of the units + argument, which will determine the triangular weighting function + used to blend that specified point with its adjacent points. Beyond + this width the weighting drops to zero. calendar (str) Calendar for parameter_unit if required. blend_time_using_forecast_period (bool): @@ -78,7 +80,11 @@ def process( Returns: iris.cube.Cube: - A processed Cube + A processed cube, with the same coordinates as the input + central_cube. The points in one dimension corresponding to + the specified coordinate will be blended with the adjacent + points based on a triangular weighting function of the + specified width. Raises: ValueError: From addb150a0b2c1a8be220d1d73022e245981dd1f1 Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Tue, 4 Jan 2022 15:07:07 +0000 Subject: [PATCH 08/18] Fix noise in precip accumulation thresholds (#1627) --- improver/cli/threshold.py | 10 ++++++---- improver/threshold.py | 9 ++++++++- improver_tests/threshold/test_BasicThreshold.py | 12 ++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/improver/cli/threshold.py b/improver/cli/threshold.py index ae298e983e..50b2c13098 100755 --- a/improver/cli/threshold.py +++ b/improver/cli/threshold.py @@ -110,8 +110,6 @@ def process( ValueError: If threshold_config and threshold_values are both set ValueError: If threshold_config is used for fuzzy thresholding """ - import numpy as np - from improver.metadata.probabilistic import in_vicinity_name_format from improver.threshold import BasicThreshold from improver.utilities.cube_manipulation import collapse_realizations @@ -129,7 +127,9 @@ def process( thresholds = [] fuzzy_bounds = [] for key in threshold_config.keys(): - thresholds.append(np.float32(key)) + # Ensure thresholds are float64 to avoid rounding errors during + # possible unit conversion. + thresholds.append(float(key)) # If the first threshold has no bounds, fuzzy_bounds is # set to None and subsequent bounds checks are skipped if threshold_config[key] == "None": @@ -137,7 +137,9 @@ def process( continue fuzzy_bounds.append(tuple(threshold_config[key])) else: - thresholds = [np.float32(x) for x in threshold_values] + # Ensure thresholds are float64 to avoid rounding errors during possible + # unit conversion. + thresholds = [float(x) for x in threshold_values] fuzzy_bounds = None each_threshold_func_list = [] diff --git a/improver/threshold.py b/improver/threshold.py index 8c7c02993d..c84f3642e5 100644 --- a/improver/threshold.py +++ b/improver/threshold.py @@ -241,6 +241,9 @@ def _add_threshold_coord(self, cube: Cube, threshold: float) -> None: Add a scalar threshold-type coordinate with correct name and units to a 2D slice containing thresholded data. + The 'threshold' coordinate will be float64 to avoid rounding errors + during possible unit conversion. + Args: cube: Cube containing thresholded data (1s and 0s) @@ -248,7 +251,7 @@ def _add_threshold_coord(self, cube: Cube, threshold: float) -> None: Value at which the data has been thresholded """ coord = iris.coords.DimCoord( - np.array([threshold], dtype=FLOAT_DTYPE), units=cube.units + np.array([threshold], dtype="float64"), units=cube.units ) coord.rename(self.threshold_coord_name) coord.var_name = "threshold" @@ -379,6 +382,10 @@ def process(self, input_cube: Cube) -> Cube: thresholded_cubes.append(cube) (cube,) = thresholded_cubes.merge() + # Re-cast to 32bit now that any unit conversion has already taken place. + cube.coord(var_name="threshold").points = cube.coord( + var_name="threshold" + ).points.astype(FLOAT_DTYPE) self._update_metadata(cube) enforce_coordinate_ordering(cube, ["realization", "percentile"]) diff --git a/improver_tests/threshold/test_BasicThreshold.py b/improver_tests/threshold/test_BasicThreshold.py index cefb9fc080..81977ffeef 100644 --- a/improver_tests/threshold/test_BasicThreshold.py +++ b/improver_tests/threshold/test_BasicThreshold.py @@ -449,6 +449,18 @@ def test_threshold_unit_conversion(self): result = plugin(self.rate_cube) self.assertArrayAlmostEqual(result.data, expected_result_array) + def test_threshold_unit_conversion_2(self): + """Test threshold coordinate points after undergoing unit conversion. + Specifically ensuring that small floating point values have no floating + point precision errors after the conversion (float equality check with no + tolerance).""" + plugin = Threshold([0.03, 0.09, 0.1], threshold_units="mm s-1") + result = plugin(self.rate_cube) + self.assertArrayEqual( + result.coord(var_name="threshold").points, + np.array([3e-5, 9.0e-05, 1e-4], dtype="float32"), + ) + def test_threshold_unit_conversion_fuzzy_factor(self): """Test for sensible fuzzy factor behaviour when units of threshold are different from input cube. A fuzzy factor of 0.75 is equivalent From f572bd5a593e47f750ea6669374e1c9264166486 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 6 Jan 2022 12:01:22 +0000 Subject: [PATCH 09/18] Weather code tree update (#1635) * Updates checksums following addition of mosg__model_configuration to wxcode-modal gridded test data * Updates weather code acceptance tests with new parameters following update to decision trees in the IMPROVER suite. - Now includes hail on UK - lightning as an accumulation and now present on global as well as UK grid * Updates checksums to only include changes applicable to this branch --- improver_tests/acceptance/SHA256SUMS | 73 +++++++++++++----------- improver_tests/acceptance/test_wxcode.py | 8 ++- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index fa37489613..59f9f49cf8 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -700,37 +700,42 @@ c798e3530eac9a45305711d5e91daa56d5005b826ce85bb48e2889e328ea965c ./wxcode-modal a6698b2aae8bc223a2d362ccb3047dd9882e6c0df21962e55c577e1ad5850e29 ./wxcode-modal/spot_ties/20201209T1700Z-weather_symbols-PT01H.nc 9290b2899eedfca038226c6707e100025ce68b0798972a60360845ccf36858d2 ./wxcode-modal/spot_ties/20201209T1800Z-weather_symbols-PT01H.nc ceb7106e5b8919590327a75d4a8b740fb1a2c7cb5a271a5bf980f0c1396c7d22 ./wxcode-modal/spot_ties/kgo.nc -a10668de8e2e0f506f4001eff6290adb3bc98eef49fd4a3eb304b166a60fe0ce ./wxcode/basic/kgo.nc -a10668de8e2e0f506f4001eff6290adb3bc98eef49fd4a3eb304b166a60fe0ce ./wxcode/basic/kgo_no_lightning.nc -59f399b9944948af2f4da472756ec91ca281fa19f28ec616aa50e429cabd760b ./wxcode/basic/probability_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold.nc -a9bf67edcc91d6e75af07a24ca5b779ea5d4014ce66a3406aa5a89a1ab7c6865 ./wxcode/basic/probability_of_low_and_medium_type_cloud_area_fraction_above_threshold.nc -cb5c16a5fd656ee157b40659187a4742f36872622b21b729ed893679cba889b4 ./wxcode/basic/probability_of_low_type_cloud_area_fraction_above_threshold.nc -68e5c9eab759d62cf399e5263245f49f5ce4c03958a626bba8d53ff0238e2c56 ./wxcode/basic/probability_of_lwe_thickness_of_precipitation_amount_above_threshold.nc -67f6d523920eb7ee108bff4f2c24a3b6dc188f3d5ede2943679e465b4b3f7cd4 ./wxcode/basic/probability_of_lwe_thickness_of_precipitation_amount_in_vicinity_above_threshold.nc -ad0095181a6131c7ff7dc068fea0f283573c1187c483d196f08bcb0c392b8e2a ./wxcode/basic/probability_of_lwe_thickness_of_sleetfall_amount_above_threshold.nc -c23b1c16af249c542f30509b8eefa8ccdf4e0b51e5d07936650c35ccc8bf53d6 ./wxcode/basic/probability_of_lwe_thickness_of_snowfall_amount_above_threshold.nc -a5e061dd1d1b476f4e9bf2d959a7ee267ceaeb41bc0ddb4f95e0208b2c267a37 ./wxcode/basic/probability_of_shower_condition_above_threshold.nc -65cf0f9c19d672a98cabf7f206faace697e506350fcb575ae56ee53a59b66f56 ./wxcode/basic/probability_of_thickness_of_rainfall_amount_above_threshold.nc -548ae40f3b027c819de1dfa791f859211b69c6cc57d7c8c17d9a9d2f24f96864 ./wxcode/basic/probability_of_visibility_in_air_below_threshold.nc -84a83c52fbb0e214c00f2f7936c245e4d44ce8d32a68a82a0138b1b50accbc8d ./wxcode/global/kgo.nc -bacfc882bb31c3c3f399dac4334818cce364c3488398593e07b0f3fe3f8818b0 ./wxcode/global/probability_of_low_and_medium_type_cloud_area_fraction_above_threshold.nc -24173fc8c9ddde1854e4a90c068e038ad1e8e3579bef1cda5bc6c5fd24f30062 ./wxcode/global/probability_of_low_type_cloud_area_fraction_above_threshold.nc -79d05a619d981811398bf6dd3a3bc235f21d9a30b998000ad7f45617a0bb30dc ./wxcode/global/probability_of_lwe_thickness_of_precipitation_amount_above_threshold.nc -9a4dbc8c6f1958efde70f618cf0aa6be858ce9f8b88ff445e7ef98905b13ee34 ./wxcode/global/probability_of_lwe_thickness_of_precipitation_amount_in_vicinity_above_threshold.nc -356d1469e500c19a7465fc1c4305ba394b27843557ae21fa653fc8d29091c7c8 ./wxcode/global/probability_of_lwe_thickness_of_sleetfall_amount_above_threshold.nc -cf2cebc9740adf7b3754bb6fc19ec6b92dfa0f0d0a266da752bb0c28eb44f440 ./wxcode/global/probability_of_lwe_thickness_of_snowfall_amount_above_threshold.nc -0d541cc310787f23979c3d9b43fd66d5188adc8bb62fc84dc40c04b2ee030a36 ./wxcode/global/probability_of_shower_condition_above_threshold.nc -70544bcbc20a173c990a91cad94c0f70d83906bfe8b9b6bd7e65cd79d8374ac5 ./wxcode/global/probability_of_thickness_of_rainfall_amount_above_threshold.nc -6bc02d671b45d3f56deee2fe6002c7c642115e5f481ec0d51c28ed7a9cef6b8a ./wxcode/global/probability_of_visibility_in_air_below_threshold.nc -59f399b9944948af2f4da472756ec91ca281fa19f28ec616aa50e429cabd760b ./wxcode/native_units/probability_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold.nc -a9bf67edcc91d6e75af07a24ca5b779ea5d4014ce66a3406aa5a89a1ab7c6865 ./wxcode/native_units/probability_of_low_and_medium_type_cloud_area_fraction_above_threshold.nc -cb5c16a5fd656ee157b40659187a4742f36872622b21b729ed893679cba889b4 ./wxcode/native_units/probability_of_low_type_cloud_area_fraction_above_threshold.nc -8121978e021117a08acae6782914f36329ac4d7cb956fc3a4a83f7572345c4ab ./wxcode/native_units/probability_of_lwe_thickness_of_precipitation_amount_above_threshold.nc -bf1f7b63fb5abf66375f162b371acf49f9ab7b327b4315bbaacb8d9ef95590f7 ./wxcode/native_units/probability_of_lwe_thickness_of_precipitation_amount_in_vicinity_above_threshold.nc -4daea2d64e5b55507b7d0cda37642a08e265e89e5f72584845182e45ffa37eab ./wxcode/native_units/probability_of_lwe_thickness_of_sleetfall_amount_above_threshold.nc -8699b5267ecfdba1259d4b3d0a9a7ef33b75838d5ded75d041d48938a556a5fa ./wxcode/native_units/probability_of_lwe_thickness_of_snowfall_amount_above_threshold.nc -a5e061dd1d1b476f4e9bf2d959a7ee267ceaeb41bc0ddb4f95e0208b2c267a37 ./wxcode/native_units/probability_of_shower_condition_above_threshold.nc -0ebf269ce0bd8bdb87322589163ec266c7e3b4fa62133e6db840910d942ad41e ./wxcode/native_units/probability_of_thickness_of_rainfall_amount_above_threshold.nc -9831c847f94f68b1a4917e9d68f123d54fc9f87447ff6d34fddf71bfa4737117 ./wxcode/native_units/probability_of_visibility_in_air_below_threshold.nc -10a3502534d286f0abb732bc4a4bb0b9e494a073b2d572693e080d5e4633d66d ./wxcode/wx_decision_tree_1h.json -f2b140ab62972b5fe5e52ddef4550a9fde440da825070999c0a51d08f10d060f ./wxcode/wx_decision_tree_3h.json +5a3a808f0cc7952e6cc4225d09136276f685c5fdd6798a795c0430aa65ca71fa ./wxcode/basic/kgo.nc +3b86d135373b44989c22bdfbc6cb0798e974ff1d77c4c6b6a83fce650863e59c ./wxcode/basic/kgo_no_lightning.nc +5c1950ac1e5f96a5eb1495369271b667bcc89b2deced21d3a51c70303ef2b4d8 ./wxcode/basic/probability_of_low_and_medium_type_cloud_area_fraction_above_threshold.nc +146d0f003cf6d9b41f87761cb18f52958a41050af716718568cdb2d09d33fdd5 ./wxcode/basic/probability_of_low_type_cloud_area_fraction_above_threshold.nc +97f5fa2e0517d6302fd4fe01d9669f40685a180e3a27506754e3c815319e0511 ./wxcode/basic/probability_of_lwe_graupel_and_hail_fall_rate_in_vicinity_above_threshold.nc +b76fff0f96958ba74f7b76167d16855223c737f464072287a7ce7e08566c0e30 ./wxcode/basic/probability_of_lwe_thickness_of_graupel_and_hail_fall_amount_above_threshold.nc +531927b4de9b8c2b2328811d6cfa6b221a9343bcb9defbe3b0f9eae0c798faa7 ./wxcode/basic/probability_of_lwe_thickness_of_precipitation_amount_above_threshold.nc +86a2bd791b84133d3bb41e08f4932362425d89708f34bb8983ed3860e7343312 ./wxcode/basic/probability_of_lwe_thickness_of_precipitation_amount_in_vicinity_above_threshold.nc +f4eb47c2a3f03ac9ca3c98e0072644df495055e369336947cd8e9a839c4ad7be ./wxcode/basic/probability_of_lwe_thickness_of_sleetfall_amount_above_threshold.nc +f20a9eabe867ea00e9570eb1177c01841b7606e2969287432d4a50ae888062a3 ./wxcode/basic/probability_of_lwe_thickness_of_snowfall_amount_above_threshold.nc +1768a398493741439fec057bb32a6b15f64ae237a251a7e8d7fafe78a1fb7e6a ./wxcode/basic/probability_of_number_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold.nc +ec818f319f2c137ab356f97a3980788357254f17c0761908b4e656170c0e5d37 ./wxcode/basic/probability_of_shower_condition_above_threshold.nc +1b8de3a02ac7dcf18ec9208a5e0b583b37c420093781b861b67eb5c8eaa37cd9 ./wxcode/basic/probability_of_thickness_of_rainfall_amount_above_threshold.nc +4fd37534bef88083d5097c443bf72209dc05a787b35a9319683e37fe994779e6 ./wxcode/basic/probability_of_visibility_in_air_below_threshold.nc +e497447b5440ef6c48bc435d0afc9c5c8b1571e77488545a31ac6122867e6ccc ./wxcode/global/kgo.nc +b34a701c91e44e7cc551e7aa4d42d94ad615cf8b0041113c166209a0c854e8ec ./wxcode/global/probability_of_low_and_medium_type_cloud_area_fraction_above_threshold.nc +d2266d931dab331b263e2f1d5f82e4c771f7ce6eaa87979ee0782b766e2aa598 ./wxcode/global/probability_of_low_type_cloud_area_fraction_above_threshold.nc +ea820a4650ec8fc5986fcaba2ce72651ee86da0ca774dc0501d9c6273072aa8d ./wxcode/global/probability_of_lwe_thickness_of_precipitation_amount_above_threshold.nc +4c39bdf6273648b2f8860420c8eda72955390c063b8a62ca3eeb48747a23a000 ./wxcode/global/probability_of_lwe_thickness_of_precipitation_amount_in_vicinity_above_threshold.nc +18e37334458f668e222683cb638a8d98989ce312db7a00167a3b04f2b71b9e23 ./wxcode/global/probability_of_lwe_thickness_of_sleetfall_amount_above_threshold.nc +005d5e870454e723102956d2993592f778b2058b076f4231b53283d9f5fd9d83 ./wxcode/global/probability_of_lwe_thickness_of_snowfall_amount_above_threshold.nc +efdb4a96e6dbd640677ac844bc34f8f38b63fd82608cf553ed5459eed384e27e ./wxcode/global/probability_of_number_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold.nc +8193d616a57b78ff43da1307d5ed6f4ea81a86f0bfa4c54b0dfc1757452e1a20 ./wxcode/global/probability_of_shower_condition_above_threshold.nc +4d12a52983136ecc7c9a38d32738eb7dfb445f0b1e0fd0568f34fd3d999d8d9f ./wxcode/global/probability_of_thickness_of_rainfall_amount_above_threshold.nc +8b81175391b5f32e15e00d447310cda5b5c53b2c3c5d39db12d1d56524606f0a ./wxcode/global/probability_of_visibility_in_air_below_threshold.nc +5c1950ac1e5f96a5eb1495369271b667bcc89b2deced21d3a51c70303ef2b4d8 ./wxcode/native_units/probability_of_low_and_medium_type_cloud_area_fraction_above_threshold.nc +146d0f003cf6d9b41f87761cb18f52958a41050af716718568cdb2d09d33fdd5 ./wxcode/native_units/probability_of_low_type_cloud_area_fraction_above_threshold.nc +ad0b750abc146e114ff828cacd56ddf409d4605953467f4d1cae4ac5b9df4e1a ./wxcode/native_units/probability_of_lwe_graupel_and_hail_fall_rate_in_vicinity_above_threshold.nc +b76fff0f96958ba74f7b76167d16855223c737f464072287a7ce7e08566c0e30 ./wxcode/native_units/probability_of_lwe_thickness_of_graupel_and_hail_fall_amount_above_threshold.nc +531927b4de9b8c2b2328811d6cfa6b221a9343bcb9defbe3b0f9eae0c798faa7 ./wxcode/native_units/probability_of_lwe_thickness_of_precipitation_amount_above_threshold.nc +86a2bd791b84133d3bb41e08f4932362425d89708f34bb8983ed3860e7343312 ./wxcode/native_units/probability_of_lwe_thickness_of_precipitation_amount_in_vicinity_above_threshold.nc +f4eb47c2a3f03ac9ca3c98e0072644df495055e369336947cd8e9a839c4ad7be ./wxcode/native_units/probability_of_lwe_thickness_of_sleetfall_amount_above_threshold.nc +f20a9eabe867ea00e9570eb1177c01841b7606e2969287432d4a50ae888062a3 ./wxcode/native_units/probability_of_lwe_thickness_of_snowfall_amount_above_threshold.nc +1768a398493741439fec057bb32a6b15f64ae237a251a7e8d7fafe78a1fb7e6a ./wxcode/native_units/probability_of_number_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold.nc +ec818f319f2c137ab356f97a3980788357254f17c0761908b4e656170c0e5d37 ./wxcode/native_units/probability_of_shower_condition_above_threshold.nc +1b8de3a02ac7dcf18ec9208a5e0b583b37c420093781b861b67eb5c8eaa37cd9 ./wxcode/native_units/probability_of_thickness_of_rainfall_amount_above_threshold.nc +554f9ed54f1092f31013b0e85de20e13932ca9533d50ac6316f170641901a209 ./wxcode/native_units/probability_of_visibility_in_air_below_threshold.nc +3de7bd3e25d9d5b9eafbef91ead03f1ebfa6202b9f90826dbe81ad99fdc74277 ./wxcode/wx_decision_tree_1h.json +80a0dc027f33a9f209d8304a92eb60fd204620492bea5bb9c0fe5c4d8a285edb ./wxcode/wx_decision_tree_3h.json diff --git a/improver_tests/acceptance/test_wxcode.py b/improver_tests/acceptance/test_wxcode.py index 54d766f892..5c1319fb7d 100644 --- a/improver_tests/acceptance/test_wxcode.py +++ b/improver_tests/acceptance/test_wxcode.py @@ -39,15 +39,17 @@ run_cli = acc.run_cli(CLI) ALL_PARAMS = [ - "lightning_flashes_per_unit_area_in_vicinity_above", "low_and_medium_type_cloud_area_fraction_above", "low_type_cloud_area_fraction_above", + "lwe_graupel_and_hail_fall_rate_in_vicinity_above", + "lwe_thickness_of_graupel_and_hail_fall_amount_above", "lwe_thickness_of_precipitation_amount_above", "lwe_thickness_of_precipitation_amount_in_vicinity_above", "lwe_thickness_of_sleetfall_amount_above", "lwe_thickness_of_snowfall_amount_above", - "thickness_of_rainfall_amount_above", + "number_of_lightning_flashes_per_unit_area_in_vicinity_above", "shower_condition_above", + "thickness_of_rainfall_amount_above", "visibility_in_air_below", ] @@ -106,7 +108,7 @@ def test_global(tmp_path): """Test global wxcode processing""" kgo_dir = acc.kgo_root() / "wxcode" kgo_path = kgo_dir / "global" / "kgo.nc" - params = [param for param in ALL_PARAMS if "lightning" not in param] + params = [param for param in ALL_PARAMS if "hail" not in param] param_paths = [ kgo_dir / "global" / f"probability_of_{p}_threshold.nc" for p in params ] From ef2007a31e21f84f12d625c9ddf663a1fe6af037 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 10 Jan 2022 10:52:42 +0000 Subject: [PATCH 10/18] Update interpret-metadata (#1632) * Fixes an unrelated typo that was staring me in the face! * Adds metadata interpreter test for probability in-vicinity over-time cube * Modifies cell-method check to exclude in-vicinity from cell methods * Removes unused import * Adds test to metadata interpreter to cover MOGREPS-G spot data on time-zones * Adds time_in_local_timezone coord to MOGREPS-G spot data on time-zones test * Updates detection of cubes on local timezones to include spot cubes. * Adds a check for certain dimensions matching the horizontal coordinates - all SPOT_COORDS - time coord for a cube on local timezones * Black and isort * Corrects type-hinting for one method * Adds test of a mode-over-time weather code cube * Interpreter accepts a mode-over-time weather code cube * Removes unnecessary if test around for loop * Black * Adds tests to show that - timezone cubes must have spatial time coords - spot cubes must have spatial spot coords * Fixes test parametrization so that the list order is always the same * Review responses - reduces duplication - improves doc-strings - moves cube set-up into fixture * Corrects doc-string --- improver/cli/__init__.py | 2 +- .../developer_tools/metadata_interpreter.py | 99 +++++++++------- improver_tests/developer_tools/conftest.py | 107 ++++++++++++++++-- .../test_MOMetadataInterpreter.py | 76 ++++++++++++- 4 files changed, 229 insertions(+), 55 deletions(-) diff --git a/improver/cli/__init__.py b/improver/cli/__init__.py index 7bbf48af2a..75f24541d6 100644 --- a/improver/cli/__init__.py +++ b/improver/cli/__init__.py @@ -483,7 +483,7 @@ def main( To write to stdout, use a hyphen (-) memprofile (str): Creates 2 files by adding a suffix to the provided arguemnt - - a tracemalloc snapsot at the point of highest memory consumption + a tracemalloc snapshot at the point of highest memory consumption of your program (suffixed with _SNAPSHOT) and a track of the maximum memory used by your program over time (suffixed with _MAX_TRACKER). diff --git a/improver/developer_tools/metadata_interpreter.py b/improver/developer_tools/metadata_interpreter.py index d0ddd738d2..0af46d267a 100644 --- a/improver/developer_tools/metadata_interpreter.py +++ b/improver/developer_tools/metadata_interpreter.py @@ -30,7 +30,7 @@ # POSSIBILITY OF SUCH DAMAGE. """Module containing classes for metadata interpretation""" -from typing import Dict, List +from typing import Callable, Dict, Iterable, List from iris.coords import CellMethod, Coord from iris.cube import Cube @@ -42,7 +42,6 @@ from improver.metadata.probabilistic import ( find_percentile_coordinate, find_threshold_coordinate, - get_diagnostic_cube_name_from_probability_name, get_threshold_coord_name_from_probability_name, ) from improver.utilities.cube_manipulation import get_coord_names @@ -109,6 +108,8 @@ "lwe_thickness_of_snowfall_amount", "thickness_of_rainfall_amount", ] +WXCODE_MODE_CM = CellMethod(method="mode", coords="time") +WXCODE_NAMES = ["weather_code"] # Compliant, required and forbidden attributes NONCOMP_ATTRS = [ @@ -181,16 +182,15 @@ def check_probability_cube_metadata(self, cube: Cube) -> None: ) try: - self.diagnostic = get_diagnostic_cube_name_from_probability_name( + self.diagnostic = get_threshold_coord_name_from_probability_name( cube.name() ) except ValueError as cause: # if the probability name is not valid self.errors.append(str(cause)) + return - expected_threshold_name = get_threshold_coord_name_from_probability_name( - cube.name() - ) + expected_threshold_name = self.diagnostic if not cube.coords(expected_threshold_name): msg = f"Cube does not have expected threshold coord '{expected_threshold_name}'; " @@ -272,31 +272,28 @@ def check_cell_methods(self, cube: Cube) -> None: if not found_cm: self.errors.append(msg) - if cube.cell_methods: - for cm in cube.cell_methods: - if cm.method in COMPLIANT_CM_METHODS: - self.methods += f" {cm.method} over {cm.coord_names[0]}" - if self.field_type == self.PROB: - if not cm.comments or cm.comments[0] != f"of {self.diagnostic}": - self.errors.append( - f"Cell method {cm} on probability data should have comment " - f"'of {self.diagnostic}'" - ) - # check point and bounds on method coordinate - if "time" in cm.coord_names: - if cube.coord("time").bounds is None: - self.errors.append( - f"Cube of{self.methods} has no time bounds" - ) - - elif cm in NONCOMP_CMS or cm.method in NONCOMP_CM_METHODS: - self.errors.append(f"Non-standard cell method {cm}") - else: - # flag method which might be invalid, but we can't be sure - self.warnings.append( - f"Unexpected cell method {cm}. Please check the standard to " - "ensure this is valid" - ) + for cm in cube.cell_methods: + if cm.method in COMPLIANT_CM_METHODS: + self.methods += f" {cm.method} over {cm.coord_names[0]}" + if self.field_type == self.PROB: + if not cm.comments or cm.comments[0] != f"of {self.diagnostic}": + self.errors.append( + f"Cell method {cm} on probability data should have comment " + f"'of {self.diagnostic}'" + ) + # check point and bounds on method coordinate + if "time" in cm.coord_names: + if cube.coord("time").bounds is None: + self.errors.append(f"Cube of{self.methods} has no time bounds") + + elif cm in NONCOMP_CMS or cm.method in NONCOMP_CM_METHODS: + self.errors.append(f"Non-standard cell method {cm}") + else: + # flag method which might be invalid, but we can't be sure + self.warnings.append( + f"Unexpected cell method {cm}. Please check the standard to " + "ensure this is valid" + ) def _check_blend_and_model_attributes(self, attrs: Dict) -> None: """Interprets attributes for model and blending information @@ -385,7 +382,7 @@ def check_attributes(self, attrs: Dict) -> None: self._check_blend_and_model_attributes(attrs) def _check_coords_present( - self, coords: List[str], expected_coords: List[str] + self, coords: List[str], expected_coords: Iterable[str] ) -> None: """Check whether all expected coordinates are present""" found_coords = [coord for coord in coords if coord in expected_coords] @@ -395,6 +392,21 @@ def _check_coords_present( f"expected {expected_coords}" ) + def _check_coords_are_horizontal(self, cube: Cube, coords: List[str]) -> None: + """Checks that all the mentioned coords share the same dimensions as the x and y coords""" + y_coord, x_coord = (cube.coord(axis=n) for n in "yx") + horizontal_dims = set([cube.coord_dims(n)[0] for n in [y_coord, x_coord]]) + for coord in coords: + try: + coord_dims = set(cube.coord_dims(coord)) + except CoordinateNotFoundError: + # The presence of coords is checked elsewhere + continue + if coord_dims != horizontal_dims: + self.errors.append( + f"Coordinate {coord} does not span all horizontal coordinates" + ) + def _check_coord_bounds(self, cube: Cube, coord: str) -> None: """If coordinate has bounds, check points are equal to upper bound""" if cube.coord(coord).bounds is not None: @@ -413,6 +425,7 @@ def check_spot_data(self, cube: Cube, coords: List[str]) -> None: ) self._check_coords_present(coords, SPOT_COORDS) + self._check_coords_are_horizontal(cube, SPOT_COORDS) def run(self, cube: Cube) -> None: """Populates self-consistent interpreted parameters, or raises collated errors @@ -435,8 +448,13 @@ def run(self, cube: Cube) -> None: elif cube.name() in SPECIAL_CASES: self.field_type = self.diagnostic = cube.name() if cube.name() == "weather_code": - if cube.cell_methods: - self.errors.append(f"Unexpected cell methods {cube.cell_methods}") + for cm in cube.cell_methods: + if cm == WXCODE_MODE_CM and cube.name() in WXCODE_NAMES: + pass + else: + self.errors.append( + f"Unexpected cell methods {cube.cell_methods}" + ) elif cube.name() == "wind_from_direction": if cube.cell_methods: expected = CellMethod(method="mean", coords="realization") @@ -492,16 +510,13 @@ def run(self, cube: Cube) -> None: if self.field_type == self.ANCIL: # there is no definitive standard for time coordinates on static ancillaries pass - elif ( - cube.coords("time") - and len(cube.coord_dims("time")) == 2 - and not self.blended - ): - # 2D time coordinates are only present on global day-max diagnostics that - # use a local time zone coordinate. These do not have a 2D forecast period. + elif cube.coords("time_in_local_timezone"): + # For data on local timezones, the time coordinate will match the horizontal + # dimensions and there will be no forecast period. expected_coords = set(LOCAL_TIME_COORDS + UNBLENDED_TIME_COORDS) expected_coords.discard("forecast_period") self._check_coords_present(coords, expected_coords) + self._check_coords_are_horizontal(cube, ["time"]) elif self.blended: self._check_coords_present(coords, BLENDED_TIME_COORDS) else: @@ -538,7 +553,7 @@ def run(self, cube: Cube) -> None: def _format_standard_cases( - interpreter: MOMetadataInterpreter, verbose: bool, vstring: str + interpreter: MOMetadataInterpreter, verbose: bool, vstring: Callable[[str], str] ) -> List[str]: """Format prob / perc / diagnostic information from a MOMetadataInterpreter instance""" diff --git a/improver_tests/developer_tools/conftest.py b/improver_tests/developer_tools/conftest.py index c1c64f0b62..c562c4e530 100644 --- a/improver_tests/developer_tools/conftest.py +++ b/improver_tests/developer_tools/conftest.py @@ -32,11 +32,13 @@ from datetime import datetime +import cf_units import iris import numpy as np import pytest from improver.developer_tools.metadata_interpreter import MOMetadataInterpreter +from improver.metadata.constants.time_types import TIME_COORDS from improver.spotdata.build_spotdata_cube import build_spotdata_cube from improver.synthetic_data.set_up_test_cubes import ( construct_scalar_time_coords, @@ -169,6 +171,38 @@ def probability_above_fixture(): ) +@pytest.fixture(name="probability_over_time_in_vicinity_above_cube") +def probability_over_time_in_vicinity_above_fixture(): + """Probability of precipitation accumulation in 15M in vicinity above threshold cube from UKV""" + data = 0.5 * np.ones((3, 3, 3), dtype=np.float32) + thresholds = np.array([280, 282, 284], dtype=np.float32) + attributes = { + "source": "Met Office Unified Model", + "title": "Post-Processed UKV Model Forecast on 2 km Standard Grid", + "institution": "Met Office", + "mosg__model_configuration": "uk_det", + } + diagnostic_name = "lwe_thickness_of_precipitation_amount" + cube = set_up_probability_cube( + data, + thresholds, + attributes=attributes, + spatial_grid="equalarea", + variable_name=f"{diagnostic_name}_in_vicinity", + ) + cube.add_cell_method( + iris.coords.CellMethod( + method="sum", coords="time", comments=(f"of {diagnostic_name}",), + ) + ) + for coord in ["time", "forecast_period"]: + cube.coord(coord).bounds = np.array( + [cube.coord(coord).points[0] - 900, cube.coord(coord).points[0]], + dtype=cube.coord(coord).dtype, + ) + return cube + + @pytest.fixture(name="blended_probability_below_cube") def probability_below_fixture(): """Probability of maximum screen temperature below threshold blended cube""" @@ -215,14 +249,13 @@ def snow_level_fixture(): ) -@pytest.fixture(name="blended_spot_median_cube") +@pytest.fixture(name="spot_template") def spot_fixture(): - """Spot temperature cube""" alts = np.array([15, 82, 0, 4, 15, 269], dtype=np.float32) lats = np.array([60.75, 60.13, 58.95, 57.37, 58.22, 57.72], dtype=np.float32) lons = np.array([-0.85, -1.18, -2.9, -7.40, -6.32, -4.90], dtype=np.float32) - wmo_ids = np.array(["3002", "3005", "3017", "3023", "3026", "3031"]) - spot_cube = build_spotdata_cube( + wmo_ids = ["3002", "3005", "3017", "3023", "3026", "3031"] + cube = build_spotdata_cube( np.arange(6).astype(np.float32), "air_temperature", "degC", @@ -231,10 +264,15 @@ def spot_fixture(): lons, wmo_ids, ) - spot_cube.add_aux_coord( - iris.coords.AuxCoord([50], long_name="percentile", units="%") - ) - spot_cube.attributes = { + cube.add_aux_coord(iris.coords.AuxCoord([50], long_name="percentile", units="%")) + return cube + + +@pytest.fixture(name="blended_spot_median_cube") +def blended_spot_median_spot_fixture(spot_template): + """Spot temperature cube from blend""" + cube = spot_template.copy() + cube.attributes = { "source": "IMPROVER", "institution": "Met Office", "title": "IMPROVER Post-Processed Multi-Model Blend UK Spot Values", @@ -244,9 +282,47 @@ def spot_fixture(): time=datetime(2021, 2, 3, 14), time_bounds=None, frt=datetime(2021, 2, 3, 10) ) blend_time.rename("blend_time") - spot_cube.add_aux_coord(time) - spot_cube.add_aux_coord(blend_time) - return spot_cube + cube.add_aux_coord(time) + cube.add_aux_coord(blend_time) + return cube + + +@pytest.fixture(name="blended_spot_timezone_cube") +def spot_timezone_fixture(spot_template): + """Spot data on local time-zones + (no forecast_period, forecast_reference_time matches spatial dimension)""" + cube = spot_template.copy() + cube.attributes = { + "source": "Met Office Unified Model", + "institution": "Met Office", + "title": "Post-Processed MOGREPS-G Model Forecast Global Spot Values", + "mosg__model_configuration": "gl_ens", + } + (time_source_coord, _), (frt_coord, _), (_, _) = construct_scalar_time_coords( + time=datetime(2021, 2, 3, 14), time_bounds=None, frt=datetime(2021, 2, 3, 10) + ) + cube.add_aux_coord(frt_coord) + (spatial_index,) = cube.coord_dims("latitude") + time_coord = iris.coords.AuxCoord( + np.full(cube.shape, fill_value=time_source_coord.points), + standard_name=time_source_coord.standard_name, + units=time_source_coord.units, + ) + cube.add_aux_coord(time_coord, spatial_index) + local_time_coord_standards = TIME_COORDS["time_in_local_timezone"] + local_time_units = cf_units.Unit( + local_time_coord_standards.units, calendar=local_time_coord_standards.calendar, + ) + timezone_points = np.array( + np.round(local_time_units.date2num(datetime(2021, 2, 3, 15))), + dtype=local_time_coord_standards.dtype, + ) + cube.add_aux_coord( + iris.coords.AuxCoord( + timezone_points, long_name="time_in_local_timezone", units=local_time_units, + ) + ) + return cube @pytest.fixture(name="wind_direction_cube") @@ -287,6 +363,15 @@ def wxcode_fixture(): units="1", attributes=attributes, spatial_grid="equalarea", + time_bounds=(datetime(2017, 11, 10, 3, 0), datetime(2017, 11, 10, 4, 0)), ) _update_blended_time_coords(cube) return cube + + +@pytest.fixture(name="wxcode_mode_cube") +def wxcode_mode_fixture(wxcode_cube): + """Weather symbols cube representing mode over time""" + cube = wxcode_cube.copy() + cube.add_cell_method(iris.coords.CellMethod("mode", coords="time")) + return cube diff --git a/improver_tests/developer_tools/test_MOMetadataInterpreter.py b/improver_tests/developer_tools/test_MOMetadataInterpreter.py index 90cd3325e0..f1db41fba4 100644 --- a/improver_tests/developer_tools/test_MOMetadataInterpreter.py +++ b/improver_tests/developer_tools/test_MOMetadataInterpreter.py @@ -32,9 +32,10 @@ import numpy as np import pytest -from iris.coords import CellMethod +from iris.coords import AuxCoord, CellMethod # Test successful outputs (input cubes in alphabetical order by fixture) +from improver.developer_tools.metadata_interpreter import SPOT_COORDS def test_realizations(ensemble_cube, interpreter): @@ -127,6 +128,15 @@ def test_handles_duplicate_model_string(probability_above_cube, interpreter): assert interpreter.model == "UKV" +def test_vicinity_cell_method( + probability_over_time_in_vicinity_above_cube, interpreter +): + """Test when precipitation accumulation in-vicinity cube has a cell method""" + cube = probability_over_time_in_vicinity_above_cube.copy() + interpreter.run(cube) + assert "sum over time" in interpreter.methods + + def test_probabilities_below(blended_probability_below_cube, interpreter): """Test interpretation of blended probability of max temperature in hour below threshold""" @@ -173,6 +183,20 @@ def test_spot_median(blended_spot_median_cube, interpreter): assert not interpreter.warnings +def test_spot_timezone(blended_spot_timezone_cube, interpreter): + """Test interpretation of spot on timezones""" + interpreter.run(blended_spot_timezone_cube) + assert interpreter.prod_type == "spot" + assert interpreter.field_type == "percentiles" + assert interpreter.diagnostic == "air_temperature" + assert interpreter.relative_to_threshold is None + assert not interpreter.methods + assert interpreter.post_processed + assert interpreter.model == "MOGREPS-G" + assert not interpreter.blended + assert not interpreter.warnings + + def test_wind_direction(wind_direction_cube, interpreter): """Test interpretation of wind direction field with mean over realizations cell method""" @@ -190,6 +214,14 @@ def test_weather_code(wxcode_cube, interpreter): assert interpreter.blended +def test_weather_mode_code(wxcode_mode_cube, interpreter): + """Test interpretation of weather code mode-in-time field""" + interpreter.run(wxcode_mode_cube) + assert interpreter.diagnostic == "weather_code" + assert interpreter.model == "UKV, MOGREPS-UK" + assert interpreter.blended + + # Test errors and warnings (input cubes in alphabetical order by fixture) @@ -398,6 +430,24 @@ def test_error_time_coord_units(probability_above_cube, interpreter): interpreter.run(probability_above_cube) +def test_error_timezone_has_scalar_time(blended_spot_timezone_cube, interpreter): + """Test error raised if a timezones cube has a scalar time coord""" + cube = blended_spot_timezone_cube.copy() + time_coord = cube.coord("time").copy() + cube.remove_coord("time") + cube.add_aux_coord( + AuxCoord( + time_coord.points[0], + standard_name=time_coord.standard_name, + units=time_coord.units, + ) + ) + with pytest.raises( + ValueError, match="Coordinate time does not span all horizontal coordinates" + ): + interpreter.run(cube) + + # Test the interpreter can return multiple errors. @@ -509,6 +559,30 @@ def test_error_missing_spot_coords(blended_spot_median_cube, interpreter): interpreter.run(blended_spot_median_cube) +@pytest.mark.parametrize( + "coord_name", [x for x in SPOT_COORDS if x not in ["latitude", "longitude"]] +) +def test_error_inconsistent_spot_coords( + blended_spot_median_cube, interpreter, coord_name +): + """Test error raised if a spot cube coord ought to apply to the x/y dim, but doesn't""" + coord = blended_spot_median_cube.coord(coord_name).copy() + blended_spot_median_cube.remove_coord(coord_name) + blended_spot_median_cube.add_aux_coord( + AuxCoord( + coord.points[0], + standard_name=coord.standard_name, + long_name=coord.long_name, + units=coord.units, + ) + ) + with pytest.raises( + ValueError, + match=f"Coordinate {coord_name} does not span all horizontal coordinates", + ): + interpreter.run(blended_spot_median_cube) + + def test_error_inconsistent_spot_title(blended_spot_median_cube, interpreter): """Test error raised if a spot cube has a non-spot title""" blended_spot_median_cube.attributes[ From 23d1decbe65a34bbd488f359152bfdfc3d4207ef Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 10 Jan 2022 14:20:16 +0000 Subject: [PATCH 11/18] Updates checksums for threshold landmask fix (#1636) --- improver_tests/acceptance/SHA256SUMS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index 59f9f49cf8..9e7f925934 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -532,7 +532,7 @@ f65f8d857751db8c183cf85ba09630ec1245e7a68af4993fc32c386072863647 ./threshold/vi d9679e8107f903c017a69ad614b5c9dfc05baedbe4ff3784b99c0dffa9a827ab ./threshold/vicinity/kgo_landmask.nc d03f83bb3a3c1a2f59c3032f3ae57e1a282dff4eeec09f020b02f72143c5147d ./threshold/vicinity/kgo_landmask_collapsed.nc 9b8ae27da1952e335acbe5b60b9bfeb13363a6046b0a0200538ea385c1c0f8ca ./threshold/vicinity/kgo_masked.nc -7b731c6d447279a50aba9a475fd4f3151768c9a5092287a6253ae538da030428 ./threshold/vicinity/landmask.nc +07eecac9f1a5888ee5162a03f4640d2713b92be971dcac8ff8f0ac32a78ee7f7 ./threshold/vicinity/landmask.nc 1180190519e4d77d2e7709c03491a209fc542366ebccbecb94ffae25b1653465 ./threshold/vicinity/masked_precip.nc 31ae5d858565ea0926a563b7b9974d82c35d134772110dab49accc93072330e0 ./time-lagged-ens/mixed_validity/20180924T1300Z-PT0001H00M-temperature_at_surface.nc 2fe8830d453d3b2a6972e9993b77278585a29f19be7df0ec7113444d1e3d14da ./time-lagged-ens/mixed_validity/20180924T1900Z-PT0006H00M-temperature_at_surface.nc From 4f295d8f01979b301f25d51ea94d6467e09cb87c Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Wed, 12 Jan 2022 12:53:48 +0000 Subject: [PATCH 12/18] Fixes bug in spot-extraction for multi-time inputs (#1633) * Fixes bug in spot-extraction so that auxiliary non-spatial coords (e.g. forecast_period) are copied onto the output cube * Moves functionality for detecting AuxCoords associated with additional_dims into build_diagnostic_cube --- improver/spotdata/build_spotdata_cube.py | 10 +++++- improver/spotdata/spot_extraction.py | 17 +++++++-- improver_tests/acceptance/SHA256SUMS | 2 +- .../spotdata/test_build_spotdata_cube.py | 35 +++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/improver/spotdata/build_spotdata_cube.py b/improver/spotdata/build_spotdata_cube.py index 223f68f12e..5ae19c533d 100644 --- a/improver/spotdata/build_spotdata_cube.py +++ b/improver/spotdata/build_spotdata_cube.py @@ -56,6 +56,7 @@ def build_spotdata_cube( neighbour_methods: Optional[List[str]] = None, grid_attributes: Optional[List[str]] = None, additional_dims: Optional[List[Coord]] = None, + additional_dims_aux: Optional[List[List[AuxCoord]]] = None, ) -> Cube: """ Function to build a spotdata cube with expected dimension and auxiliary @@ -108,6 +109,9 @@ def build_spotdata_cube( Optional list of grid attribute names, e.g. x-index, y-index additional_dims: Optional list of additional dimensions to preceed the spot data dimension. + additional_dims_aux: + Optional list of auxiliary coordinates associated with each dimension in + additional_dims Returns: A cube containing the extracted spot data with spot data being the final dimension. @@ -179,8 +183,12 @@ def build_spotdata_cube( current_dim += 1 if additional_dims is not None: - for coord in additional_dims: + for coord, aux_coords in zip( + additional_dims, additional_dims_aux or [[] for _ in additional_dims] + ): dim_coords_and_dims.append((coord, current_dim)) + for aux_coord in aux_coords: + aux_coords_and_dims.append((aux_coord, current_dim)) current_dim += 1 dim_coords_and_dims.append((spot_index, current_dim)) diff --git a/improver/spotdata/spot_extraction.py b/improver/spotdata/spot_extraction.py index 8bb6868041..5f039c5e62 100644 --- a/improver/spotdata/spot_extraction.py +++ b/improver/spotdata/spot_extraction.py @@ -207,7 +207,7 @@ def build_diagnostic_cube( neighbour_cube: Cube, diagnostic_cube: Cube, spot_values: ndarray, - additional_dims: Optional[List[DimCoord]] = None, + additional_dims: Optional[List[DimCoord]] = [], scalar_coords: Optional[List[AuxCoord]] = None, auxiliary_coords: Optional[List[AuxCoord]] = None, unique_site_id: Optional[Union[List[str], ndarray]] = None, @@ -243,6 +243,17 @@ def build_diagnostic_cube( Returns: A spot data cube containing the extracted diagnostic data. """ + # Find any AuxCoords associated with the additional_dims so these can be copied too + additional_dims_aux = [] + for dim_coord in additional_dims: + dim_coord_dim = diagnostic_cube.coord_dims(dim_coord) + aux_coords = [ + aux_coord + for aux_coord in diagnostic_cube.aux_coords + if diagnostic_cube.coord_dims(aux_coord) == dim_coord_dim + ] + additional_dims_aux.append(aux_coords if aux_coords else []) + spot_diagnostic_cube = build_spotdata_cube( spot_values, diagnostic_cube.name(), @@ -256,6 +267,7 @@ def build_diagnostic_cube( scalar_coords=scalar_coords, auxiliary_coords=auxiliary_coords, additional_dims=additional_dims, + additional_dims_aux=additional_dims_aux, ) return spot_diagnostic_cube @@ -314,10 +326,9 @@ def process( x_indices, y_indices = coordinate_cube.data spot_values = diagnostic_cube.data[..., y_indices, x_indices] - additional_dims = None + additional_dims = [] if len(spot_values.shape) > 1: additional_dims = diagnostic_cube.dim_coords[:-2] - scalar_coords, nonscalar_coords = self.get_aux_coords( diagnostic_cube, x_indices, y_indices ) diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index 9e7f925934..5c096b581c 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -479,7 +479,7 @@ d79d5449451570f7e7f479b5ecf1e5524fbf2f6d5107550c0372dd744448c7a8 ./spot-extract 5904853a7ff3ccfe09fdcdfd79ed4edd5f1f5af12d3c6f9511a73bf0bb825068 ./spot-extract/outputs/lapse_rate_adjusted_uk_temperatures.nc 081a821471206a864d6723bedda6eb066af2514068160114eed16a156810d316 ./spot-extract/outputs/mindz_land_constraint_uk_temperatures.nc eff5c671c33c3520e9e7f0bcab4fe2a4eb5557fd84767da1a8c494f3ab7f3aa6 ./spot-extract/outputs/mindz_uk_temperatures.nc -15d7221f43a11a8bac467a03a6bf7bac1b5d713d6583767fcc9f11687c112300 ./spot-extract/outputs/multi_time_kgo.nc +99af8bc4a2d6dab2da309326aed3d762f72d945fe9e864c5056369f2747cd85b ./spot-extract/outputs/multi_time_kgo.nc 522f5331735a778f65ad5ed6ca5ebca03bdcb8d4c7a783d6f7afe61b61efa0ab ./spot-extract/outputs/nearest_uk_temperatures.nc 522f5331735a778f65ad5ed6ca5ebca03bdcb8d4c7a783d6f7afe61b61efa0ab ./spot-extract/outputs/nearest_uk_temperatures_amended_metadata.nc 7f7734eb79abe416b0587cff2fbff286854e978037be3d63054c93d67b8bdc39 ./spot-extract/outputs/nearest_uk_temperatures_unique_ids.nc diff --git a/improver_tests/spotdata/test_build_spotdata_cube.py b/improver_tests/spotdata/test_build_spotdata_cube.py index 833101fb77..9ceb9083df 100755 --- a/improver_tests/spotdata/test_build_spotdata_cube.py +++ b/improver_tests/spotdata/test_build_spotdata_cube.py @@ -35,10 +35,14 @@ import iris import numpy as np +from cf_units import Unit +from iris.coords import AuxCoord, DimCoord from iris.tests import IrisTest +from improver.metadata.constants.time_types import TIME_COORDS from improver.spotdata.build_spotdata_cube import build_spotdata_cube from improver.synthetic_data.set_up_test_cubes import construct_scalar_time_coords +from improver.utilities.round import round_close class Test_build_spotdata_cube(IrisTest): @@ -190,6 +194,37 @@ def test_3d_spot_cube_with_unequal_length_coordinates(self): grid_attributes=self.grid_attributes, ) + def test_3d_spot_cube_for_time(self): + """Test output with two extra dimensions, one of which is time with + forecast_period as an auxiliary coordinate""" + data = np.ones((3, 2, 4), dtype=np.float32) + time_spec = TIME_COORDS["time"] + time_units = Unit(time_spec.units) + time_as_dt = [datetime(2021, 12, 25, 12, 0), datetime(2021, 12, 25, 12, 1)] + time_points = round_close( + np.array([time_units.date2num(t) for t in time_as_dt]), + dtype=time_spec.dtype, + ) + time_coord = DimCoord(time_points, units=time_units, standard_name="time") + + fp_spec = TIME_COORDS["forecast_period"] + fp_units = Unit(fp_spec.units) + fp_points = np.array([0, 3600], dtype=fp_spec.dtype) + fp_coord = AuxCoord(fp_points, units=fp_units, standard_name="forecast_period") + + result = build_spotdata_cube( + data, + *self.args, + grid_attributes=self.grid_attributes, + additional_dims=[time_coord], + additional_dims_aux=[[fp_coord]], + ) + + self.assertArrayAlmostEqual(result.data, data) + self.assertEqual(result.coord_dims("grid_attributes")[0], 0) + self.assertEqual(result.coord_dims("time")[0], 1) + self.assertEqual(result.coord_dims("forecast_period")[0], 1) + def test_scalar_coords(self): """Test additional scalar coordinates""" [(time_coord, _), (frt_coord, _), (fp_coord, _)] = construct_scalar_time_coords( From 53350936688d42335445d6930bfc5ee0c4cb0e0f Mon Sep 17 00:00:00 2001 From: gavinevans Date: Wed, 12 Jan 2022 15:19:12 +0000 Subject: [PATCH 13/18] Update checksums after updating the title of files in apply-emos-coefficients/sites. (#1640) --- improver_tests/acceptance/SHA256SUMS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index 5c096b581c..94331ed780 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -19,13 +19,13 @@ bd56f09033f7a37323e04be7b509e8e363eb2474c04fb8aa4fb5e3769acb81f7 ./apply-emos-c 615ddb59a838024bb055b9cbbb44b7c18ce28d5734952777774980760e0f36fc ./apply-emos-coefficients/rebadged_percentiles/input.nc 9b48f3eabeed93a90654a5e6a2f06fcb7297cdba40f650551d99a5e310cf27dd ./apply-emos-coefficients/sites/additional_predictor/altitude.nc 84fc5fbdc782ecff9262f26232a971b327e3adbf3f0b122d8c3d4b147fec172b ./apply-emos-coefficients/sites/additional_predictor/coefficients.nc -e610e4329ef038a5f91e190a3312c24b3e742000a35068268a8c17b9bc83944e ./apply-emos-coefficients/sites/additional_predictor/percentile_kgo.nc -60edea7a92af49d6afd10c687fd3f780caa8cd6f8a3bc29d33d5b3e618c949d8 ./apply-emos-coefficients/sites/additional_predictor/probability_kgo.nc -a3bfead4d08e7ec2ac66067cd16d041b1aabe32eb242f3e072c67efd1d10d8f2 ./apply-emos-coefficients/sites/additional_predictor/probability_template.nc +9e4782449a3236fbbe921c70b52e71fb3650743c1e45b70acc74ce50de718254 ./apply-emos-coefficients/sites/additional_predictor/percentile_kgo.nc +6fc149bc393ff53847c936788b6157455a5676dec30ca2c7b52a915c091329b1 ./apply-emos-coefficients/sites/additional_predictor/probability_kgo.nc +e7e5aa3ac0848a94d536e04d32c6a82cbc64bdd062a7bf0dc76cd9d0caef8030 ./apply-emos-coefficients/sites/additional_predictor/probability_template.nc a0112b5a48ba99a1a4a345d43b0c453caaf25181504fb9a13786b5722f84cc10 ./apply-emos-coefficients/sites/offset/coefficients.nc 22d950494dc9d76d642b0ecc75414e95bfb78fa693e5701fe7cd2fc462a0c6a7 ./apply-emos-coefficients/sites/offset/kgo.nc 2722b1d08a87cebf1f36ac914d5badb9af38859b2150da83311dc202ce9be4dd ./apply-emos-coefficients/sites/offset/offset_input.nc -3c496867a4a77a4627426b79d8d963a83590b5440984cc2d7a31085cc0efbe10 ./apply-emos-coefficients/sites/percentile_input.nc +d14ddcda89123ee0200a4921f59d47d8499a1e65adb865e8522a40e71db5ec9d ./apply-emos-coefficients/sites/percentile_input.nc 63291b3435066f0b0fc56bc78b9774e0e7457d5a6b20086a67ecc6eb826d618d ./apply-emos-coefficients/sites/point_by_point/coefficients.nc 833a703e14a02999a7d70e9a37ce199ab36a09b13b96f87295a0fb91739658ce ./apply-emos-coefficients/sites/point_by_point/kgo.nc 0bc91af1e7003d696aa440a07d79e653bca566733cdf9bf525ca92cff821548f ./apply-emos-coefficients/sites/realization_input.nc From 86ab975ea58637d45fb41ae81d10c7928325bc87 Mon Sep 17 00:00:00 2001 From: bayliffe Date: Thu, 13 Jan 2022 16:04:42 +0000 Subject: [PATCH 14/18] Make flake8 report that it is okay when running improver-tests. (#1645) --- bin/improver-tests | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/improver-tests b/bin/improver-tests index 39ee77cea1..94cc0b916d 100755 --- a/bin/improver-tests +++ b/bin/improver-tests @@ -68,6 +68,7 @@ function improver_test_isort { function improver_test_flake8 { ${FLAKE8:-flake8} $FILES_TO_TEST + echo_ok "flake8" } function improver_test_doc { From f3e612baa1db00630e80334cc0abe2280664d3f9 Mon Sep 17 00:00:00 2001 From: bayliffe Date: Thu, 13 Jan 2022 16:33:34 +0000 Subject: [PATCH 15/18] IM-1621: Make ECC error and warning tests more rigorous (#1641) * Make ECC error and warning tests more rigorous. * Black. --- .../test_ConvertProbabilitiesToPercentiles.py | 16 ++++++++++++-- .../test_ResamplePercentiles.py | 22 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/improver_tests/ensemble_copula_coupling/test_ConvertProbabilitiesToPercentiles.py b/improver_tests/ensemble_copula_coupling/test_ConvertProbabilitiesToPercentiles.py index 17fc7bb73d..c608d886ae 100644 --- a/improver_tests/ensemble_copula_coupling/test_ConvertProbabilitiesToPercentiles.py +++ b/improver_tests/ensemble_copula_coupling/test_ConvertProbabilitiesToPercentiles.py @@ -114,7 +114,12 @@ def test_endpoints_of_distribution_exceeded(self): """ probabilities_for_cdf = np.array([[0.05, 0.7, 0.95]]) threshold_points = np.array([8, 10, 60]) - msg = "The calculated threshold values" + msg = ( + "The calculated threshold values \\[-40 8 10 60 50\\] are " + "not in ascending order as required for the cumulative distribution " + "function \\(CDF\\). This is due to the threshold values exceeding " + "the range given by the ECC bounds \\(-40, 50\\)." + ) with self.assertRaisesRegex(ValueError, msg): Plugin()._add_bounds_to_thresholds_and_probabilities( threshold_points, probabilities_for_cdf, self.bounds_pairing @@ -131,7 +136,14 @@ def test_endpoints_of_distribution_exceeded_warning(self, warning_list=None): probabilities_for_cdf = np.array([[0.05, 0.7, 0.95]]) threshold_points = np.array([8, 10, 60]) plugin = Plugin(ecc_bounds_warning=True) - warning_msg = "The calculated threshold values" + warning_msg = ( + "The calculated threshold values [-40 8 10 60 50] are " + "not in ascending order as required for the cumulative distribution " + "function (CDF). This is due to the threshold values exceeding " + "the range given by the ECC bounds (-40, 50). The threshold " + "points that have exceeded the existing bounds will be used as " + "new bounds." + ) plugin._add_bounds_to_thresholds_and_probabilities( threshold_points, probabilities_for_cdf, self.bounds_pairing ) diff --git a/improver_tests/ensemble_copula_coupling/test_ResamplePercentiles.py b/improver_tests/ensemble_copula_coupling/test_ResamplePercentiles.py index 694465f801..59394da95b 100644 --- a/improver_tests/ensemble_copula_coupling/test_ResamplePercentiles.py +++ b/improver_tests/ensemble_copula_coupling/test_ResamplePercentiles.py @@ -117,7 +117,16 @@ def test_endpoints_of_distribution_exceeded(self): """ forecast_at_percentiles = np.array([[8, 10, 60]]) percentiles = np.array([5, 70, 95]) - msg = "Forecast values exist that fall outside the expected extrema" + + msg = ( + "Forecast values exist that fall outside the expected extrema " + "values that are defined as bounds in ensemble_copula_coupling" + "\\/constants.py. Applying the extrema values as end points to " + "the distribution would result in non-monotonically increasing " + "values. The defined extremes are \\(-40, 50\\), whilst the " + "following forecast values exist outside this range: \\[60\\]." + ) + with self.assertRaisesRegex(ValueError, msg): Plugin()._add_bounds_to_percentiles_and_forecast_at_percentiles( percentiles, forecast_at_percentiles, self.bounds_pairing @@ -134,7 +143,16 @@ def test_endpoints_of_distribution_exceeded_warning(self, warning_list=None): forecast_at_percentiles = np.array([[8, 10, 60]]) percentiles = np.array([5, 70, 95]) plugin = Plugin(ecc_bounds_warning=True) - warning_msg = "Forecast values exist that fall outside the expected extrema" + warning_msg = ( + "Forecast values exist that fall outside the expected extrema " + "values that are defined as bounds in ensemble_copula_coupling" + "/constants.py. Applying the extrema values as end points to " + "the distribution would result in non-monotonically increasing " + "values. The defined extremes are (-40, 50), whilst the " + "following forecast values exist outside this range: [60]. " + "The percentile values that have exceeded the existing bounds " + "will be used as new bounds." + ) plugin._add_bounds_to_percentiles_and_forecast_at_percentiles( percentiles, forecast_at_percentiles, self.bounds_pairing ) From 73712803cb53da6aa8758d27b104fe7d9749063d Mon Sep 17 00:00:00 2001 From: bayliffe Date: Fri, 14 Jan 2022 08:26:39 +0000 Subject: [PATCH 16/18] MOBT-180: Weather symbol speed up (#1638) * Speed up step. * Test update. * Review changes. * Typehint change. --- improver/wxcode/weather_symbols.py | 24 ++++--- .../wxcode/wxcode/test_WeatherSymbols.py | 65 +++++++++++++++---- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/improver/wxcode/weather_symbols.py b/improver/wxcode/weather_symbols.py index e71552020d..54a18cdcc6 100644 --- a/improver/wxcode/weather_symbols.py +++ b/improver/wxcode/weather_symbols.py @@ -135,22 +135,26 @@ def __repr__(self) -> str: """Represent the configured plugin instance as a string.""" return "".format(self.start_node) - def check_input_cubes(self, cubes: CubeList) -> Optional[List[str]]: + def prepare_input_cubes( + self, cubes: CubeList + ) -> Tuple[CubeList, Optional[List[str]]]: """ Check that the input cubes contain all the diagnostics and thresholds required by the decision tree. Sets self.coord_named_threshold to "True" if threshold-type coordinates have the name "threshold" (as opposed to the standard name of the diagnostic), for backward - compatibility. + compatibility. A cubelist containing only cubes of the required + diagnostic-threshold combinations is returned. Args: cubes: A CubeList containing the input diagnostic cubes. Returns: - A list of node names where the diagnostic data is missing and - this is indicated as allowed by the presence of the if_diagnostic_missing - key. + - A CubeList containing only the required cubes. + - A list of node names where the diagnostic data is missing and + this is indicated as allowed by the presence of the if_diagnostic_missing + key. Raises: IOError: @@ -160,6 +164,7 @@ def check_input_cubes(self, cubes: CubeList) -> Optional[List[str]]: # Check that all cubes are valid at or over the same periods self.check_coincidence(cubes) + used_cubes = iris.cube.CubeList() optional_node_data_missing = [] missing_data = [] for key, query in self.queries.items(): @@ -220,6 +225,8 @@ def check_input_cubes(self, cubes: CubeList) -> Optional[List[str]]: matched_threshold = matched_cube.extract(test_condition) if not matched_threshold: missing_data.append([diagnostic, threshold, condition]) + else: + used_cubes.extend(matched_threshold) if missing_data: msg = ( @@ -234,7 +241,7 @@ def check_input_cubes(self, cubes: CubeList) -> Optional[List[str]]: if not optional_node_data_missing: optional_node_data_missing = None - return optional_node_data_missing + return used_cubes, optional_node_data_missing def check_coincidence(self, cubes: Union[List[Cube], CubeList]) -> Cube: """ @@ -710,8 +717,9 @@ def process(self, cubes: CubeList) -> Cube: Returns: A cube of weather symbols. """ - # Check input cubes contain required data - optional_node_data_missing = self.check_input_cubes(cubes) + # Check input cubes contain required data and return only those that + # are needed to speed up later cube extractions. + cubes, optional_node_data_missing = self.prepare_input_cubes(cubes) # Reroute the decision tree around missing optional nodes if optional_node_data_missing is not None: diff --git a/improver_tests/wxcode/wxcode/test_WeatherSymbols.py b/improver_tests/wxcode/wxcode/test_WeatherSymbols.py index 8560e71454..24d535dec1 100644 --- a/improver_tests/wxcode/wxcode/test_WeatherSymbols.py +++ b/improver_tests/wxcode/wxcode/test_WeatherSymbols.py @@ -40,7 +40,10 @@ from iris.coords import AuxCoord from iris.tests import IrisTest -from improver.metadata.probabilistic import find_threshold_coordinate +from improver.metadata.probabilistic import ( + find_threshold_coordinate, + get_threshold_coord_name_from_probability_name, +) from improver.synthetic_data.set_up_test_cubes import set_up_probability_cube from improver.wxcode.utilities import WX_DICT from improver.wxcode.weather_symbols import WeatherSymbols @@ -286,46 +289,82 @@ def test_basic(self): self.assertEqual(result, msg) -class Test_check_input_cubes(Test_WXCode): +class Test_prepare_input_cubes(Test_WXCode): - """Test the check_input_cubes method.""" + """Test the prepare_input_cubes method.""" def test_basic(self): - """Test check_input_cubes method raises no error if the data is OK""" + """Test prepare_input_cubes method raises no error if the data is OK""" plugin = WeatherSymbols(wxtree=wxcode_decision_tree()) - self.assertEqual(plugin.check_input_cubes(self.cubes), None) + plugin.prepare_input_cubes(self.cubes) def test_no_lightning(self): - """Test check_input_cubes raises no error if lightning missing""" + """Test prepare_input_cubes raises no error if lightning missing""" cubes = self.cubes.extract(self.missing_diagnostic) - result = self.plugin.check_input_cubes(cubes) + _, result = self.plugin.prepare_input_cubes(cubes) self.assertIsInstance(result, list) self.assertEqual(len(result), 1) self.assertTrue("lightning" in result) def test_raises_error_missing_cubes(self): - """Test check_input_cubes method raises error if data is missing""" + """Test prepare_input_cubes method raises error if data is missing""" cubes = self.cubes[0:2] msg = "Weather Symbols input cubes are missing" with self.assertRaisesRegex(IOError, msg): - self.plugin.check_input_cubes(cubes) + self.plugin.prepare_input_cubes(cubes) def test_raises_error_missing_threshold(self): - """Test check_input_cubes method raises error if data is missing""" + """Test prepare_input_cubes method raises error if data is missing""" cubes = self.cubes cubes[0] = cubes[0][0] msg = "Weather Symbols input cubes are missing" with self.assertRaisesRegex(IOError, msg): - self.plugin.check_input_cubes(cubes) + self.plugin.prepare_input_cubes(cubes) def test_incorrect_units(self): - """Test that check_input_cubes method raises an error if the units are + """Test that prepare_input_cubes method raises an error if the units are incompatible between the input cube and the decision tree.""" msg = "Unable to convert from" threshold_coord = find_threshold_coordinate(self.cubes[0]) self.cubes[0].coord(threshold_coord).units = Unit("mm kg-1") with self.assertRaisesRegex(ValueError, msg): - self.plugin.check_input_cubes(self.cubes) + self.plugin.prepare_input_cubes(self.cubes) + + def test_returns_used_cubes(self): + """Test that prepare_input_cubes method returns a list of cubes that is + reduced to include only those diagnostics and thresholds that are used + in the decision tree. Rain, sleet and snow all have a redundant + threshold in the input cubes. The test below ensures that for all other + diagnostics all thresholds are returned, but for rain, sleet and snow + the extra threshold is omitted.""" + + expected = [] + unexpected = [] + for cube in self.cubes: + threshold_name = get_threshold_coord_name_from_probability_name(cube.name()) + threshold_values = cube.coord(threshold_name).points + if ( + "rain" in threshold_name + or "sleet" in threshold_name + or "snow" in threshold_name + ): + unexpected.append( + iris.Constraint( + coord_values={ + threshold_name: lambda cell: 2.7e-08 < cell < 2.8e-08 + } + ) + ) + threshold_values = threshold_values[0::2] + for value in threshold_values: + expected.append(iris.Constraint(coord_values={threshold_name: value})) + + result, _ = self.plugin.prepare_input_cubes(self.cubes) + + for constraint in expected: + self.assertTrue(len(result.extract(constraint)) > 0) + for constraint in unexpected: + self.assertEqual(len(result.extract(constraint)), 0) class Test_invert_condition(IrisTest): From 671b39ced85be9b2c17178f88e2de7c4bf20d199 Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Mon, 17 Jan 2022 10:08:02 +0000 Subject: [PATCH 17/18] ENH: Avoiding lazy loading in select command calls (#1617) --- improver/cli/__init__.py | 19 +++++++ improver/cli/blend_adjacent_points.py | 2 +- improver/cli/nowcast_accumulate.py | 40 +++++++++++-- improver/cli/phase_change_level.py | 2 +- .../cli/nowcast_accumulate/__init__.py | 0 .../test_name_constraint.py | 57 +++++++++++++++++++ improver_tests/cli/test_init.py | 35 +++++++++++- 7 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 improver_tests/cli/nowcast_accumulate/__init__.py create mode 100644 improver_tests/cli/nowcast_accumulate/test_name_constraint.py diff --git a/improver/cli/__init__.py b/improver/cli/__init__.py index 75f24541d6..1cd7ced499 100644 --- a/improver/cli/__init__.py +++ b/improver/cli/__init__.py @@ -155,6 +155,25 @@ def inputcube(to_convert): return maybe_coerce_with(load_cube, to_convert) +@value_converter +def inputcube_nolazy(to_convert): + """Loads cube from file or returns passed object. + Where a load is performed, it will not have lazy data. + Args: + to_convert (string or iris.cube.Cube): + File name or Cube object. + Returns: + Loaded cube or passed object. + """ + from improver.utilities.load import load_cube + + if getattr(to_convert, "has_lazy_data", False): + # Realise data if lazy + to_convert.data + + return maybe_coerce_with(load_cube, to_convert, no_lazy_load=True) + + @value_converter def inputcubelist(to_convert): """Loads a cubelist from file or returns passed object. diff --git a/improver/cli/blend_adjacent_points.py b/improver/cli/blend_adjacent_points.py index 556e4ce8cc..c383ad1555 100755 --- a/improver/cli/blend_adjacent_points.py +++ b/improver/cli/blend_adjacent_points.py @@ -38,7 +38,7 @@ @cli.clizefy @cli.with_output def process( - *cubes: cli.inputcube, + *cubes: cli.inputcube_nolazy, coordinate, central_point: float, units=None, diff --git a/improver/cli/nowcast_accumulate.py b/improver/cli/nowcast_accumulate.py index 7d1b8e5bce..3524cb178a 100755 --- a/improver/cli/nowcast_accumulate.py +++ b/improver/cli/nowcast_accumulate.py @@ -31,26 +31,54 @@ # POSSIBILITY OF SUCH DAMAGE. """Script to accumulate input data given advection velocity fields.""" +from typing import Callable, List + from improver import cli # The accumulation frequency in minutes. ACCUMULATION_FIDELITY = 1 + +def name_constraint(names: List[str]) -> Callable: + """ + Generates a callable constraint for matching cube names. + + The callable constraint will realise the data of those cubes matching the + constraint. + + Args: + name: + List of cube names to constrain our cubes. + + Returns: + A callable which when called, returns True or False for the provided cube, + depending on whether it matches the names provided. A matching cube + will also have its data realised by the callable. + """ + + def constraint(cube): + ret = False + if cube.name() in names: + ret = True + cube.data + return ret + + return constraint + + # Creates the value_converter that clize needs. inputadvection = cli.create_constrained_inputcubelist_converter( - lambda cube: cube.name() - in ["precipitation_advection_x_velocity", "grid_eastward_wind"], - lambda cube: cube.name() - in ["precipitation_advection_y_velocity", "grid_northward_wind"], + name_constraint(["precipitation_advection_x_velocity", "grid_eastward_wind"]), + name_constraint(["precipitation_advection_y_velocity", "grid_northward_wind"]), ) @cli.clizefy @cli.with_output def process( - cube: cli.inputcube, + cube: cli.inputcube_nolazy, advection_velocity: inputadvection, - orographic_enhancement: cli.inputcube, + orographic_enhancement: cli.inputcube_nolazy, *, attributes_config: cli.inputjson = None, max_lead_time=360, diff --git a/improver/cli/phase_change_level.py b/improver/cli/phase_change_level.py index 920f990e6c..d7d7ffb41d 100755 --- a/improver/cli/phase_change_level.py +++ b/improver/cli/phase_change_level.py @@ -37,7 +37,7 @@ @cli.clizefy @cli.with_output def process( - *cubes: cli.inputcube, + *cubes: cli.inputcube_nolazy, phase_change, grid_point_radius=2, horizontal_interpolation=True, diff --git a/improver_tests/cli/nowcast_accumulate/__init__.py b/improver_tests/cli/nowcast_accumulate/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/improver_tests/cli/nowcast_accumulate/test_name_constraint.py b/improver_tests/cli/nowcast_accumulate/test_name_constraint.py new file mode 100644 index 0000000000..ee39ddb07e --- /dev/null +++ b/improver_tests/cli/nowcast_accumulate/test_name_constraint.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# (C) British Crown Copyright 2017-2021 Met Office. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +"""Test nowcast_accumulate name_constraint function""" + +import dask.array as da +import pytest +from iris import Constraint +from iris.cube import Cube, CubeList + +from improver.cli.nowcast_accumulate import name_constraint + + +def test_all(): + """Check that cubes returned using the 'name_constraint' are not lazy""" + constraint = name_constraint(["dummy1"]) + dummy1_cube = Cube(da.zeros((1, 1), chunks=(1, 1)), long_name="dummy2") + dummy2_cube = Cube(da.zeros((1, 1), chunks=(1, 1)), long_name="dummy1") + assert dummy1_cube.has_lazy_data() + assert dummy2_cube.has_lazy_data() + + res = CubeList([dummy1_cube, dummy2_cube]).extract_cube( + Constraint(cube_func=constraint) + ) + assert res.name() == "dummy1" + assert not res.has_lazy_data() + + +if __name__ == "__main__": + pytest.main() diff --git a/improver_tests/cli/test_init.py b/improver_tests/cli/test_init.py index 5899b7a594..3dede0b929 100644 --- a/improver_tests/cli/test_init.py +++ b/improver_tests/cli/test_init.py @@ -33,8 +33,9 @@ import unittest from unittest.mock import patch +import dask.array as da import numpy as np -from iris.cube import CubeList +from iris.cube import Cube, CubeList from iris.exceptions import ConstraintMismatchError import improver @@ -43,6 +44,7 @@ create_constrained_inputcubelist_converter, docutilize, inputcube, + inputcube_nolazy, inputcubelist, inputjson, maybe_coerce_with, @@ -136,6 +138,37 @@ def test_basic(self, m): self.assertEqual(result, "return") +class Test_inputcube_nolazy(unittest.TestCase): + """Tests the input cube no lazy function""" + + def setUp(self): + coerce_patch = patch("improver.cli.maybe_coerce_with", return_value="return") + self.coerce_patch = coerce_patch.start() + self.addCleanup(coerce_patch.stop) + + def test_string_arg(self): + """ + Check that inputcube_nolazy calls the coerce func with the input + string. + """ + result = inputcube_nolazy("foo") + self.coerce_patch.assert_called_with( + improver.utilities.load.load_cube, "foo", no_lazy_load=True + ) + self.assertEqual(result, "return") + + def test_cube_arg(self): + """Check that a input lazy cube will be realised before return.""" + cube = Cube(da.zeros((1, 1), chunks=(1, 1)), long_name="dummy") + self.assertTrue(cube.has_lazy_data()) + result = inputcube_nolazy(cube) + self.coerce_patch.assert_called_with( + improver.utilities.load.load_cube, cube, no_lazy_load=True + ) + self.assertFalse(cube.has_lazy_data()) + self.assertEqual(result, "return") + + class Test_inputcubelist(unittest.TestCase): """Tests the input cubelist function""" From 71899626e29e169a232adc9e3d80e3d96d10d6f2 Mon Sep 17 00:00:00 2001 From: fionaRust Date: Mon, 17 Jan 2022 12:35:26 +0000 Subject: [PATCH 18/18] Remove __repr__ methods from all neighbourhood plugins (#1648) --- improver/nbhood/circular_kernel.py | 10 ------ improver/nbhood/nbhood.py | 12 ------- improver/nbhood/recursive_filter.py | 5 --- improver/nbhood/square_kernel.py | 8 ----- improver/nbhood/use_nbhood.py | 19 ------------ .../test_CircularNeighbourhood.py | 14 --------- ...tePercentilesFromACircularNeighbourhood.py | 15 --------- .../test_BaseNeighbourhoodProcessing.py | 25 --------------- ...t_GeneratePercentilesFromANeighbourhood.py | 23 +------------- .../nbhood/test_NeighbourhoodProcessing.py | 22 +------------ .../recursive_filter/test_RecursiveFilter.py | 15 --------- .../square_kernel/test_SquareNeighbourhood.py | 14 --------- ...t_ApplyNeighbourhoodProcessingWithAMask.py | 31 ------------------- 13 files changed, 2 insertions(+), 211 deletions(-) diff --git a/improver/nbhood/circular_kernel.py b/improver/nbhood/circular_kernel.py index e26e8be58d..21009fd2bd 100644 --- a/improver/nbhood/circular_kernel.py +++ b/improver/nbhood/circular_kernel.py @@ -164,11 +164,6 @@ def __init__( self.re_mask = re_mask self.kernel = None - def __repr__(self) -> str: - """Represent the configured plugin instance as a string.""" - result = "" - return result.format(self.weighted_mode, self.sum_or_fraction) - def apply_circular_kernel(self, cube: Cube, ranges: int) -> Cube: """ Method to apply a circular kernel to the data within the input cube in @@ -263,11 +258,6 @@ def __init__( except TypeError: self.percentiles = (percentiles,) - def __repr__(self) -> str: - """Represent the configured class instance as a string.""" - result = "" - return result.format(self.percentiles) - def pad_and_unpad_cube(self, slice_2d: Cube, kernel: ndarray) -> Cube: """ Method to pad and unpad a two dimensional cube. The input array is diff --git a/improver/nbhood/nbhood.py b/improver/nbhood/nbhood.py index 4cb1b5ca93..9f0cdd4e4c 100644 --- a/improver/nbhood/nbhood.py +++ b/improver/nbhood/nbhood.py @@ -128,18 +128,6 @@ def _find_radii( radii = np.interp(cube_lead_times, self.lead_times, self.radii) return radii - def __repr__(self) -> str: - """Represent the configured plugin instance as a string.""" - if callable(self.neighbourhood_method): - neighbourhood_method = self.neighbourhood_method() - else: - neighbourhood_method = self.neighbourhood_method - result = ( - "" - ) - return result.format(neighbourhood_method, self.radii, self.lead_times) - def process(self, cube: Cube, mask_cube: Optional[Cube] = None) -> Cube: """ Supply neighbourhood processing method, in order to smooth the diff --git a/improver/nbhood/recursive_filter.py b/improver/nbhood/recursive_filter.py index f76c78a9ce..909411f401 100644 --- a/improver/nbhood/recursive_filter.py +++ b/improver/nbhood/recursive_filter.py @@ -82,11 +82,6 @@ def __init__(self, iterations: Optional[int] = None, edge_width: int = 15) -> No self.edge_width = edge_width self.smoothing_coefficient_name_format = "smoothing_coefficient_{}" - def __repr__(self) -> str: - """Represent the configured plugin instance as a string.""" - result = " str: - """Represent the configured plugin instance as a string.""" - result = ( - "" - ) - return result.format(self.weighted_mode, self.sum_or_fraction, self.re_mask) - @staticmethod def _calculate_neighbourhood( data: ndarray, mask: ndarray, nb_size: int, sum_only: bool, re_mask: bool diff --git a/improver/nbhood/use_nbhood.py b/improver/nbhood/use_nbhood.py index 78ea0337b1..e52b9ba74f 100644 --- a/improver/nbhood/use_nbhood.py +++ b/improver/nbhood/use_nbhood.py @@ -199,25 +199,6 @@ def __init__( message = "re_mask should be set to False when using collapse_weights" raise ValueError(message) - def __repr__(self) -> str: - """Represent the configured plugin instance as a string.""" - result = ( - "" - ) - return result.format( - self.coord_for_masking, - self.neighbourhood_method, - self.radii, - self.lead_times, - self.collapse_weights, - self.weighted_mode, - self.sum_or_fraction, - self.re_mask, - ) - def collapse_mask_coord(self, cube: Cube) -> Cube: """ Collapse the chosen coordinate with the available weights. The result diff --git a/improver_tests/nbhood/circular_kernel/test_CircularNeighbourhood.py b/improver_tests/nbhood/circular_kernel/test_CircularNeighbourhood.py index d36ffa7a27..960d70e061 100644 --- a/improver_tests/nbhood/circular_kernel/test_CircularNeighbourhood.py +++ b/improver_tests/nbhood/circular_kernel/test_CircularNeighbourhood.py @@ -61,20 +61,6 @@ def test_sum_or_fraction(self): CircularNeighbourhood(sum_or_fraction=sum_or_fraction) -class Test__repr__(IrisTest): - - """Test the repr method.""" - - def test_basic(self): - """Test that the __repr__ returns the expected string.""" - - result = str(CircularNeighbourhood()) - msg = ( - "" - ) - self.assertEqual(str(result), msg) - - class Test_apply_circular_kernel(IrisTest): """Test neighbourhood circular probabilities plugin.""" diff --git a/improver_tests/nbhood/circular_kernel/test_GeneratePercentilesFromACircularNeighbourhood.py b/improver_tests/nbhood/circular_kernel/test_GeneratePercentilesFromACircularNeighbourhood.py index ecd8163149..d820f374c1 100644 --- a/improver_tests/nbhood/circular_kernel/test_GeneratePercentilesFromACircularNeighbourhood.py +++ b/improver_tests/nbhood/circular_kernel/test_GeneratePercentilesFromACircularNeighbourhood.py @@ -51,21 +51,6 @@ ) -class Test__repr__(IrisTest): - - """Test the repr method.""" - - def test_basic(self): - """Test that the __repr__ returns the expected string.""" - - result = str(GeneratePercentilesFromACircularNeighbourhood()) - msg = ( - "".format(DEFAULT_PERCENTILES) - ) - self.assertEqual(str(result), msg) - - class Test_make_percentile_cube(IrisTest): """Test the make_percentile_cube method from diff --git a/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py b/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py index 0d07fc855d..5ef07aceed 100644 --- a/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py +++ b/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py @@ -189,31 +189,6 @@ def test_radii_varying_with_lead_time_mismatch(self): NBHood(neighbourhood_method, radii, lead_times=lead_times) -class Test__repr__(IrisTest): - - """Test the repr method.""" - - def test_callable(self): - """Test that the __repr__ returns the expected string.""" - result = str(NBHood(CircularNeighbourhood(), 10000)) - msg = ( - "; " - "radii: 10000.0; lead_times: None>" - ) - self.assertEqual(result, msg) - - def test_not_callable(self): - """Test that the __repr__ returns the expected string.""" - result = str(NBHood("circular", 10000)) - msg = ( - "" - ) - self.assertEqual(result, msg) - - class Test__find_radii(IrisTest): """Test the internal _find_radii function is working correctly.""" diff --git a/improver_tests/nbhood/nbhood/test_GeneratePercentilesFromANeighbourhood.py b/improver_tests/nbhood/nbhood/test_GeneratePercentilesFromANeighbourhood.py index d581cd2e55..fb7d389495 100644 --- a/improver_tests/nbhood/nbhood/test_GeneratePercentilesFromANeighbourhood.py +++ b/improver_tests/nbhood/nbhood/test_GeneratePercentilesFromANeighbourhood.py @@ -52,12 +52,7 @@ def test_neighbourhood_method_exists(self): """ neighbourhood_method = "circular" radii = 10000 - result = NBHood(neighbourhood_method, radii) - msg = ( - "" - ) - self.assertEqual(str(result.neighbourhood_method), msg) + NBHood(neighbourhood_method, radii) def test_neighbourhood_method_does_not_exist(self): """ @@ -71,22 +66,6 @@ def test_neighbourhood_method_does_not_exist(self): NBHood(neighbourhood_method, radii) -class Test__repr__(IrisTest): - - """Test the repr method.""" - - def test_basic(self): - """Test that the __repr__ returns the expected string.""" - result = str(NBHood("circular", 10000)) - msg = ( - "; " - "radii: 10000.0; lead_times: None>" - ) - self.assertEqual(result, msg) - - class Test_process(IrisTest): """Test the process method.""" diff --git a/improver_tests/nbhood/nbhood/test_NeighbourhoodProcessing.py b/improver_tests/nbhood/nbhood/test_NeighbourhoodProcessing.py index fe39759ede..416d759563 100644 --- a/improver_tests/nbhood/nbhood/test_NeighbourhoodProcessing.py +++ b/improver_tests/nbhood/nbhood/test_NeighbourhoodProcessing.py @@ -50,11 +50,7 @@ def test_neighbourhood_method_exists(self): method exists.""" neighbourhood_method = "circular" radii = 10000 - result = NBHood(neighbourhood_method, radii) - msg = ( - "" - ) - self.assertEqual(str(result.neighbourhood_method), msg) + NBHood(neighbourhood_method, radii) def test_neighbourhood_method_does_not_exist(self): """Test that desired error message is raised, if the neighbourhood @@ -66,22 +62,6 @@ def test_neighbourhood_method_does_not_exist(self): NBHood(neighbourhood_method, radii) -class Test__repr__(IrisTest): - - """Test the repr method.""" - - def test_basic(self): - """Test that the __repr__ returns the expected string.""" - result = str(NBHood("circular", 10000)) - msg = ( - "; " - "radii: 10000.0; lead_times: None>" - ) - self.assertEqual(result, msg) - - class Test_process(IrisTest): """Test the process method.""" diff --git a/improver_tests/nbhood/recursive_filter/test_RecursiveFilter.py b/improver_tests/nbhood/recursive_filter/test_RecursiveFilter.py index eb6c0bc7bc..96575bb560 100644 --- a/improver_tests/nbhood/recursive_filter/test_RecursiveFilter.py +++ b/improver_tests/nbhood/recursive_filter/test_RecursiveFilter.py @@ -53,21 +53,6 @@ def _mean_points(points): return np.array((points[:-1] + points[1:]) / 2, dtype=np.float32) -class Test__repr__(IrisTest): - - """Test the repr method.""" - - def test_basic(self): - """Test that the __repr__ returns the expected string.""" - iterations = None - edge_width = 1 - result = str(RecursiveFilter(iterations, edge_width)) - msg = "".format(True, "fraction", True) - ) - self.assertEqual(result, msg) - - class Test_run(IrisTest): """Test the run method on the SquareNeighbourhood class.""" diff --git a/improver_tests/nbhood/use_nbhood/test_ApplyNeighbourhoodProcessingWithAMask.py b/improver_tests/nbhood/use_nbhood/test_ApplyNeighbourhoodProcessingWithAMask.py index cd5d0eec72..b703ee20e9 100644 --- a/improver_tests/nbhood/use_nbhood/test_ApplyNeighbourhoodProcessingWithAMask.py +++ b/improver_tests/nbhood/use_nbhood/test_ApplyNeighbourhoodProcessingWithAMask.py @@ -48,19 +48,6 @@ class Test__init__(unittest.TestCase): """Test the __init__ method of ApplyNeighbourhoodProcessingWithAMask.""" - def test_basic(self): - """Test that the __init__ method returns the expected string.""" - coord_for_masking = "topographic_zone" - radii = 2000 - result = ApplyNeighbourhoodProcessingWithAMask(coord_for_masking, radii) - msg = ( - "" - ) - self.assertEqual(str(result), msg) - def test_raises_error(self): """Test raises an error if re_mask=True when using collapse_weights""" message = "re_mask should be set to False when using collapse_weights" @@ -73,24 +60,6 @@ def test_raises_error(self): ) -class Test__repr__(unittest.TestCase): - - """Test the __repr__ method of ApplyNeighbourhoodProcessingWithAMask.""" - - def test_basic(self): - """Test that the __repr__ method returns the expected string.""" - coord_for_masking = "topographic_zone" - radii = 2000 - result = str(ApplyNeighbourhoodProcessingWithAMask(coord_for_masking, radii)) - msg = ( - "" - ) - self.assertEqual(result, msg) - - class Test_collapse_mask_coord(unittest.TestCase): """ Test the collapse_mask_coord method.