Merge branch 'refs/heads/master' into cli_gradient

* refs/heads/master:
  Failing CI (metoppv#2040)
  Gradient between vertical levels (metoppv#2030)
  Categorical fix for Deterministic data (metoppv#2038)

# Conflicts:
#	improver_tests/acceptance/SHA256SUMS

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 jobs:
   Sphinx-Pytest-Coverage:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -63,7 +63,7 @@ jobs:
         name:  ${{ matrix.env }}
       if: matrix.env != 'conda_forge'
   Codestyle-and-flake8:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -105,7 +105,7 @@ jobs:
         conda activate im${{ matrix.env }}
         flake8 improver improver_tests
   PR-standards:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
       - name: Checkout
@@ -115,7 +115,7 @@ jobs:
       - name: Check CONTRIBUTING.md
         uses: cylc/release-actions/check-shortlog@v1
   Safety-Bandit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:

.github/workflows/scheduled.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 jobs:
   Sphinx-Pytest-Coverage:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -58,7 +58,7 @@ jobs:
       uses: codecov/codecov-action@v4
       if: matrix.env == 'environment_a'
   Safety-Bandit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -90,7 +90,7 @@ jobs:
         conda activate im${{ matrix.env }}
         bandit -r improver
   Type-checking:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:

.github/workflows/stale.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   stale:
     if: "github.repository == 'metoppv/improver'"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/stale@v9
         with:

envs/environment_b.yml

@@ -44,3 +44,5 @@ dependencies:
   - sphinx-autodoc-typehints
   - sphinx_rtd_theme
   - threadpoolctl
+  # pinned dependencies of dependencies
+  - pyparsing=3.1.2

improver/categorical/decision_tree.py

@@ -390,21 +390,25 @@ def create_condition_chain(self, test_conditions: Dict) -> List:
                 # We have a list which could contain variable names, operators and
                 # numbers. The variable names need converting into Iris Constraint
                 # syntax while operators and numbers remain unchanged.
-                # We expect an entry in p_threshold for each variable name, so
-                # d_threshold_index is used to track these.
+                # We expect an entry in p_threshold for each condition in diagnostic
+                # fields. d_threshold_index is used for probabilistic trees to track
+                # the diagnostic threshold to extract for each variable in each condition.
                 d_threshold_index = -1
                 extract_constraint = []
                 for item in diagnostic:
                     if is_variable(item):
                         # Add a constraint from the variable name and threshold value
                         d_threshold_index += 1
-                        extract_constraint.append(
-                            self.construct_extract_constraint(
-                                item,
-                                d_threshold[d_threshold_index],
-                                self.coord_named_threshold,
+                        if test_conditions.get("deterministic"):
+                            extract_constraint.append(iris.Constraint(item))
+                        else:
+                            extract_constraint.append(
+                                self.construct_extract_constraint(
+                                    item,
+                                    d_threshold[d_threshold_index],
+                                    self.coord_named_threshold,
+                                )
                             )
-                        )
                     else:
                         # Add this operator or variable as-is
                         extract_constraint.append(item)
@@ -701,10 +705,10 @@ def evaluate_extract_expression(
                         + curr_expression[idx + 1 :]
                     )
             # evaluate operators in order of precedence
-            for op_str in ["/", "*", "+", "-"]:
+            for op_str in [["/", "*"], ["+", "-"]]:
                 while len(curr_expression) > 1:
                     for idx, item in enumerate(curr_expression):
-                        if isinstance(item, str) and (item == op_str):
+                        if isinstance(item, str) and (item in op_str):
                             left_arg = curr_expression[idx - 1]
                             right_arg = curr_expression[idx + 1]
                             if isinstance(left_arg, iris.Constraint):
@@ -715,7 +719,7 @@ def evaluate_extract_expression(
                                 right_eval = cubes.extract(right_arg)[0].data
                             else:
                                 right_eval = right_arg
-                            op = operator_map[op_str]
+                            op = operator_map[item]
                             res = op(left_eval, right_eval)
                             curr_expression = (
                                 curr_expression[: idx - 1]

improver/cli/gradient_between_vertical_levels.py

@@ -0,0 +1,43 @@
+# (C) Crown copyright, Met Office. All rights reserved.
+#
+# This file is part of IMPROVER and is released under a BSD 3-Clause license.
+# See LICENSE in the root of the repository for full licensing details.
+"""CLI to calculate a gradient between two vertical levels."""
+
+from improver import cli
+
+
+@cli.clizefy
+@cli.with_output
+def process(*cubes: cli.inputcubelist):
+    """
+    Calculate the gradient between two vertical levels. The gradient is calculated as the
+    difference between the input cubes divided by the difference in height.
+
+    Input cubes can be provided at height or pressure levels. If the cubes are provided
+    at a pressure level, the height above sea level is extracted from height_of_pressure_levels
+    cube. If exactly one of the cubes are provided at height levels this is assumed to be a
+    height above ground level and the height above sea level is calculated by adding the height
+    of the orography to the "height" coordinate of the cube. If both cubes have height coordinates
+    no additional cubes are required.
+    It is possible for one cube to be defined at height levels and the other at pressure levels.
+
+    Args:
+        cubes (iris.cube.CubeList):
+            Contains two cubes of a diagnostic at two different vertical levels. The cubes must
+            either have a height coordinate or a pressure coordinate. If only one cube is defined at
+            height levels, an orography cube must also be provided. If either cube is defined at
+            pressure levels, a geopotential_height cube must also be provided.
+
+    Returns:
+        iris.cube.Cube:
+            A single cube containing the gradient between the two height levels. This cube will be
+            renamed to "gradient_of" the cube name and will have a units attribute of the input
+            cube units per metre.
+
+    """
+    from improver.utilities.gradient_between_vertical_levels import (
+        GradientBetweenVerticalLevels,
+    )
+
+    return GradientBetweenVerticalLevels()(*cubes)

improver/utilities/gradient_between_vertical_levels.py

@@ -0,0 +1,164 @@
+# (C) Crown copyright, Met Office. All rights reserved.
+#
+# This file is part of IMPROVER and is released under a BSD 3-Clause license.
+# See LICENSE in the root of the repository for full licensing details.
+"""Calculate the gradient between two vertical levels."""
+
+from typing import Optional
+
+import iris
+import numpy as np
+from iris.cube import Cube, CubeList
+from iris.exceptions import CoordinateNotFoundError
+
+from improver import BasePlugin
+from improver.cube_combiner import Combine
+from improver.utilities.common_input_handle import as_cubelist
+from improver.utilities.cube_checker import assert_time_coords_valid
+
+
+class GradientBetweenVerticalLevels(BasePlugin):
+    """Calculate the gradient between two vertical levels. The gradient is calculated as the
+    difference between the input cubes divided by the difference in height."""
+
+    @staticmethod
+    def extract_cube_from_list(cubes: CubeList, name: str) -> Cube:
+        """Extract a cube from a cubelist based on the name if it exists. If the cube is found
+        it is removed from the cubelist.
+
+        Args:
+            cubes:
+                A cube list containing cubes.
+            name:
+                The name of the cube to be extracted.
+        Returns:
+            The extracted cube or None if there is no cube with the specified name. Also returns
+            the input cubelist with the extracted cube removed.
+        """
+        try:
+            extracted_cube = cubes.extract_cube(iris.Constraint(name))
+        except iris.exceptions.ConstraintMismatchError:
+            extracted_cube = None
+        else:
+            cubes.remove(extracted_cube)
+
+        return extracted_cube, cubes
+
+    def gradient_over_vertical_levels(
+        self,
+        cubes: CubeList,
+        geopotential_height: Optional[Cube],
+        orography: Optional[Cube],
+    ) -> Cube:
+        """Calculate the gradient between two vertical levels. The gradient is calculated as the
+        difference between the two cubes in cubes divided by the difference in height.
+
+        If the cubes are provided at height levels this is assumed to be a height above ground level
+        and the height above sea level is calculated by adding the height of the orography to the
+        height coordinate. If the cubes are provided at pressure levels, the height above sea level
+        is extracted from a geopotential_height cube. If both cubes have a
+        height coordinate then no additional cubes are required.
+
+        Args:
+            cubes:
+                Two cubes containing a diagnostic at two different vertical levels. The cubes must
+                contain either a height or pressure scalar coordinate. If the cubes contain a height
+                scalar coordinate this is assumed to be a height above ground level.
+            geopotential_height:
+                Optional cube that contains the height above sea level of pressure levels. This cube
+                is required if any input cube is defined at pressure level. This is used to extract
+                the height above sea level at the pressure level of the input cubes.
+            orography:
+                Optional cube containing the orography height above sea level. This cube is required
+                if exactly one input cube is defined at height levels and is used to convert the
+                height above ground level to height above sea level.
+
+        Returns:
+            A cube containing the gradient between the cubes between two vertical levels.
+
+        Raises:
+            ValueError: If exactly one input cube is defined at height levels and no orography cube
+                        is provided.
+            ValueError: If either input cube is defined at pressure levels and no
+                        geopotential_height cube is provided.
+        """
+
+        cube_heights = []
+        coord_used = []
+        for cube in cubes:
+            try:
+                cube_height = np.array(cube.coord("height").points)
+            except CoordinateNotFoundError:
+                if geopotential_height:
+                    height_ASL = geopotential_height.extract(
+                        iris.Constraint(pressure=cube.coord("pressure").points)
+                    )
+                    coord_used.append("pressure")
+                else:
+                    raise ValueError(
+                        """No geopotential height cube provided but one of the inputs cubes has a
+                        pressure coordinate"""
+                    )
+            else:
+                if orography:
+                    height_ASL = orography + cube_height
+                    coord_used.append("height")
+                elif not (orography or geopotential_height):
+                    height_ASL = cube_height
+                    coord_used.append("height")
+                else:
+                    raise ValueError(
+                        """No orography cube provided but one of the input cubes has height
+                        coordinate"""
+                    )
+            cube_heights.append(height_ASL)
+
+        height_diff = cube_heights[0] - cube_heights[1]
+        height_diff.data = np.ma.masked_where(height_diff.data == 0, height_diff.data)
+
+        if "height" in coord_used and "pressure" in coord_used:
+
+            try:
+                missing_coord = cubes[1].coord("pressure")
+            except CoordinateNotFoundError:
+                missing_coord = cubes[1].coord("height")
+            cubes[0].add_aux_coord(missing_coord)
+
+        diff = Combine(operation="-", expand_bound=True)([cubes[0], cubes[1]])
+
+        gradient = diff / height_diff
+
+        return gradient
+
+    def process(self, *cubes: CubeList) -> Cube:
+        """
+            Process the input cubes to calculate the gradient between two vertical levels.
+
+            Args:
+                cubes:
+                    A cubelist of two cubes containing a diagnostic at two vertical levels.
+                    The cubes must contain either a height or pressure scalar coordinate.
+                    If exactly one of the cubes contain a height scalar coordinate this is assumed
+                    to be a height above ground level and a cube with the name "surface_altitude"
+                    must also be provided. If either cube contains a pressure scalar coordinate
+                    a cube with the name "geopotential_height" must be provided.If both cubes are
+                    on height levels then no additional cubes are required.
+
+            Returns:
+                A cube containing the gradient between two vertical levels. The cube will be
+                named gradient of followed by the name of the input cubes.
+            """
+        cubes = as_cubelist(cubes)
+        orography, cubes = self.extract_cube_from_list(cubes, "surface_altitude")
+        geopotential_height, cubes = self.extract_cube_from_list(
+            cubes, "geopotential_height"
+        )
+        assert_time_coords_valid(cubes, time_bounds=False)
+
+        gradient = self.gradient_over_vertical_levels(
+            cubes, geopotential_height, orography
+        )
+
+        gradient.rename(f"gradient_of_{cubes[0].name()}")
+        gradient.units = f"{cubes[0].units} m-1"
+        return gradient

improver_tests/acceptance/SHA256SUMS

@@ -535,6 +535,11 @@ ba7a8a4dab8552656cfc38962d8cc6dce7e32d0965823c77008906d0c6afe7b5  ./generate-top
 4fe3a154c46e079ab0f42c04db93c94a35b1b7e73a6b3a303f29b8a7167aaa5b  ./gradient-between-adjacent-grid-squares/input.nc
 9792de8e7df23e9272c0fe8fa647256e5871c4dc1daf78d0584d5750c868bd43  ./gradient-between-adjacent-grid-squares/with_regrid/kgo.nc
 1324ec666cd38b5ee9974396c556457a91c020a00d90c34eac1d842013f869d8  ./gradient-between-adjacent-grid-squares/without_regrid/kgo.nc
+19e48705dd34036cf8e974dacdfeb510a55720f387daf64044d12b73d8d18ca7  ./gradient-between-vertical-levels/height_of_pressure_levels.nc
+fafd9c642ef7a63b1c453dae66cb55eab3d7816b02122d12da9320763d2d8db2  ./gradient-between-vertical-levels/kgo.nc
+a8eb2c78a65acb58d9535c3390fbf22249e98929002c21b5a4b5c534640d18e1  ./gradient-between-vertical-levels/orography.nc
+2048e3a2dc7ea8f9e990dceb1108d69223c52fad5a1b5e21bc6206c60e2115a0  ./gradient-between-vertical-levels/temperature_at_850hpa.nc
+0f61a96ffc719c656a6461b4f5562170e12dd1a4fb5096f5ada6ed89cc41ba7b  ./gradient-between-vertical-levels/temperature_at_screen_level.nc
 129a2405d8460804e577a1dbff2c0ab39aab709bd12f7d92b096af5ea0981380  ./hail-fraction/cloud_condensation_level.nc
 6be57a39af9927f742d6805bf7196d2667c94452dfb94b9ac9f8e65faa69471a  ./hail-fraction/convective_cloud_top_temperature.nc
 a01f1ab686e1570b89a91e2eb4d9c0fa1bb71bd1abf26d1e58b5b5610ae953dd  ./hail-fraction/hail_melting_level.nc

improver_tests/acceptance/test_gradient_between_vertical_levels.py

@@ -0,0 +1,37 @@
+# (C) Crown copyright, Met Office. All rights reserved.
+#
+# This file is part of IMPROVER and is released under a BSD 3-Clause license.
+# See LICENSE in the root of the repository for full licensing details.
+"""Tests for the gradient-between-vertical-levels CLI"""
+
+import pytest
+
+from . import acceptance as acc
+
+pytestmark = [pytest.mark.acc, acc.skip_if_kgo_missing]
+CLI = acc.cli_name_with_dashes(__file__)
+run_cli = acc.run_cli(CLI)
+
+
+def test_gradient_vertical_levels(tmp_path):
+    """Test gradient between vertical levels calculation returns expected result using
+    temperature at screen level and 850hPa, in combination with orography and height of
+    pressure levels."""
+    kgo_dir = acc.kgo_root() / CLI
+    kgo_path = kgo_dir / "kgo.nc"
+    temp_at_screen = kgo_dir / "temperature_at_screen_level.nc"
+    temp_at_850 = kgo_dir / "temperature_at_850hpa.nc"
+    orography = kgo_dir / "orography.nc"
+    height_of_pressure_levels = kgo_dir / "height_of_pressure_levels.nc"
+
+    output_path = tmp_path / "output.nc"
+    args = [
+        temp_at_screen,
+        temp_at_850,
+        orography,
+        height_of_pressure_levels,
+        "--output",
+        f"{output_path}",
+    ]
+    run_cli(args)
+    acc.compare(output_path, kgo_path)