#3615: Validate RTW user config file

.codacy.yml

@@ -21,5 +21,6 @@ engines:
 exclude_paths: [
     'doc/sphinx/**',
     'esmvaltool/cmor/tables/**',
-    'tests/**'
+    'tests/**',
+    'esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/test_configure.py'
 ]

esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst

@@ -20,7 +20,7 @@ The |RTW| performs the following steps:
 
 ``get_esmval``
   :Description:
-     Either clones the latest versions of |ESMValTool| and |ESMValCore| from GitHub, 
+     Either clones the latest versions of |ESMValTool| and |ESMValCore| from GitHub,
      or gets the latest container image from DockerHub and converts to a singularity
      image, depending on ``SITE``.
   :Runs on:
@@ -35,13 +35,13 @@ The |RTW| performs the following steps:
 
 ``configure``
   :Description:
-     Creates the |ESMValTool| user configuration file
+     Creates the |ESMValTool| user configuration file and validates it.
   :Runs on:
      Localhost
   :Executes:
      The ``configure.py`` script from the |Rose| app
   :Details:
-     ``configure`` should run at the start of each cycle after 
+     ``configure`` should run at the start of each cycle after
      ``install_env_file`` has completed.
 
 ``process``

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py

@@ -4,6 +4,7 @@
 import pprint
 
 import yaml
+from esmvalcore.config._config_validators import ValidationError, _validators
 
 
 def main():
@@ -17,6 +18,9 @@ def main():
     # 'configure' task.
     config_values = get_config_values_from_task_env()
 
+    # Validate the user config file content.
+    validate_user_config_file(config_values)
+
     # Update the configuration from OS environment.
     user_config_path = os.environ["USER_CONFIG_PATH"]
     config_values["config_file"] = user_config_path
@@ -80,6 +84,49 @@ def get_config_values_from_task_env():
     return config_values_from_task_env
 
 
+def validate_user_config_file(user_config_file_content):
+    """Validate a user config with ``ESMValCore.config._validators`` functions.
+
+    Parameters
+    ----------
+    user_config_file_content: dict
+        An ESMValTool user configuration file loaded in memory as a Python
+        dictionary.
+
+    Raises
+    ------
+    KeyError
+        If ``user_config_file_content`` includes a key for which there is no
+        validator listed in ``_validators``,
+    ValidationError
+        If any of the called validation functions raise a ValidationError.
+    """
+    errors = [
+        "There were validation errors in your user configuration file. See "
+        "details below.\n"
+    ]
+    for user_config_key, usr_config_value in user_config_file_content.items():
+        try:
+            validatation_function = _validators[user_config_key]
+        except KeyError as err:
+            errors.append(
+                f'Key Error for {user_config_key.upper()}. May not be a valid '
+                f'ESMValTool user configuration key\nERROR: {err}\n')
+        else:
+            try:
+                print(f'Validating {user_config_key.upper()} with value '
+                      f'"{usr_config_value}" using function '
+                      f'{validatation_function.__name__.upper()}.')
+                validatation_function(usr_config_value)
+            except ValidationError as err:
+                errors.append(
+                    f'Validation error for {user_config_key.upper()} with '
+                    f'value "{usr_config_value}"\nERROR: {err}\n')
+    if len(errors) > 1:
+        raise ValidationError("\n".join(errors))
+    print("All validation checks passed.")
+
+
 def write_yaml(file_path, contents):
     """Write ``contents`` to the YAML file ``file_path``.
 

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/test_configure.py

@@ -0,0 +1,76 @@
+import pytest
+from bin.configure import validate_user_config_file
+from esmvalcore.config._config_validators import ValidationError
+
+
+def test_validate_user_config_file():
+    mock_valid_config = {
+        "output_dir": "~/esmvaltool_output",
+        "auxiliary_data_dir": "~/auxiliary_data",
+        "search_esgf": "never",
+        "download_dir": "~/climate_data",
+        "max_parallel_tasks": None,
+        "log_level": "info",
+        "exit_on_warning": True,
+        "output_file_type": "png",
+    }
+    # No assert statement is needed. If the function call errors Pytest
+    # considers the test failed.
+    validate_user_config_file(mock_valid_config)
+
+
+def test_validate_user_config_file_one_validation_error():
+    mock_one_invalid_config = {
+        "output_dir": "~/esmvaltool_output",
+        "auxiliary_data_dir": "~/auxiliary_data",
+        "search_esgf": "never",
+        "download_dir": "~/climate_data",
+        "max_parallel_tasks": None,
+        "log_level": "info",
+        "exit_on_warning": 100,
+        "output_file_type": "png",
+    }
+    with pytest.raises(
+            ValidationError,
+            match='Validation error for EXIT_ON_WARNING with value "100"\n'
+            'ERROR: Could not convert `100` to `bool`\n'):
+        validate_user_config_file(mock_one_invalid_config)
+
+
+def test_validate_user_config_file_two_validation_errors():
+    mock_two_invalids_config = {
+        "output_dir": 111,
+        "auxiliary_data_dir": "~/auxiliary_data",
+        "search_esgf": "never",
+        "download_dir": "~/climate_data",
+        "max_parallel_tasks": None,
+        "log_level": "info",
+        "exit_on_warning": 100,
+        "output_file_type": "png",
+    }
+    with pytest.raises(
+            ValidationError,
+            match='Validation error for OUTPUT_DIR with value "111"\nERROR: '
+            'Expected a path, but got 111\n\nValidation error for '
+            'EXIT_ON_WARNING with value "100"\nERROR: Could not convert `100` '
+            'to `bool`\n'):
+        validate_user_config_file(mock_two_invalids_config)
+
+
+def test_validate_user_config_file_key_error():
+    mock_one_key_error = {
+        "output_dir": "~/esmvaltool_output",
+        "auxiliary_data_dir": "~/auxiliary_data",
+        "search_esgf": "never",
+        "download_dir": "~/climate_data",
+        "one_rogue_field": 90210,
+        "max_parallel_tasks": None,
+        "log_level": "info",
+        "exit_on_warning": True,
+        "output_file_type": "png",
+    }
+    with pytest.raises(
+            ValidationError,
+            match="Key Error for ONE_ROGUE_FIELD. May not be a valid "
+            "ESMValTool user configuration key\nERROR: 'one_rogue_field'\n"):
+        validate_user_config_file(mock_one_key_error)