From f4e2853a309850cf626296876c6e47b180089b96 Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:38:27 +0000 Subject: [PATCH 01/20] Fix bugs in `cylc broadcast` (#5933) broadcast: fix issues with filepaths and hash characters * Fix traceback when using `cylc broadcast -F` with a relative filepath * Fix `#` char bug in `cylc broadcast` --- changes.d/5933.fix.md | 1 + cylc/flow/parsec/validate.py | 87 +++++++++++------- cylc/flow/scripts/broadcast.py | 3 +- .../broadcast/10-file-1/broadcast.cylc | 3 + tests/unit/parsec/test_validate.py | 89 ++++++++++++++++--- 5 files changed, 137 insertions(+), 46 deletions(-) create mode 100644 changes.d/5933.fix.md diff --git a/changes.d/5933.fix.md b/changes.d/5933.fix.md new file mode 100644 index 00000000000..8ba0546c760 --- /dev/null +++ b/changes.d/5933.fix.md @@ -0,0 +1 @@ +Fixed bug in `cylc broadcast` (and the GUI Edit Runtime command) where everything after a `#` character in a setting would be stripped out. diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py index 4ef66e19862..ecd481a86bb 100644 --- a/cylc/flow/parsec/validate.py +++ b/cylc/flow/parsec/validate.py @@ -26,7 +26,7 @@ import shlex from collections import deque from textwrap import dedent -from typing import List, Dict, Any, Tuple +from typing import List, Dict, Any, Optional, Tuple from metomi.isodatetime.data import Duration, TimePoint from metomi.isodatetime.dumpers import TimePointDumper @@ -373,7 +373,7 @@ def coerce_range(cls, value, keys): return Range((int(min_), int(max_))) @classmethod - def coerce_str(cls, value, keys): + def coerce_str(cls, value, keys) -> str: """Coerce value to a string. Examples: @@ -385,7 +385,7 @@ def coerce_str(cls, value, keys): """ if isinstance(value, list): # handle graph string merging - vraw = [] + vraw: List[str] = [] vals = [value] while vals: val = vals.pop() @@ -512,16 +512,32 @@ def parse_int_range(cls, value): return None @classmethod - def strip_and_unquote(cls, keys, value): + def _unquote(cls, keys: List[str], value: str) -> Optional[str]: + """Unquote value.""" + for substr, rec in ( + ("'''", cls._REC_MULTI_LINE_SINGLE), + ('"""', cls._REC_MULTI_LINE_DOUBLE), + ('"', cls._REC_DQ_VALUE), + ("'", cls._REC_SQ_VALUE) + ): + if value.startswith(substr): + match = rec.match(value) + if not match: + raise IllegalValueError("string", keys, value) + return match[1] + return None + + @classmethod + def strip_and_unquote(cls, keys: List[str], value: str) -> str: """Remove leading and trailing spaces and unquote value. Args: - keys (list): + keys: Keys in nested dict that represents the raw configuration. - value (str): + value: String value in raw configuration. - Return (str): + Return: Processed value. Examples: @@ -529,24 +545,13 @@ def strip_and_unquote(cls, keys, value): 'foo' """ - for substr, rec in [ - ["'''", cls._REC_MULTI_LINE_SINGLE], - ['"""', cls._REC_MULTI_LINE_DOUBLE], - ['"', cls._REC_DQ_VALUE], - ["'", cls._REC_SQ_VALUE]]: - if value.startswith(substr): - match = rec.match(value) - if not match: - raise IllegalValueError("string", keys, value) - value = match.groups()[0] - break - else: - # unquoted - value = value.split(r'#', 1)[0] + val = cls._unquote(keys, value) + if val is None: + val = value.split(r'#', 1)[0] # Note strip() removes leading and trailing whitespace, including # initial newlines on a multiline string: - return dedent(value).strip() + return dedent(val).strip() @classmethod def strip_and_unquote_list(cls, keys, value): @@ -1136,23 +1141,25 @@ def _coerce_type(cls, value): return val -# BACK COMPAT: BroadcastConfigValidator -# The DB at 8.0.x stores Interval values as neither ISO8601 duration -# string or DurationFloat. This has been fixed at 8.1.0, and -# the following class acts as a bridge between fixed and broken. -# url: -# https://github.com/cylc/cylc-flow/pull/5138 -# from: -# 8.0.x -# to: -# 8.1.x -# remove at: -# 8.x class BroadcastConfigValidator(CylcConfigValidator): """Validate and Coerce DB loaded broadcast config to internal objects.""" def __init__(self): CylcConfigValidator.__init__(self) + @classmethod + def coerce_str(cls, value, keys) -> str: + """Coerce value to a string. Unquotes & strips lead/trail whitespace. + + Prevents ParsecValidator from assuming '#' means comments; + '#' has valid uses in shell script such as parameter substitution. + + Examples: + >>> BroadcastConfigValidator.coerce_str('echo "${FOO#*bar}"', None) + 'echo "${FOO#*bar}"' + """ + val = ParsecValidator._unquote(keys, value) or value + return dedent(val).strip() + @classmethod def strip_and_unquote_list(cls, keys, value): """Remove leading and trailing spaces and unquote list value. @@ -1177,6 +1184,18 @@ def strip_and_unquote_list(cls, keys, value): value = value.lstrip('[').rstrip(']') return ParsecValidator.strip_and_unquote_list(keys, value) + # BACK COMPAT: BroadcastConfigValidator.coerce_interval + # The DB at 8.0.x stores Interval values as neither ISO8601 duration + # string or DurationFloat. This has been fixed at 8.1.0, and + # the following method acts as a bridge between fixed and broken. + # url: + # https://github.com/cylc/cylc-flow/pull/5138 + # from: + # 8.0.x + # to: + # 8.1.x + # remove at: + # 8.x @classmethod def coerce_interval(cls, value, keys): """Coerce an ISO 8601 interval into seconds. diff --git a/cylc/flow/scripts/broadcast.py b/cylc/flow/scripts/broadcast.py index d1a62d708da..28bd3bae4e3 100755 --- a/cylc/flow/scripts/broadcast.py +++ b/cylc/flow/scripts/broadcast.py @@ -82,6 +82,7 @@ from ansimarkup import parse as cparse from copy import deepcopy from functools import partial +import os.path import re import sys from tempfile import NamedTemporaryFile @@ -203,7 +204,7 @@ def files_to_settings(settings, setting_files, cancel_mode=False): handle.seek(0, 0) cfg.loadcfg(handle.name) else: - cfg.loadcfg(setting_file) + cfg.loadcfg(os.path.abspath(setting_file)) stack = [([], cfg.get(sparse=True))] while stack: keys, item = stack.pop() diff --git a/tests/functional/broadcast/10-file-1/broadcast.cylc b/tests/functional/broadcast/10-file-1/broadcast.cylc index f429f8a2740..243502ed547 100644 --- a/tests/functional/broadcast/10-file-1/broadcast.cylc +++ b/tests/functional/broadcast/10-file-1/broadcast.cylc @@ -1,6 +1,9 @@ script=""" printenv CYLC_FOOBAR +# This hash char should not cause the rest of the script to be stripped out +# - https://github.com/cylc/cylc-flow/pull/5933 + if (($CYLC_TASK_TRY_NUMBER < 2 )); then false fi diff --git a/tests/unit/parsec/test_validate.py b/tests/unit/parsec/test_validate.py index 8c73b840eb3..3e24bcf6365 100644 --- a/tests/unit/parsec/test_validate.py +++ b/tests/unit/parsec/test_validate.py @@ -18,12 +18,13 @@ from typing import List import pytest -from pytest import approx +from pytest import approx, param from cylc.flow.parsec.config import ConfigNode as Conf from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.exceptions import IllegalValueError from cylc.flow.parsec.validate import ( + BroadcastConfigValidator, CylcConfigValidator as VDR, DurationFloat, ListValueError, @@ -456,11 +457,9 @@ def test_coerce_int_list(): validator.coerce_int_list(value, ['whatever']) -def test_coerce_str(): - """Test coerce_str.""" - validator = ParsecValidator() - # The good - for value, result in [ +@pytest.mark.parametrize( + 'value, expected', + [ ('', ''), ('Hello World!', 'Hello World!'), ('"Hello World!"', 'Hello World!'), @@ -474,9 +473,15 @@ def test_coerce_str(): 'Hello:\n foo\nGreet\n baz'), ('False', 'False'), ('None', 'None'), - (['a', 'b'], 'a\nb') - ]: - assert validator.coerce_str(value, ['whatever']) == result + (['a', 'b'], 'a\nb'), + ('abc#def', 'abc'), + ] +) +def test_coerce_str(value: str, expected: str): + """Test coerce_str.""" + validator = ParsecValidator() + # The good + assert validator.coerce_str(value, ['whatever']) == expected def test_coerce_str_list(): @@ -498,9 +503,51 @@ def test_coerce_str_list(): assert validator.coerce_str_list(value, ['whatever']) == results -def test_strip_and_unquote(): +@pytest.mark.parametrize('value, expected', [ + param( + "'a'", + 'a', + id="single quotes" + ), + param( + '"\'a\'"', + "'a'", + id="single quotes inside double quotes" + ), + param( + '" a b" # comment', + ' a b', + id="comment outside" + ), + param( + '"""bene\ngesserit"""', + 'bene\ngesserit', + id="multiline double quotes" + ), + param( + "'''kwisatz\n haderach'''", + 'kwisatz\n haderach', + id="multiline single quotes" + ), + param( + '"""a\nb""" # comment', + 'a\nb', + id="multiline with comment outside" + ), +]) +def test_unquote(value: str, expected: str): + """Test strip_and_unquote.""" + assert ParsecValidator._unquote(['a'], value) == expected + + +@pytest.mark.parametrize('value', [ + '"""', + "'''", + "'don't do this'", +]) +def test_strip_and_unquote__bad(value: str): with pytest.raises(IllegalValueError): - ParsecValidator.strip_and_unquote(['a'], '"""') + ParsecValidator.strip_and_unquote(['a'], value) def test_strip_and_unquote_list_parsec(): @@ -692,3 +739,23 @@ def test_type_help_examples(): except Exception: raise Exception( f'Example "{example}" failed for type "{vdr}"') + + +@pytest.mark.parametrize('value, expected', [ + param( + """ + a="don't have a cow" + a=${a#*have} + echo "$a" # let's see what happens + """, + "a=\"don't have a cow\"\na=${a#*have}\necho \"$a\" # let's see what happens", + id="multiline" + ), + param( + '"sleep 30 # ja!" ', + 'sleep 30 # ja!', + id="quoted" + ), +]) +def test_broadcast_coerce_str(value: str, expected: str): + assert BroadcastConfigValidator.coerce_str(value, ['whatever']) == expected From 08d9c531f1adc7cf83651dbb7befdadac5bd8aed Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Mon, 29 Jan 2024 16:28:54 +0000 Subject: [PATCH 02/20] add example workflows (#5773) add example workflows * Partially addresses https://github.com/cylc/cylc-doc/issues/627 * Add some examples of Cylc workflow implementation patterns. --------- Co-authored-by: Hilary James Oliver Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> --- README.md | 22 ++----- cylc/flow/etc/examples/README.md | 27 +++++++++ .../examples/converging-workflow/.validate | 33 +++++++++++ .../examples/converging-workflow/flow.cylc | 35 ++++++++++++ .../examples/converging-workflow/index.rst | 39 +++++++++++++ .../etc/examples/datetime-cycling/.validate | 28 +++++++++ .../etc/examples/datetime-cycling/flow.cylc | 44 ++++++++++++++ .../etc/examples/datetime-cycling/index.rst | 16 ++++++ .../examples/event-driven-cycling/.validate | 47 +++++++++++++++ .../examples/event-driven-cycling/README.rst | 49 ++++++++++++++++ .../examples/event-driven-cycling/bin/trigger | 32 +++++++++++ .../examples/event-driven-cycling/flow.cylc | 32 +++++++++++ .../examples/event-driven-cycling/index.rst | 14 +++++ cylc/flow/etc/examples/hello-world/.validate | 23 ++++++++ cylc/flow/etc/examples/hello-world/flow.cylc | 13 +++++ cylc/flow/etc/examples/hello-world/index.rst | 21 +++++++ .../etc/examples/integer-cycling/.validate | 23 ++++++++ .../etc/examples/integer-cycling/flow.cylc | 28 +++++++++ .../etc/examples/integer-cycling/index.rst | 16 ++++++ .../inter-workflow-triggers/.validate | 57 +++++++++++++++++++ .../inter-workflow-triggers/README.rst | 14 +++++ .../downstream/flow.cylc | 25 ++++++++ .../inter-workflow-triggers/index.rst | 28 +++++++++ .../upstream/flow.cylc | 27 +++++++++ cylc/flow/resources.py | 57 +++++++++++++++---- etc/bin/run-validate-tutorials | 18 +++--- 26 files changed, 730 insertions(+), 38 deletions(-) create mode 100644 cylc/flow/etc/examples/README.md create mode 100755 cylc/flow/etc/examples/converging-workflow/.validate create mode 100644 cylc/flow/etc/examples/converging-workflow/flow.cylc create mode 100644 cylc/flow/etc/examples/converging-workflow/index.rst create mode 100755 cylc/flow/etc/examples/datetime-cycling/.validate create mode 100644 cylc/flow/etc/examples/datetime-cycling/flow.cylc create mode 100644 cylc/flow/etc/examples/datetime-cycling/index.rst create mode 100755 cylc/flow/etc/examples/event-driven-cycling/.validate create mode 100644 cylc/flow/etc/examples/event-driven-cycling/README.rst create mode 100755 cylc/flow/etc/examples/event-driven-cycling/bin/trigger create mode 100644 cylc/flow/etc/examples/event-driven-cycling/flow.cylc create mode 100644 cylc/flow/etc/examples/event-driven-cycling/index.rst create mode 100755 cylc/flow/etc/examples/hello-world/.validate create mode 100644 cylc/flow/etc/examples/hello-world/flow.cylc create mode 100644 cylc/flow/etc/examples/hello-world/index.rst create mode 100755 cylc/flow/etc/examples/integer-cycling/.validate create mode 100644 cylc/flow/etc/examples/integer-cycling/flow.cylc create mode 100644 cylc/flow/etc/examples/integer-cycling/index.rst create mode 100755 cylc/flow/etc/examples/inter-workflow-triggers/.validate create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/README.rst create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/index.rst create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc diff --git a/README.md b/README.md index bc28920d48f..e97a7731ab4 100644 --- a/README.md +++ b/README.md @@ -29,28 +29,14 @@ domains. # install cylc conda install cylc-flow -# write your first workflow -mkdir -p ~/cylc-src/example -cat > ~/cylc-src/example/flow.cylc <<__CONFIG__ -[scheduling] - initial cycle point = 1 - cycling mode = integer - [[graph]] - P1 = """ - a => b => c & d - b[-P1] => b - """ -[runtime] - [[a, b, c, d]] - script = echo "Hello $CYLC_TASK_NAME" -__CONFIG__ +# extract an example to run +cylc get-resources examples/integer-cycling # install and run it -cylc install example -cylc play example +cylc vip integer-cycling # vip = validate, install and play # watch it run -cylc tui example +cylc tui integer-cycling ``` ### The Cylc Ecosystem diff --git a/cylc/flow/etc/examples/README.md b/cylc/flow/etc/examples/README.md new file mode 100644 index 00000000000..ae4e1391d23 --- /dev/null +++ b/cylc/flow/etc/examples/README.md @@ -0,0 +1,27 @@ +# Examples + +These examples are intended to illustrate the major patterns for implementing +Cylc workflows. The hope is that users can find a workflow which fits their +pattern, make a copy and fill in the details. Keep the examples minimal and +abstract. We aren't trying to document every Cylc feature here, just the +major design patterns. + +These examples are auto-documented in cylc-doc which looks for an `index.rst` +file in each example. + +Users can extract them using `cylc get-resources` which will put them into the +configured Cylc source directory (`~/cylc-src` by default). They can then be +run using the directory name, e.g. `cylc vip hello-world`. + +Files: + +* `index.rst` + This file is used to generate a page in the documentation for the example. + This file is excluded when the user extracts the example. +* `.validate` + This is a test file, it gets detected and run automatically. + This file is excluded when the user extracts the example. +* `README.rst` + Examples can include a README file, to save duplication, you can + `.. include::` this in the `index.rst` file (hence using ReStructuredText + rather than Markdown). diff --git a/cylc/flow/etc/examples/converging-workflow/.validate b/cylc/flow/etc/examples/converging-workflow/.validate new file mode 100755 index 00000000000..ad61b518ab2 --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/.validate @@ -0,0 +1,33 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + +# start the workflow +cylc vip --check-circular --no-run-name --no-detach --workflow-name "$ID" + +# it should have reached the forth cycle +test -d "${HOME}/cylc-run/${ID}/log/job/4" +test ! -d "${HOME}/cylc-run/${ID}/log/job/5" + +# lint +cylc lint "$ID" + +# clean up +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/converging-workflow/flow.cylc b/cylc/flow/etc/examples/converging-workflow/flow.cylc new file mode 100644 index 00000000000..b2069a3f479 --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/flow.cylc @@ -0,0 +1,35 @@ +[meta] + title = Converging Workflow + description = """ + A workflow which runs a pattern of tasks over and over until a + convergence condition has been met. + """ + +[scheduling] + cycling mode = integer + initial cycle point = 1 + [[graph]] + P1 = """ + # run "increment" then check the convergence condition + check_convergence[-P1]:not_converged? => increment => check_convergence + + # if the workflow has converged, then do nothing + check_convergence:converged? + """ + +[runtime] + [[increment]] + # a task which evolves the data + [[check_convergence]] + # a task which checks whether the convergence condition has been met + script = """ + if (( CYLC_TASK_CYCLE_POINT == 4 )); then + # for the purpose of example, assume convergence at cycle point 4 + cylc message -- 'convergence condition met' + else + cylc message -- 'convergence condition not met' + fi + """ + [[[outputs]]] + converged = 'convergence condition met' + not_converged = 'convergence condition not met' diff --git a/cylc/flow/etc/examples/converging-workflow/index.rst b/cylc/flow/etc/examples/converging-workflow/index.rst new file mode 100644 index 00000000000..27da079d58f --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/index.rst @@ -0,0 +1,39 @@ +Converging Workflow +=================== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/converging-workflow + +A workflow which runs a pattern of tasks over and over until a convergence +condition has been met. + +* The ``increment`` task runs some kind of model or process which increments + us toward the solution. +* The ``check_convergence`` task, checks if the convergence condition has been + met. + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip converging-workflow + +.. admonition:: Example - Genetic algorithms + :class: hint + + .. _genetic algorithm: https://en.wikipedia.org/wiki/Genetic_algorithm + + An example of a converging workflow might be a `genetic algorithm`_, where you + "breed" entities, then test their "fitness", and breed again, over and over + until you end up with an entity which is able to satisfy the requirement. + + .. digraph:: Example + + random_seed -> breed -> test_fitness + test_fitness -> breed + test_fitness -> stop diff --git a/cylc/flow/etc/examples/datetime-cycling/.validate b/cylc/flow/etc/examples/datetime-cycling/.validate new file mode 100755 index 00000000000..32c2a51908b --- /dev/null +++ b/cylc/flow/etc/examples/datetime-cycling/.validate @@ -0,0 +1,28 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip \ + --check-circular \ + --no-detach \ + --no-run-name \ + --final-cycle-point "$(isodatetime now --format 'CCYYMMDD')T00" \ + --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/datetime-cycling/flow.cylc b/cylc/flow/etc/examples/datetime-cycling/flow.cylc new file mode 100644 index 00000000000..76945349987 --- /dev/null +++ b/cylc/flow/etc/examples/datetime-cycling/flow.cylc @@ -0,0 +1,44 @@ +[meta] + title = Datetime Cycling + description = """ + A basic cycling workflow which runs the same set of tasks over + and over. Each cycle will be given a datetime identifier. + + The task "a" will wait until the real-world (or wallclock) time passes + the cycle time. + + Try changing the "initial cycle point" to "previous(00T00) - P1D" to + see how this works. + """ + +[scheduling] + # set the start of the graph to 00:00 this morning + initial cycle point = previous(T00) + + [[graph]] + # repeat this with a "P"eriod of "1" "D"ay -> P1D + P1D = """ + # this is the workflow we want to repeat: + a => b => c & d + + # this is an "inter-cycle dependency", it makes the task "b" + # wait until its previous instance has successfully completed: + b[-P1D] => b + + # this makes the task "a" wait until its cycle point matches + # the real world time - i.e. it prevents the workflow from getting + # ahead of the clock. If the workflow is running behind (e.g. after + # a delay, or from an earlier initial cycle point) it will catch + # until the clock-trigger constrains it again. To run entirely in + # "simulated time" remove this line: + @wall_clock => a + """ + +[runtime] + [[root]] + # all tasks will "inherit" the configuration in the "root" section + script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!" + [[a]] + [[b]] + [[c]] + [[d]] diff --git a/cylc/flow/etc/examples/datetime-cycling/index.rst b/cylc/flow/etc/examples/datetime-cycling/index.rst new file mode 100644 index 00000000000..803a532551f --- /dev/null +++ b/cylc/flow/etc/examples/datetime-cycling/index.rst @@ -0,0 +1,16 @@ +Datetime Cycling +================ + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/datetime-cycling + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip datetime-cycling diff --git a/cylc/flow/etc/examples/event-driven-cycling/.validate b/cylc/flow/etc/examples/event-driven-cycling/.validate new file mode 100755 index 00000000000..ef224cd9e2c --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/.validate @@ -0,0 +1,47 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + +# start the workflow +cylc vip --check-circular --no-run-name --workflow-name "$ID" +sleep 1 # give it a reasonable chance to start up + +# kick off the first cycle +./bin/trigger "$ID" WORLD=earth + +# wait for it to complete +cylc workflow-state "$ID" \ + --task=run \ + --point=1 \ + --status=succeeded \ + --max-polls=60 \ + --interval=1 + +# check the job received the environment variable we provided +grep 'Hello earth' "$HOME/cylc-run/$ID/log/job/1/run/NN/job.out" + +# stop the workflow +cylc stop --kill --max-polls=10 --interval=2 "$ID" + +# lint +cylc lint "$ID" + +# clean up +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/event-driven-cycling/README.rst b/cylc/flow/etc/examples/event-driven-cycling/README.rst new file mode 100644 index 00000000000..4fa4ddde241 --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/README.rst @@ -0,0 +1,49 @@ +Cylc is good at orchestrating tasks to a schedule, e.g: + +* ``PT1H`` - every hour +* ``P1D`` - every day +* ``P1M`` - every month +* ``PT1H ! (T00, T12)`` - every hour, except midnight and midday. + +But sometimes the things you want to run don't have a schedule. + +This example uses ``cylc ext-trigger`` to establish a pattern where Cylc waits +for an external signal and starts a new cycle every time a signal is recieved. + +The signal can carry data using the ext-trigger ID, this example sets the ID +as a file path containing some data that we want to make available to the tasks +that run in the cycle it triggers. + +To use this example, first start the workflow as normal:: + + cylc vip event-driven-cycling + +Then, when you're ready, kick off a new cycle, specifying any +environment variables you want to configure this cycle with:: + + ./bin/trigger WORLD=earth + +Replacing ```` with the ID you installed this workflow as. + +.. admonition:: Example - CI/CD + :class: hint + + This pattern is good for CI/CD type workflows where you're waiting on + external events. This pattern is especially powerful when used with + sub-workflows where it provides a solution to two-dimensional cycling + problems. + +.. admonition:: Example - Polar satellite data processing + :class: hint + + Polar satellites pass overhead at irregular intervals. This makes it tricky + to schedule data processing because you don't know when the satellite will + pass over the receiver station. With the event driven cycling approach you + could start a new cycle every time data arrives. + +.. note:: + + * The number of parallel cycles can be adjusted by changing the + :cylc:conf:`[scheduling]runahead limit`. + * To avoid hitting the runahead limit, ensure that failures are handled in + the graph. diff --git a/cylc/flow/etc/examples/event-driven-cycling/bin/trigger b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger new file mode 100755 index 00000000000..9f917f35f5c --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu + +if [[ $# -lt 1 ]]; then + echo 'Usage ./trigger WORKFLOW_ID [KEY=VALUE ...]' >&2 + echo + echo 'Trigger a new cycle in the target workflow.' + echo 'Any environment variable KEY=VALUE pairs will be broadcasted to' + echo 'all tasks in the cycle.' + exit 1 +fi + +# determine the workflow +WORKFLOW_ID="$1" +shift +WORKFLOW_RUN_DIR="${HOME}/cylc-run/${WORKFLOW_ID}" +EXT_TRIGGER_DIR="${WORKFLOW_RUN_DIR}/triggers" +mkdir -p "$EXT_TRIGGER_DIR" + +# pick a trigger-id +TRIGGER_ID="$(isodatetime --print-format CCYYMMDDThhmmss)" + +# write environment variables to a broadcast file +TRIGGER_FILE="${EXT_TRIGGER_DIR}/${TRIGGER_ID}.cylc" +echo '[environment]' >"$TRIGGER_FILE" +for env in "$@"; do + echo " $env" >> "$TRIGGER_FILE" +done + +# issue the xtrigger +cylc ext-trigger "$WORKFLOW_ID" 'trigger' "$TRIGGER_ID" diff --git a/cylc/flow/etc/examples/event-driven-cycling/flow.cylc b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc new file mode 100644 index 00000000000..7711c2ba258 --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc @@ -0,0 +1,32 @@ +[scheduling] + cycling mode = integer + initial cycle point = 1 + runahead limit = P5 # max number of cycles which can run in parallel + [[special tasks]] + # register the external trigger, it must be given a name, + # here, 'trigger' is used as a placeholder, the bash script will + # need to be updated if this is changed + external-trigger = configure("trigger") + [[graph]] + P1 = """ + # use a "?" to prevent failures causing runahead stalls + configure? => run + """ + +[runtime] + [[configure]] + # this task reads in the broadcast file the trigger wrote + # and broadcasts any variables set to all tasks in this cycle + script = """ + echo "received new ext-trigger ID=$CYLC_EXT_TRIGGER_ID" + TRIGGER_FILE="${CYLC_WORKFLOW_RUN_DIR}/triggers/${CYLC_EXT_TRIGGER_ID}.cylc" + cylc broadcast "${CYLC_WORKFLOW_ID}" \ + -p "${CYLC_TASK_CYCLE_POINT}" \ + -F "${TRIGGER_FILE}" + """ + + [[run]] + # this task could be a sub-workflow + script = """ + echo "Hello $WORLD!" + """ diff --git a/cylc/flow/etc/examples/event-driven-cycling/index.rst b/cylc/flow/etc/examples/event-driven-cycling/index.rst new file mode 100644 index 00000000000..5980591f0fd --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/index.rst @@ -0,0 +1,14 @@ +Event Driven Cycling +==================== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/event-driven-cycling + +.. include:: README.rst + +.. literalinclude:: flow.cylc + :language: cylc diff --git a/cylc/flow/etc/examples/hello-world/.validate b/cylc/flow/etc/examples/hello-world/.validate new file mode 100755 index 00000000000..968a7b349cb --- /dev/null +++ b/cylc/flow/etc/examples/hello-world/.validate @@ -0,0 +1,23 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip --check-circular --no-detach --no-run-name --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/hello-world/flow.cylc b/cylc/flow/etc/examples/hello-world/flow.cylc new file mode 100644 index 00000000000..92510d1928b --- /dev/null +++ b/cylc/flow/etc/examples/hello-world/flow.cylc @@ -0,0 +1,13 @@ +[meta] + title = Hello World + description = """ + A simple workflow which runs a single task (hello_world) once. + """ + +[scheduling] + [[graph]] + R1 = hello_world + +[runtime] + [[hello_world]] + script = echo "Hello World!" diff --git a/cylc/flow/etc/examples/hello-world/index.rst b/cylc/flow/etc/examples/hello-world/index.rst new file mode 100644 index 00000000000..a73afdf1e44 --- /dev/null +++ b/cylc/flow/etc/examples/hello-world/index.rst @@ -0,0 +1,21 @@ +Hello World +----------- + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/hello-world + +In the time honoured tradition, this is the minimal Cylc workflow: + +.. literalinclude:: flow.cylc + :language: cylc + +It writes the phrase "Hello World!" to standard output (captured to the +``job.out`` log file). + +Run it with:: + + $ cylc vip hello-world diff --git a/cylc/flow/etc/examples/integer-cycling/.validate b/cylc/flow/etc/examples/integer-cycling/.validate new file mode 100755 index 00000000000..054f2662862 --- /dev/null +++ b/cylc/flow/etc/examples/integer-cycling/.validate @@ -0,0 +1,23 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip --check-circular --no-detach --no-run-name --final-cycle-point=1 --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/integer-cycling/flow.cylc b/cylc/flow/etc/examples/integer-cycling/flow.cylc new file mode 100644 index 00000000000..b8d2f2f05ce --- /dev/null +++ b/cylc/flow/etc/examples/integer-cycling/flow.cylc @@ -0,0 +1,28 @@ +[meta] + title = Integer Cycling + description = """ + A basic cycling workflow which runs the same set of tasks over + and over. Each cycle will be given an integer number. + """ + +[scheduling] + # tell Cylc to count cycles as numbers starting from the number 1 + cycling mode = integer + initial cycle point = 1 + [[graph]] + P1 = """ + # this is the workflow we want to repeat: + a => b => c & d + # this is an "inter-cycle dependency", it makes the task "b" + # wait until its previous instance has completed: + b[-P1] => b + """ + +[runtime] + [[root]] + # all tasks will "inherit" the configuration in the "root" section + script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!" + [[a]] + [[b]] + [[c]] + [[d]] diff --git a/cylc/flow/etc/examples/integer-cycling/index.rst b/cylc/flow/etc/examples/integer-cycling/index.rst new file mode 100644 index 00000000000..89be2bb0871 --- /dev/null +++ b/cylc/flow/etc/examples/integer-cycling/index.rst @@ -0,0 +1,16 @@ +Integer Cycling +=============== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/integer-cycling + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip integer-cycling diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/.validate b/cylc/flow/etc/examples/inter-workflow-triggers/.validate new file mode 100755 index 00000000000..bdd414e275d --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/.validate @@ -0,0 +1,57 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +UPID=inter-workflow-triggers/upstream +DOID=inter-workflow-triggers/downstream + +ICP="$(isodatetime now --format=CCYYMMDDThh --offset=-PT2H)" + +# run the workflows +cylc vip \ + --check-circular \ + --no-run-name \ + --final-cycle-point="$ICP" \ + --workflow-name "$UPID" \ + ./upstream +cylc vip \ + --check-circular \ + --no-run-name \ + --final-cycle-point="$ICP" \ + --workflow-name "$DOID" \ + ./downstream + +# wait for the first task in the downstream to succeed +cylc workflow-state "$DOID" \ + --task=process \ + --point="$ICP" \ + --status=succeeded \ + --max-polls=60 \ + --interval=1 + +# stop the workflows +cylc stop --kill --max-polls=10 --interval=2 "$UPID" +cylc stop --kill --max-polls=10 --interval=2 "$DOID" + +# lint'em +cylc lint "$UPID" +cylc lint "$DOID" + +# clean up +cylc clean "$UPID" +cylc clean "$DOID" diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/README.rst b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst new file mode 100644 index 00000000000..8aa8d7d7b6f --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst @@ -0,0 +1,14 @@ +This example shows how one workflow can "trigger off" of tasks in another +workflow. + +In this example, there are two workflows: + +* "upstream" writes a file. +* "downstream" reads this file. + +Run both workflows simultaneously to see this in action: + +.. code-block:: console + + $ cylc vip inter-workflow-triggers/upstream + $ cylc vip inter-workflow-triggers/downstream diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc new file mode 100644 index 00000000000..115ecd94755 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc @@ -0,0 +1,25 @@ +[meta] + title = Downstream Workflow + description = """ + This workflow uses the data provided by the upstream workflow. + """ + +[scheduling] + # start two hours before the current hour + initial cycle point = previous(T-00) - PT2H + [[xtriggers]] + # this is an "xtrigger" - it will wait for the task "b" in the same + # cycle from the workflow "upstream" + upstream = workflow_state(workflow="inter-workflow-triggers/upstream", task="b", point="%(point)s") + [[graph]] + PT1H = """ + @upstream => process + """ + +[runtime] + [[process]] + script = echo "The random number is: $(cat "$file")" + [[[environment]]] + # this is where the data should be written to in the upstream workflow + # Note: "runN" will point to the most recent run of a workflow + file = $HOME/cylc-run/upstream/runN/share/$CYLC_TASK_CYCLE_POINT diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/index.rst b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst new file mode 100644 index 00000000000..0fd70b17d21 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst @@ -0,0 +1,28 @@ +Inter-Workflow Triggering +========================= + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/inter-workflow-triggers + +.. include:: README.rst + +.. literalinclude:: upstream/flow.cylc + :language: cylc + :caption: Upstream Workflow + +.. literalinclude:: downstream/flow.cylc + :language: cylc + :caption: Downstream Workflow + +.. admonition:: Example - Decoupled workflows + :class: hint + + This pattern is useful where you have workflows that you want to keep decoupled + from one another, but still need to exchange data. E.G. in operational + meteorology we might have a global model (covering the whole Earth) and a + regional model (just covering a little bit of it) where the regional model + obtains its boundary condition from the global model. diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc new file mode 100644 index 00000000000..145c3b323db --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc @@ -0,0 +1,27 @@ +[meta] + title = Upstream Workflow + description = """ + This is the workflow which is providing the data that the downstream + workflow wants to use. + """ + +[scheduling] + # start two hours before the current time on the whole hour + initial cycle point = previous(T-00) - PT2H + [[graph]] + PT1H = """ + # wait for the "real world" time before running "a": + @wall_clock => a + + # then run task "b" + a => b + """ + +[runtime] + [[a]] + [[b]] + # write a random number to ~/cylc-run//share/ + # for the downstream workflow to use + script = echo "$RANDOM" > "$file" + [[[environment]]] + file = ${CYLC_WORKFLOW_SHARE_DIR}/${CYLC_TASK_CYCLE_POINT} diff --git a/cylc/flow/resources.py b/cylc/flow/resources.py index 79884e27476..d81b7f7211b 100644 --- a/cylc/flow/resources.py +++ b/cylc/flow/resources.py @@ -16,6 +16,7 @@ """Extract named resources from the cylc.flow package.""" +from contextlib import suppress from pathlib import Path from random import shuffle import shutil @@ -31,6 +32,7 @@ RESOURCE_DIR = Path(cylc.flow.__file__).parent / 'etc' TUTORIAL_DIR = RESOURCE_DIR / 'tutorial' +EXAMPLE_DIR = RESOURCE_DIR / 'examples' # {resource: brief description} @@ -52,6 +54,11 @@ def list_resources(write=print, headers=True): for path in TUTORIAL_DIR.iterdir() if path.is_dir() ] + examples = [ + path.relative_to(RESOURCE_DIR) + for path in EXAMPLE_DIR.iterdir() + if path.is_dir() + ] if headers: write('Resources:') max_len = max(len(res) for res in RESOURCE_NAMES) @@ -62,15 +69,21 @@ def list_resources(write=print, headers=True): for tutorial in tutorials: write(f' {tutorial}') write(f' {API_KEY}') + if headers: + write('\nExamples:') + for example in examples: + write(f' {example}') -def path_is_tutorial(src: Path) -> bool: - """Returns True if the src path is in the tutorial directory.""" - try: +def path_is_source_workflow(src: Path) -> bool: + """Returns True if the src path is a Cylc workflow.""" + with suppress(ValueError): src.relative_to(TUTORIAL_DIR) - except ValueError: - return False - return True + return True + with suppress(ValueError): + src.relative_to(EXAMPLE_DIR) + return True + return False def get_resources(resource: str, tgt_dir: Optional[str]): @@ -95,11 +108,11 @@ def get_resources(resource: str, tgt_dir: Optional[str]): '\nRun `cylc get-resources --list` for resource names.' ) - is_tutorial = path_is_tutorial(src) + is_source_workflow = path_is_source_workflow(src) # get the target path if not tgt_dir: - if is_tutorial: + if is_source_workflow: # this is a tutorial => use the primary source dir _tgt_dir = Path(glbl_cfg().get(['install', 'source dirs'])[0]) else: @@ -113,8 +126,8 @@ def get_resources(resource: str, tgt_dir: Optional[str]): tgt = tgt.resolve() # extract resources - extract_resource(src, tgt, is_tutorial) - if is_tutorial: + extract_resource(src, tgt, is_source_workflow) + if is_source_workflow: set_api_key(tgt) @@ -131,16 +144,32 @@ def _backup(tgt: Path) -> None: shutil.move(str(tgt), str(backup)) -def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None: +def extract_resource( + src: Path, + tgt: Path, + is_source_workflow: bool = False, +) -> None: """Extract src into tgt. NOTE: src can be a dir or a file. """ LOG.info(f"Extracting {src.relative_to(RESOURCE_DIR)} to {tgt}") - if is_tutorial and tgt.exists(): + if is_source_workflow and tgt.exists(): # target exists, back up the old copy _backup(tgt) + # files to exclude + if is_source_workflow: + excludes = [ + # test files + '.validate', + 'reftest', + # documentation files + 'index.rst', + ] + else: + excludes = [] + # create the target directory try: tgt.parent.mkdir(parents=True, exist_ok=True) @@ -151,6 +180,10 @@ def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None: shutil.copytree(str(src), str(tgt)) else: shutil.copyfile(str(src), str(tgt)) + for exclude in excludes: + path = tgt / exclude + if path.exists(): + path.unlink() except IsADirectoryError as exc: LOG.error( f'Cannot extract file {exc.filename} as there is an ' diff --git a/etc/bin/run-validate-tutorials b/etc/bin/run-validate-tutorials index e9b7be1ecde..72bf164fe2a 100755 --- a/etc/bin/run-validate-tutorials +++ b/etc/bin/run-validate-tutorials @@ -17,13 +17,15 @@ set -eu -cd "$(dirname "$0")" +cd "$(dirname "$0")/../../" -for FILE in $(echo ../../cylc/flow/etc/tutorial/*/.validate) ; do - echo "running $FILE" - ( - cd "$(dirname "$FILE")" - ./.validate - ) +for DIR in tutorial examples; do + echo "# Running tests for: $DIR" + for FILE in $(echo "cylc/flow/etc/$DIR/"*/.validate); do + echo "## Running test: $FILE" + ( + cd "$(dirname "$FILE")" + ./.validate + ) + done done - From 94fd7ac4129b8345a702382dbfdf6b55911eadf4 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Mon, 29 Jan 2024 17:28:13 +0000 Subject: [PATCH 03/20] docs: document exceptions (#5940) --- cylc/flow/exceptions.py | 76 ++++++++++++++------------ cylc/flow/jinja/filters/duration_as.py | 10 ++-- cylc/flow/jinja/filters/pad.py | 10 ++-- cylc/flow/jinja/filters/strftime.py | 14 +++-- cylc/flow/network/client.py | 8 +-- cylc/flow/network/scan.py | 32 +++++++---- cylc/flow/parsec/exceptions.py | 4 +- cylc/flow/workflow_files.py | 23 +++++++- 8 files changed, 111 insertions(+), 66 deletions(-) diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py index d1a459f996b..43591e7be9c 100644 --- a/cylc/flow/exceptions.py +++ b/cylc/flow/exceptions.py @@ -15,16 +15,12 @@ # along with this program. If not, see . """Exceptions for "expected" errors.""" -import errno + from textwrap import wrap from typing import ( - Callable, Dict, Iterable, - NoReturn, Optional, - Tuple, - Type, Union, TYPE_CHECKING, ) @@ -42,7 +38,6 @@ class CylcError(Exception): message to the user is more appropriate than traceback. CLI commands will catch this exception and exit with str(exception). - """ @@ -50,10 +45,14 @@ class PluginError(CylcError): """Represents an error arising from a Cylc plugin. Args: - entry_point: The plugin entry point as defined in setup.cfg + entry_point: + The plugin entry point as defined in setup.cfg (e.g. 'cylc.main_loop') - plugin_name: Name of the plugin - exc: Original exception caught when trying to run the plugin + plugin_name: + Name of the plugin + exc: + Original exception caught when trying to run the plugin + """ def __init__(self, entry_point: str, plugin_name: str, exc: Exception): @@ -78,12 +77,8 @@ class InputError(CylcError): class CylcConfigError(CylcError): - """Generic exception to handle an error in a Cylc configuration file. - - TODO: - * reference the configuration element causing the problem - - """ + """Generic exception to handle an error in a Cylc configuration file.""" + # TODO: reference the configuration element causing the problem class WorkflowConfigError(CylcConfigError): @@ -142,19 +137,6 @@ class ContactFileExists(CylcError): """Workflow contact file exists.""" -def handle_rmtree_err( - function: Callable, - path: str, - excinfo: Tuple[Type[Exception], Exception, object] -) -> NoReturn: - """Error handler for shutil.rmtree.""" - exc = excinfo[1] - if isinstance(exc, OSError) and exc.errno == errno.ENOTEMPTY: - # "Directory not empty", likely due to filesystem lag - raise FileRemovalError(exc) - raise exc - - class FileRemovalError(CylcError): """Exception for errors during deletion of files/directories, which are probably the filesystem's fault, not Cylc's.""" @@ -269,6 +251,18 @@ def __str__(self): class ClientError(CylcError): + """Base class for errors raised by Cylc client instances. + + For example, the workflow you are trying to connect to is stopped. + + Attributes: + message: + The exception message. + traceback: + The underlying exception instance if available. + workflow: + The workflow ID if available. + """ def __init__( self, @@ -291,7 +285,7 @@ def __str__(self) -> str: class WorkflowStopped(ClientError): - """Special case of ClientError for a stopped workflow.""" + """The Cylc scheduler you attempted to connect to is stopped.""" def __init__(self, workflow): self.workflow = workflow @@ -301,6 +295,16 @@ def __str__(self): class ClientTimeout(CylcError): + """The scheduler did not respond within the timeout. + + This could be due to: + * Network issues. + * Scheduler issues. + * Insufficient timeout. + + To increase the timeout, use the ``--comms-timeout`` option. + + """ def __init__(self, message: str, workflow: Optional[str] = None): self.message = message @@ -311,7 +315,7 @@ def __str__(self) -> str: class CyclingError(CylcError): - pass + """Base class for errors in cycling configuration.""" class CyclerTypeError(CyclingError): @@ -446,10 +450,14 @@ class NoPlatformsError(PlatformLookupError): """None of the platforms of a given set were reachable. Args: - identity: The name of the platform group or install target - set_type: Whether the set of platforms is a platform group or an - install target - place: Where the attempt to get the platform failed. + identity: + The name of the platform group or install target. + set_type: + Whether the set of platforms is a platform group or an install + target. + place: + Where the attempt to get the platform failed. + """ def __init__( self, identity: str, set_type: str = 'group', place: str = '' diff --git a/cylc/flow/jinja/filters/duration_as.py b/cylc/flow/jinja/filters/duration_as.py index bc692a68017..7f2775facb9 100644 --- a/cylc/flow/jinja/filters/duration_as.py +++ b/cylc/flow/jinja/filters/duration_as.py @@ -15,6 +15,8 @@ # along with this program. If not, see . """Filter for formatting ISO8601 duration strings.""" +from typing import Callable, Dict, Tuple + from metomi.isodatetime.parsers import DurationParser SECONDS_PER_MINUTE = 60.0 @@ -26,7 +28,7 @@ SECONDS_PER_DAY = SECONDS_PER_HOUR * HOURS_PER_DAY SECONDS_PER_WEEK = SECONDS_PER_DAY * DAYS_PER_WEEK -CONVERSIONS = { +CONVERSIONS: Dict[Tuple[str, str], Callable] = { ('s', 'seconds'): float, ('m', 'minutes'): lambda s: float(s) / SECONDS_PER_MINUTE, ('h', 'hours'): lambda s: float(s) / SECONDS_PER_HOUR, @@ -35,7 +37,7 @@ } -def duration_as(iso8601_duration, units): +def duration_as(iso8601_duration: str, units: str) -> float: """Format an :term:`ISO8601 duration` string as the specified units. Units for the conversion can be specified in a case-insensitive short or @@ -57,8 +59,8 @@ def duration_as(iso8601_duration, units): - ``{{CYCLE_SUBINTERVAL | duration_as('s') | int}}`` - 1800 Args: - iso8601_duration (str): Any valid ISO8601 duration as a string. - units (str): Destination unit for the duration conversion + iso8601_duration: Any valid ISO8601 duration as a string. + units: Destination unit for the duration conversion Return: The total number of the specified unit contained in the specified diff --git a/cylc/flow/jinja/filters/pad.py b/cylc/flow/jinja/filters/pad.py index 75c5e117dd3..1c013b22ce4 100644 --- a/cylc/flow/jinja/filters/pad.py +++ b/cylc/flow/jinja/filters/pad.py @@ -15,18 +15,20 @@ # along with this program. If not, see . """Filter for padding strings to a set number of chars.""" +from typing import Union -def pad(value, length, fillchar=' '): + +def pad(value: str, length: Union[int, str], fillchar: str = ' '): """Pads a string to some length with a fill character Useful for generating task names and related values in ensemble workflows. Args: - value (str): + value: The string to pad. - length (int/str): + length: The length for the returned string. - fillchar (str - optional): + fillchar: The character to fill in surplus space (space by default). Returns: diff --git a/cylc/flow/jinja/filters/strftime.py b/cylc/flow/jinja/filters/strftime.py index d1fcb5d3eef..1af97d75abc 100644 --- a/cylc/flow/jinja/filters/strftime.py +++ b/cylc/flow/jinja/filters/strftime.py @@ -15,10 +15,16 @@ # along with this program. If not, see . """Filter for formatting ISO8601 datetime strings.""" +from typing import Optional + from metomi.isodatetime.parsers import TimePointParser -def strftime(iso8601_datetime, strftime_str, strptime_str=None): +def strftime( + iso8601_datetime: str, + strftime_str: str, + strptime_str: Optional[str] = None, +): """Format an :term:`ISO8601 datetime` string using an strftime string. .. code-block:: cylc @@ -29,11 +35,11 @@ def strftime(iso8601_datetime, strftime_str, strptime_str=None): strptime string as the second argument. Args: - iso8601_datetime (str): + iso8601_datetime: Any valid ISO8601 datetime as a string. - strftime_str (str): + strftime_str: A valid strftime string to format the output datetime. - strptime_str (str - optional): + strptime_str: A valid strptime string defining the format of the provided iso8601_datetime. diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py index b3955d3b058..e7e26954d56 100644 --- a/cylc/flow/network/client.py +++ b/cylc/flow/network/client.py @@ -195,15 +195,15 @@ class WorkflowRuntimeClient( # type: ignore[misc] the contact file. Attributes: - host (str): + host: Workflow host name. - port (int): + port: Workflow host port. - timeout_handler (function): + timeout_handler: Optional function which runs before ClientTimeout is raised. This provides an interface for raising more specific exceptions in the event of a communication timeout. - header (dict): + header: Request "header" data to attach to each request. Usage: diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index c2202f3f31e..6de28ada517 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -48,7 +48,16 @@ import asyncio from pathlib import Path import re -from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + AsyncGenerator, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from packaging.version import parse as parse_version from packaging.specifiers import SpecifierSet @@ -422,7 +431,7 @@ def format_query(fields, filters=None): @pipe(preproc=format_query) -async def graphql_query(flow, fields, filters=None): +async def graphql_query(flow: dict, fields: Iterable, filters=None): """Obtain information from a GraphQL request to the flow. Requires: @@ -430,9 +439,9 @@ async def graphql_query(flow, fields, filters=None): * contact_info Args: - flow (dict): + flow: Flow information dictionary, provided by scan through the pipe. - fields (iterable): + fields: Iterable containing the fields to request e.g:: ['id', 'name'] @@ -464,12 +473,15 @@ async def graphql_query(flow, fields, filters=None): LOG.warning(f'Workflow not running: {flow["name"]}') return False try: - ret = await client.async_request( - 'graphql', - { - 'request_string': query, - 'variables': {} - } + ret = cast( + 'dict', + await client.async_request( + 'graphql', + { + 'request_string': query, + 'variables': {} + } + ) ) except WorkflowStopped: LOG.warning(f'Workflow not running: {flow["name"]}') diff --git a/cylc/flow/parsec/exceptions.py b/cylc/flow/parsec/exceptions.py index 3acfe31c820..690d583b885 100644 --- a/cylc/flow/parsec/exceptions.py +++ b/cylc/flow/parsec/exceptions.py @@ -78,8 +78,6 @@ class FileParseError(ParsecError): Classification of error (e.g. Jinja2Error). help_lines: Additional info to include in the exception. - - Args (ways of providing exception context - TODO rationalise this!): lines: (preferred) Dictionary in the format {filename: [context_line, ..., error_line]} @@ -140,7 +138,7 @@ def __str__(self) -> str: class TemplateVarLanguageClash(FileParseError): - ... + """Multiple workflow configuration templating engines configured.""" class EmPyError(FileParseError): diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py index de30bc1e7d2..2f03c8d4394 100644 --- a/cylc/flow/workflow_files.py +++ b/cylc/flow/workflow_files.py @@ -22,21 +22,25 @@ """ +from enum import Enum +import errno import os +from pathlib import Path import re import shutil -from enum import Enum -from pathlib import Path from subprocess import ( PIPE, Popen, TimeoutExpired, ) from typing import ( + Callable, Dict, Optional, Tuple, Union, + NoReturn, + Type, ) import cylc.flow.flags @@ -48,7 +52,7 @@ InputError, ServiceFileError, WorkflowFilesError, - handle_rmtree_err, + FileRemovalError, ) from cylc.flow.hostuserutil import ( get_user, @@ -64,6 +68,19 @@ from cylc.flow.util import cli_format +def handle_rmtree_err( + function: Callable, + path: str, + excinfo: Tuple[Type[Exception], Exception, object] +) -> NoReturn: + """Error handler for shutil.rmtree.""" + exc = excinfo[1] + if isinstance(exc, OSError) and exc.errno == errno.ENOTEMPTY: + # "Directory not empty", likely due to filesystem lag + raise FileRemovalError(exc) + raise exc + + class KeyType(Enum): """Used for authentication keys - public or private""" From 82af1f3c59e8c2f87752928f3c659166983be765 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 30 Jan 2024 16:35:13 +0000 Subject: [PATCH 04/20] docs: fix sort order of the first few examples (#5948) --- cylc/flow/etc/examples/{hello-world => 1-hello-world}/.validate | 0 cylc/flow/etc/examples/{hello-world => 1-hello-world}/flow.cylc | 0 cylc/flow/etc/examples/{hello-world => 1-hello-world}/index.rst | 0 .../etc/examples/{integer-cycling => 2-integer-cycling}/.validate | 0 .../etc/examples/{integer-cycling => 2-integer-cycling}/flow.cylc | 0 .../etc/examples/{integer-cycling => 2-integer-cycling}/index.rst | 0 .../examples/{datetime-cycling => 3-datetime-cycling}/.validate | 0 .../examples/{datetime-cycling => 3-datetime-cycling}/flow.cylc | 0 .../examples/{datetime-cycling => 3-datetime-cycling}/index.rst | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename cylc/flow/etc/examples/{hello-world => 1-hello-world}/.validate (100%) rename cylc/flow/etc/examples/{hello-world => 1-hello-world}/flow.cylc (100%) rename cylc/flow/etc/examples/{hello-world => 1-hello-world}/index.rst (100%) rename cylc/flow/etc/examples/{integer-cycling => 2-integer-cycling}/.validate (100%) rename cylc/flow/etc/examples/{integer-cycling => 2-integer-cycling}/flow.cylc (100%) rename cylc/flow/etc/examples/{integer-cycling => 2-integer-cycling}/index.rst (100%) rename cylc/flow/etc/examples/{datetime-cycling => 3-datetime-cycling}/.validate (100%) rename cylc/flow/etc/examples/{datetime-cycling => 3-datetime-cycling}/flow.cylc (100%) rename cylc/flow/etc/examples/{datetime-cycling => 3-datetime-cycling}/index.rst (100%) diff --git a/cylc/flow/etc/examples/hello-world/.validate b/cylc/flow/etc/examples/1-hello-world/.validate similarity index 100% rename from cylc/flow/etc/examples/hello-world/.validate rename to cylc/flow/etc/examples/1-hello-world/.validate diff --git a/cylc/flow/etc/examples/hello-world/flow.cylc b/cylc/flow/etc/examples/1-hello-world/flow.cylc similarity index 100% rename from cylc/flow/etc/examples/hello-world/flow.cylc rename to cylc/flow/etc/examples/1-hello-world/flow.cylc diff --git a/cylc/flow/etc/examples/hello-world/index.rst b/cylc/flow/etc/examples/1-hello-world/index.rst similarity index 100% rename from cylc/flow/etc/examples/hello-world/index.rst rename to cylc/flow/etc/examples/1-hello-world/index.rst diff --git a/cylc/flow/etc/examples/integer-cycling/.validate b/cylc/flow/etc/examples/2-integer-cycling/.validate similarity index 100% rename from cylc/flow/etc/examples/integer-cycling/.validate rename to cylc/flow/etc/examples/2-integer-cycling/.validate diff --git a/cylc/flow/etc/examples/integer-cycling/flow.cylc b/cylc/flow/etc/examples/2-integer-cycling/flow.cylc similarity index 100% rename from cylc/flow/etc/examples/integer-cycling/flow.cylc rename to cylc/flow/etc/examples/2-integer-cycling/flow.cylc diff --git a/cylc/flow/etc/examples/integer-cycling/index.rst b/cylc/flow/etc/examples/2-integer-cycling/index.rst similarity index 100% rename from cylc/flow/etc/examples/integer-cycling/index.rst rename to cylc/flow/etc/examples/2-integer-cycling/index.rst diff --git a/cylc/flow/etc/examples/datetime-cycling/.validate b/cylc/flow/etc/examples/3-datetime-cycling/.validate similarity index 100% rename from cylc/flow/etc/examples/datetime-cycling/.validate rename to cylc/flow/etc/examples/3-datetime-cycling/.validate diff --git a/cylc/flow/etc/examples/datetime-cycling/flow.cylc b/cylc/flow/etc/examples/3-datetime-cycling/flow.cylc similarity index 100% rename from cylc/flow/etc/examples/datetime-cycling/flow.cylc rename to cylc/flow/etc/examples/3-datetime-cycling/flow.cylc diff --git a/cylc/flow/etc/examples/datetime-cycling/index.rst b/cylc/flow/etc/examples/3-datetime-cycling/index.rst similarity index 100% rename from cylc/flow/etc/examples/datetime-cycling/index.rst rename to cylc/flow/etc/examples/3-datetime-cycling/index.rst From 870fb473399f6be909c041633c8af98691e4e08c Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Mon, 29 Jan 2024 15:12:40 +0000 Subject: [PATCH 05/20] stop after cycle point: support offsets * Closes #5939 * Support offsets (measured from the ICP) for the `stop after cycle point` to mirror the behaviour of the `final cycle point`. * Add integration test to lock-down `stop after cycle point` interactions. --- changes.d/5943.feat.md | 1 + cylc/flow/cfgspec/workflow.py | 21 ++- cylc/flow/config.py | 13 +- cylc/flow/parsec/validate.py | 28 ++++ cylc/flow/scheduler.py | 1 + cylc/flow/task_pool.py | 2 +- .../functional/cylc-play/09-invalid-cp-opt.t | 2 +- .../test_stop_after_cycle_point.py | 132 ++++++++++++++++++ tests/unit/test_config.py | 115 ++++++++------- 9 files changed, 258 insertions(+), 57 deletions(-) create mode 100644 changes.d/5943.feat.md create mode 100644 tests/integration/test_stop_after_cycle_point.py diff --git a/changes.d/5943.feat.md b/changes.d/5943.feat.md new file mode 100644 index 00000000000..6db31d952be --- /dev/null +++ b/changes.d/5943.feat.md @@ -0,0 +1 @@ +The `stop after cycle point` can now be specified as an offset from the inital cycle point. diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index de919c27c0f..7ae7cf014b2 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -539,7 +539,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): ''') # NOTE: final cycle point is not a V_CYCLE_POINT to allow expressions # such as '+P1Y' (relative to initial cycle point) - Conf('final cycle point', VDR.V_STRING, desc=''' + Conf('final cycle point', VDR.V_CYCLE_POINT_WITH_OFFSETS, desc=''' The (optional) last cycle point at which tasks are run. Once all tasks have reached this cycle point, the @@ -547,6 +547,12 @@ def get_script_common_text(this: str, example: Optional[str] = None): This item can be overridden on the command line using ``cylc play --final-cycle-point`` or ``--fcp``. + + Examples: + + - ``2000`` - Shorthand for ``2000-01-01T00:00``. + - ``+P1D`` - The initial cycle point plus one day. + - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day. ''') Conf('initial cycle point constraints', VDR.V_STRING_LIST, desc=''' Rules to allow only some initial datetime cycle points. @@ -599,7 +605,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): {REPLACES}``[scheduling]hold after point``. ''') - Conf('stop after cycle point', VDR.V_CYCLE_POINT, desc=''' + Conf('stop after cycle point', VDR.V_CYCLE_POINT_WITH_OFFSETS, desc=''' Shut down the workflow after all tasks pass this cycle point. The stop cycle point can be overridden on the command line using @@ -612,7 +618,18 @@ def get_script_common_text(this: str, example: Optional[str] = None): choosing not to run that part of the graph. You can play the workflow and continue. + Examples: + + - ``2000`` - Shorthand for ``2000-01-01T00:00``. + - ``+P1D`` - The initial cycle point plus one day. + - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day. + .. versionadded:: 8.0.0 + + .. versionchanged:: 8.3.0 + + This now supports offsets (e.g. ``+P1D``) in the same way the + :cylc:conf:`[..]final cycle point` does. ''') Conf('cycling mode', VDR.V_STRING, Calendar.MODE_GREGORIAN, options=list(Calendar.MODES) + ['integer'], desc=''' diff --git a/cylc/flow/config.py b/cylc/flow/config.py index d80456266bf..76c8b4e1980 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -851,9 +851,16 @@ def process_stop_cycle_point(self) -> None: if stopcp_str is None: stopcp_str = self.cfg['scheduling']['stop after cycle point'] - if stopcp_str is not None: - self.stop_point = get_point(stopcp_str).standardise() - if self.final_point and (self.stop_point > self.final_point): + if stopcp_str: + self.stop_point = get_point_relative( + stopcp_str, + self.initial_point, + ).standardise() + if ( + self.final_point is not None + and self.stop_point is not None + and self.stop_point > self.final_point + ): LOG.warning( f"Stop cycle point '{self.stop_point}' will have no " "effect as it is after the final cycle " diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py index 4ef66e19862..6e0e604d0c5 100644 --- a/cylc/flow/parsec/validate.py +++ b/cylc/flow/parsec/validate.py @@ -657,6 +657,7 @@ class CylcConfigValidator(ParsecValidator): V_CYCLE_POINT = 'V_CYCLE_POINT' V_CYCLE_POINT_FORMAT = 'V_CYCLE_POINT_FORMAT' V_CYCLE_POINT_TIME_ZONE = 'V_CYCLE_POINT_TIME_ZONE' + V_CYCLE_POINT_WITH_OFFSETS = 'V_CYCLE_POINT_WITH_OFFSETS' V_INTERVAL = 'V_INTERVAL' V_INTERVAL_LIST = 'V_INTERVAL_LIST' V_PARAMETER_LIST = 'V_PARAMETER_LIST' @@ -699,6 +700,30 @@ class CylcConfigValidator(ParsecValidator): '-0830': 'UTC minus 8 hours and 30 minutes.' } ), + V_CYCLE_POINT_WITH_OFFSETS: ( + 'cycle point with support for offsets', + 'An integer of date-time cycle point.', + { + '1': 'An integer cycle point.', + '1 +P5': ( + 'An integer cycle point with offsets.', + ' (this evaluates as ``6``)' + ), + '+P5': ( + 'An integer cycle point offset.' + ' This offset is added to the initial cycle point' + ), + '2000-01-01T00:00Z': 'A date-time cycle point.', + '2000-01-01T00:00Z +P1D +P1M': ( + 'A date-time cycle point with offsets.' + ' (this evaluates as ``2000-02-02T00:00Z``' + ), + '2000-01-01T00:00Z +P1D': ( + 'A date-time offset.' + ' This offset is added to the initial cycle point' + ), + } + ), V_INTERVAL: ( 'time interval', 'An ISO8601 duration.', @@ -751,6 +776,9 @@ def __init__(self): self.V_CYCLE_POINT: self.coerce_cycle_point, self.V_CYCLE_POINT_FORMAT: self.coerce_cycle_point_format, self.V_CYCLE_POINT_TIME_ZONE: self.coerce_cycle_point_time_zone, + # NOTE: This type exists for documentation reasons + # it doesn't actually process offsets, that happens later + self.V_CYCLE_POINT_WITH_OFFSETS: self.coerce_str, self.V_INTERVAL: self.coerce_interval, self.V_INTERVAL_LIST: self.coerce_interval_list, self.V_PARAMETER_LIST: self.coerce_parameter_list, diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index 0e02f9384d9..ec3ba6f0c58 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -993,6 +993,7 @@ def command_stop( point = TaskID.get_standardised_point(cycle_point) if point is not None and self.pool.set_stop_point(point): self.options.stopcp = str(point) + self.config.stop_point = point self.workflow_db_mgr.put_workflow_stop_cycle_point( self.options.stopcp) elif clock_time is not None: diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py index a13ba34d3cb..daa4d8fe00e 100644 --- a/cylc/flow/task_pool.py +++ b/cylc/flow/task_pool.py @@ -1026,7 +1026,7 @@ def set_stop_point(self, stop_point: 'PointBase') -> bool: LOG.info(f"Stop point unchanged: {stop_point}") return False - LOG.info("Setting stop point: {stop_point}") + LOG.info(f"Setting stop point: {stop_point}") self.stop_point = stop_point if ( diff --git a/tests/functional/cylc-play/09-invalid-cp-opt.t b/tests/functional/cylc-play/09-invalid-cp-opt.t index 05fa6d2a2d6..ae633d5e75c 100644 --- a/tests/functional/cylc-play/09-invalid-cp-opt.t +++ b/tests/functional/cylc-play/09-invalid-cp-opt.t @@ -37,7 +37,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_fail "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --no-detach --stopcp='potato' -grep_ok "ERROR - Workflow shutting down .* potato" "${TEST_NAME_BASE}-run.stderr" +grep_ok "ERROR - Workflow shutting down .*potato" "${TEST_NAME_BASE}-run.stderr" # Check that we haven't got a database exists_ok "${WORKFLOW_RUN_DIR}/.service" diff --git a/tests/integration/test_stop_after_cycle_point.py b/tests/integration/test_stop_after_cycle_point.py new file mode 100644 index 00000000000..663e03649f8 --- /dev/null +++ b/tests/integration/test_stop_after_cycle_point.py @@ -0,0 +1,132 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test logic pertaining to the stop after cycle points. + +This may be defined in different ways: +* In the workflow configuration. +* On the command line. +* Or loaded from the database. + +When the workflow hits the "stop after" point, it should be wiped (i.e. set +to None). +""" + +from typing import Optional + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.id import Tokens +from cylc.flow.workflow_status import StopMode + + +async def test_stop_after_cycle_point( + flow, + scheduler, + run, + reflog, + complete, +): + """Test the stop after cycle point. + + This ensures: + * The stop after point gets loaded from the config. + * The workflow stops when it hits this point. + * The point gets wiped when the workflow hits this point. + * The point is stored/retrieved from the DB as appropriate. + + """ + async def stops_after_cycle(schd) -> Optional[str]: + """Run the workflow until it stops and return the cycle point.""" + triggers = reflog(schd) + await complete(schd, timeout=2) + assert len(triggers) == 1 # only one task (i.e. cycle) should be run + return Tokens(list(triggers)[0][0], relative=True)['cycle'] + + def get_db_value(schd) -> Optional[str]: + """Return the cycle point value stored in the DB.""" + with schd.workflow_db_mgr.get_pri_dao() as pri_dao: + return dict(pri_dao.select_workflow_params())['stopcp'] + + config = { + 'scheduling': { + 'cycling mode': 'integer', + 'initial cycle point': '1', + 'stop after cycle point': '1', + 'graph': { + 'P1': 'a[-P1] => a', + }, + }, + } + id_ = flow(config) + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the cycle point should be loaded from the workflow configuration + assert schd.config.stop_point == IntegerPoint('1') + + # this value should *not* be written to the database + assert get_db_value(schd) is None + + # the workflow should stop after cycle 1 + assert await stops_after_cycle(schd) == '1' + + # change the configured cycle point to "2" + config['scheduling']['stop after cycle point'] = '2' + id_ = flow(config, id_=id_) + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the cycle point should be reloaded from the workflow configuration + assert schd.config.stop_point == IntegerPoint('2') + + # this value should not be written to the database + assert get_db_value(schd) is None + + # the workflow should stop after cycle 2 + assert await stops_after_cycle(schd) == '2' + + # override the configured value via the CLI option + schd = scheduler(id_, paused_start=False, **{'stopcp': '3'}) + async with run(schd): + # the CLI should take precedence over the config + assert schd.config.stop_point == IntegerPoint('3') + + # this value *should* be written to the database + assert get_db_value(schd) == '3' + + # the workflow should stop after cycle 3 + assert await stops_after_cycle(schd) == '3' + + # once the workflow hits this point, it should get cleared + assert get_db_value(schd) is None + + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the workflow should fall back to the configured value + assert schd.config.stop_point == IntegerPoint('2') + + # override this value whilst the workflow is running + schd.command_stop( + cycle_point=IntegerPoint('4'), + mode=StopMode.REQUEST_CLEAN, + ) + assert schd.config.stop_point == IntegerPoint('4') + + # the new *should* be written to the database + assert get_db_value(schd) == '4' + + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the workflow should stop after cycle 4 + assert await stops_after_cycle(schd) == '4' diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 67f5f0f559a..57b06433181 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -114,7 +114,7 @@ def test_xfunction_imports( """ flow_file.write_text(flow_config) workflow_config = WorkflowConfig( - workflow="name_a_tree", fpath=flow_file, options=Mock(spec=[]), + workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(spec=[]), xtrigger_mgr=xtrigger_mgr ) assert 'tree' in workflow_config.xtrigger_mgr.functx_map @@ -148,7 +148,7 @@ def test_xfunction_import_error(self, mock_glbl_cfg, tmp_path): WorkflowConfig( workflow="caiman_workflow", fpath=flow_file, - options=Mock(spec=[]) + options=SimpleNamespace(spec=[]) ) assert "not found" in str(excinfo.value) @@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path): flow_file.write_text(flow_config) with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig(workflow="capybara_workflow", fpath=flow_file, - options=Mock(spec=[])) + options=SimpleNamespace(spec=[])) assert "not found" in str(excinfo.value) def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): @@ -211,7 +211,7 @@ def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): WorkflowConfig( workflow="workflow_with_not_callable", fpath=flow_file, - options=Mock(spec=[]) + options=SimpleNamespace(spec=[]) ) assert "callable" in str(excinfo.value) @@ -378,14 +378,16 @@ def test_process_icp( expected_err: Exception class expected to be raised plus the message. """ set_cycling_type(cycling_type, time_zone="+0530") - mocked_config = Mock(cycling_type=cycling_type) - mocked_config.cfg = { - 'scheduling': { - 'initial cycle point constraints': [], - **scheduling_cfg - } - } - mocked_config.options.icp = None + mocked_config = SimpleNamespace( + cycling_type=cycling_type, + options=SimpleNamespace(icp=None), + cfg={ + 'scheduling': { + 'initial cycle point constraints': [], + **scheduling_cfg + }, + }, + ) monkeypatch.setattr('cylc.flow.config.get_current_time_string', lambda: '20050102T0615+0530') @@ -461,9 +463,10 @@ def test_process_startcp( expected_err: Expected exception. """ set_cycling_type(ISO8601_CYCLING_TYPE, time_zone="+0530") - mocked_config = Mock(initial_point='18990501T0000+0530') - mocked_config.options.startcp = startcp - mocked_config.options.starttask = starttask + mocked_config = SimpleNamespace( + initial_point='18990501T0000+0530', + options=SimpleNamespace(startcp=startcp, starttask=starttask), + ) monkeypatch.setattr('cylc.flow.config.get_current_time_string', lambda: '20050102T0615+0530') if expected_err is not None: @@ -662,17 +665,20 @@ def test_process_fcp( expected_err: Exception class expected to be raised plus the message. """ set_cycling_type(cycling_type, time_zone='+0530') - mocked_config = Mock(cycling_type=cycling_type) - mocked_config.cfg = { - 'scheduling': { - 'final cycle point constraints': [], - **scheduling_cfg - } - } - mocked_config.initial_point = loader.get_point( - scheduling_cfg['initial cycle point']).standardise() - mocked_config.final_point = None - mocked_config.options.fcp = options_fcp + mocked_config = SimpleNamespace( + cycling_type=cycling_type, + cfg={ + 'scheduling': { + 'final cycle point constraints': [], + **scheduling_cfg, + }, + }, + initial_point=loader.get_point( + scheduling_cfg['initial cycle point'] + ).standardise(), + final_point = None, + options=SimpleNamespace(fcp=options_fcp), + ) if expected_err: err, msg = expected_err @@ -692,24 +698,28 @@ def test_process_fcp( [ pytest.param( None, None, None, None, None, - id="No stopcp" + id="no-stopcp" ), pytest.param( '1993', None, '1993', None, None, - id="From config by default" + id="stopcp" ), pytest.param( '1993', '1066', '1066', '1066', None, - id="From options" + id="stop-cp-and-cli-option" ), pytest.param( '1993', 'reload', '1993', None, None, - id="From cfg if --stopcp=reload on restart" + id="stop-cp-and-cli-reload-option" ), pytest.param( '3000', None, None, None, "will have no effect as it is after the final cycle point", - id="stopcp > fcp" + id="stopcp-beyond-fcp" + ), + pytest.param( + '+P12Y -P2Y', None, '2000', None, None, + id="stopcp-relative-to-icp" ), ] ) @@ -734,12 +744,13 @@ def test_process_stop_cycle_point( set_cycling_type(ISO8601_CYCLING_TYPE, dump_format='CCYY') caplog.set_level(logging.WARNING, CYLC_LOG) fcp = loader.get_point('2012').standardise() - mock_config = Mock( + mock_config = SimpleNamespace( cfg={ 'scheduling': { 'stop after cycle point': cfg_stopcp } }, + initial_point=ISO8601Point('1990'), final_point=fcp, stop_point=None, options=RunOptions(stopcp=options_stopcp), @@ -887,7 +898,7 @@ def test_prelim_process_graph( processing. expected_err: Exception class expected to be raised plus the message. """ - mock_config = Mock(cfg={ + mock_config = SimpleNamespace(cfg={ 'scheduling': scheduling_cfg }) @@ -913,13 +924,15 @@ def _test(utc_mode, expected, expected_warnings=0): UTC mode = {utc_mode['glbl']} ''' ) - mock_config = Mock() - mock_config.cfg = { - 'scheduler': { - 'UTC mode': utc_mode['workflow'] - } - } - mock_config.options.utc_mode = utc_mode['stored'] + mock_config = SimpleNamespace( + cfg={ + 'scheduler': { + 'UTC mode': utc_mode['workflow'] + } + }, + options=SimpleNamespace(utc_mode=utc_mode['stored']), + ) + WorkflowConfig.process_utc_mode(mock_config) assert mock_config.cfg['scheduler']['UTC mode'] is expected assert get_utc_mode() is expected @@ -963,13 +976,15 @@ def test_cycle_point_tz(caplog, monkeypatch): def _test(cp_tz, utc_mode, expected, expected_warnings=0): set_utc_mode(utc_mode) - mock_config = Mock() - mock_config.cfg = { - 'scheduler': { - 'cycle point time zone': cp_tz['workflow'] - } - } - mock_config.options.cycle_point_tz = cp_tz['stored'] + mock_config = SimpleNamespace( + cfg={ + 'scheduler': { + 'cycle point time zone': cp_tz['workflow'], + }, + }, + options=SimpleNamespace(cycle_point_tz=cp_tz['stored']), + ) + WorkflowConfig.process_cycle_point_tz(mock_config) assert mock_config.cfg['scheduler'][ 'cycle point time zone'] == expected @@ -1148,7 +1163,7 @@ def test_process_runahead_limit( set_cycling_type: Callable ) -> None: set_cycling_type(cycling_type) - mock_config = Mock(cycling_type=cycling_type) + mock_config = SimpleNamespace(cycling_type=cycling_type) mock_config.cfg = { 'scheduling': { 'runahead limit': runahead_limit @@ -1170,7 +1185,7 @@ def test_check_circular(opt, monkeypatch, caplog, tmp_flow_config): # ----- Setup ----- caplog.set_level(logging.WARNING, CYLC_LOG) - options = Mock(spec=[], is_validate=True) + options = SimpleNamespace(spec=[], is_validate=True) if opt: setattr(options, opt, True) @@ -1739,7 +1754,7 @@ def test_cylc_env_at_parsing( # Parse the workflow config then check the environment. WorkflowConfig( - workflow="name", fpath=flow_file, options=Mock(spec=[]), + workflow="name", fpath=flow_file, options=SimpleNamespace(spec=[]), run_dir=run_dir ) From a83df172a1a77efdc0938a777f66828fb1db6c6c Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 30 Jan 2024 17:06:28 +0000 Subject: [PATCH 06/20] doc: correct spelling --- cylc/flow/etc/examples/event-driven-cycling/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cylc/flow/etc/examples/event-driven-cycling/README.rst b/cylc/flow/etc/examples/event-driven-cycling/README.rst index 4fa4ddde241..d1e112374f1 100644 --- a/cylc/flow/etc/examples/event-driven-cycling/README.rst +++ b/cylc/flow/etc/examples/event-driven-cycling/README.rst @@ -8,7 +8,7 @@ Cylc is good at orchestrating tasks to a schedule, e.g: But sometimes the things you want to run don't have a schedule. This example uses ``cylc ext-trigger`` to establish a pattern where Cylc waits -for an external signal and starts a new cycle every time a signal is recieved. +for an external signal and starts a new cycle every time a signal is received. The signal can carry data using the ext-trigger ID, this example sets the ID as a file path containing some data that we want to make available to the tasks From 2bcad6bd8ef6863e4cb19127a8378f2cdc6b80ef Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Wed, 31 Jan 2024 12:42:41 +0000 Subject: [PATCH 07/20] Apply suggestions from code review Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> --- cylc/flow/cfgspec/workflow.py | 4 ++-- cylc/flow/parsec/validate.py | 16 ++++++---------- tests/unit/test_config.py | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index 7ae7cf014b2..6a951ae02fb 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -552,7 +552,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): - ``2000`` - Shorthand for ``2000-01-01T00:00``. - ``+P1D`` - The initial cycle point plus one day. - - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day. + - ``2000 +P1D +P1Y`` - The year ``2000`` plus one day and one year. ''') Conf('initial cycle point constraints', VDR.V_STRING_LIST, desc=''' Rules to allow only some initial datetime cycle points. @@ -622,7 +622,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): - ``2000`` - Shorthand for ``2000-01-01T00:00``. - ``+P1D`` - The initial cycle point plus one day. - - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day. + - ``2000 +P1D +P1Y`` - The year ``2000`` plus one day and one year. .. versionadded:: 8.0.0 diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py index 6e0e604d0c5..2309a1cc4aa 100644 --- a/cylc/flow/parsec/validate.py +++ b/cylc/flow/parsec/validate.py @@ -702,25 +702,21 @@ class CylcConfigValidator(ParsecValidator): ), V_CYCLE_POINT_WITH_OFFSETS: ( 'cycle point with support for offsets', - 'An integer of date-time cycle point.', + 'An integer or date-time cycle point, with optional offset(s).', { '1': 'An integer cycle point.', '1 +P5': ( - 'An integer cycle point with offsets.', - ' (this evaluates as ``6``)' + 'An integer cycle point with an offset' + ' (this evaluates as ``6``).' ), '+P5': ( 'An integer cycle point offset.' ' This offset is added to the initial cycle point' ), '2000-01-01T00:00Z': 'A date-time cycle point.', - '2000-01-01T00:00Z +P1D +P1M': ( - 'A date-time cycle point with offsets.' - ' (this evaluates as ``2000-02-02T00:00Z``' - ), - '2000-01-01T00:00Z +P1D': ( - 'A date-time offset.' - ' This offset is added to the initial cycle point' + '2000-02-29T00:00Z +P1D +P1M': ( + 'A date-time cycle point with offsets' + ' (this evaluates as ``2000-04-01T00:00Z``).' ), } ), diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 57b06433181..fd07b2c8c18 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -114,7 +114,7 @@ def test_xfunction_imports( """ flow_file.write_text(flow_config) workflow_config = WorkflowConfig( - workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(spec=[]), + workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(), xtrigger_mgr=xtrigger_mgr ) assert 'tree' in workflow_config.xtrigger_mgr.functx_map From a8988df0591bf77706547c427e215aa20b3291b3 Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Wed, 31 Jan 2024 13:05:12 +0000 Subject: [PATCH 08/20] Tidy (#5886) --- cylc/flow/network/schema.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index f9ed95c7158..d076882ae73 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -349,8 +349,6 @@ async def get_nodes_all(root, info, **args): _, field_ids = process_resolver_info(root, info, args) - if hasattr(args, 'id'): - args['ids'] = [args.get('id')] if field_ids: if isinstance(field_ids, str): field_ids = [field_ids] @@ -377,10 +375,8 @@ async def get_nodes_all(root, info, **args): else: # live objects can be represented by a universal ID args[arg] = [Tokens(n_id, relative=True) for n_id in args[arg]] - args['workflows'] = [ - Tokens(w_id) for w_id in args['workflows']] - args['exworkflows'] = [ - Tokens(w_id) for w_id in args['exworkflows']] + for arg in ('workflows', 'exworkflows'): + args[arg] = [Tokens(w_id) for w_id in args[arg]] resolvers = info.context.get('resolvers') return await resolvers.get_nodes_all(node_type, args) From 85492ae3290e024cf65994f510c2aaa61abbf638 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:49:46 +0000 Subject: [PATCH 09/20] build(deps): bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test_fast.yml | 2 +- .github/workflows/test_functional.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml index 66f08ce61ac..5776185557e 100644 --- a/.github/workflows/test_fast.yml +++ b/.github/workflows/test_fast.yml @@ -140,7 +140,7 @@ jobs: uses: actions/download-artifact@v4 - name: Codecov upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.workflow }} flags: fast-tests diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml index ed411c2eae8..f54f3b3d710 100644 --- a/.github/workflows/test_functional.yml +++ b/.github/workflows/test_functional.yml @@ -316,7 +316,7 @@ jobs: uses: actions/download-artifact@v4 - name: Codecov upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.workflow }} flags: functional-tests From 38a4e152a8eabea5a0243d6eeb265983c774c9d4 Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Thu, 1 Feb 2024 14:53:06 +0000 Subject: [PATCH 10/20] cylc lint: deprecate `pyproject.toml` section `cylc-lint` in favour of `tool.cylc.lint` --- cylc/flow/scripts/lint.py | 67 ++++++++++------ tests/functional/cylc-lint/01.lint-toml.t | 55 +++++++------- tests/unit/scripts/test_lint.py | 93 ++++++++++++++++------- 3 files changed, 136 insertions(+), 79 deletions(-) diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index f038d569d78..a0f52da9f33 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -22,8 +22,6 @@ # NOTE: docstring needed for `cylc help all` output # (if editing check this still comes out as expected) -LINT_SECTIONS = ['cylc-lint', 'cylclint', 'cylc_lint'] - COP_DOC = """cylc lint [OPTIONS] ARGS Check .cylc and .rc files for code style, deprecated syntax and other issues. @@ -44,12 +42,12 @@ """ TOMLDOC = """ -pyproject.toml configuration:{} - [cylc-lint] # any of {} - ignore = ['S001', 'S002'] # List of rules to ignore - exclude = ['etc/foo.cylc'] # List of files to ignore - rulesets = ['style', '728'] # Sets default rulesets to check - max-line-length = 130 # Max line length for linting +pyproject.toml configuration: + [tool.cylc.lint] + ignore = ['S001', 'S002'] # List of rules to ignore + exclude = ['etc/foo.cylc'] # List of files to ignore + rulesets = ['style', '728'] # Sets default rulesets to check + max-line-length = 130 # Max line length for linting """ from colorama import Fore import functools @@ -77,6 +75,8 @@ from cylc.flow import LOG from cylc.flow.exceptions import CylcError +import cylc.flow.flags +from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( CylcOptionParser as COP, WORKFLOW_ID_OR_PATH_ARG_DOC @@ -90,6 +90,20 @@ if TYPE_CHECKING: from optparse import Values +LINT_TABLE = ['tool', 'cylc', 'lint'] +LINT_SECTION = '.'.join(LINT_TABLE) + +# BACK COMPAT: DEPR_LINT_SECTION +# url: +# https://github.com/cylc/cylc-flow/issues/5811 +# from: +# 8.1.0 +# to: +# 8.3.0 +# remove at: +# 8.4.0 ? +DEPR_LINT_SECTION = 'cylc-lint' + DEPRECATED_ENV_VARS = { 'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST', 'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER', @@ -758,7 +772,7 @@ def validate_toml_items(tomldata): if key not in EXTRA_TOML_VALIDATION.keys(): raise CylcError( f'Only {[*EXTRA_TOML_VALIDATION.keys()]} ' - f'allowed as toml sections but you used {key}' + f'allowed as toml sections but you used "{key}"' ) if key != 'max-line-length': # Item should be a list... @@ -780,26 +794,35 @@ def validate_toml_items(tomldata): return True -def get_pyproject_toml(dir_): +def get_pyproject_toml(dir_: Path) -> Dict[str, Any]: """if a pyproject.toml file is present open it and return settings. """ - keys = ['rulesets', 'ignore', 'exclude', 'max-line-length'] - tomlfile = Path(dir_ / 'pyproject.toml') - tomldata = {} + tomlfile = dir_ / 'pyproject.toml' + tomldata = { + 'rulesets': [], + 'ignore': [], + 'exclude': [], + 'max-line-length': None, + } if tomlfile.is_file(): try: loadeddata = toml_loads(tomlfile.read_text()) except TOMLDecodeError as exc: raise CylcError(f'pyproject.toml did not load: {exc}') - if any( - i in loadeddata for i in LINT_SECTIONS - ): - for key in keys: - tomldata[key] = loadeddata.get('cylc-lint').get(key, []) - validate_toml_items(tomldata) - if not tomldata: - tomldata = {key: [] for key in keys} + _tool, _cylc, _lint = LINT_TABLE + try: + data = loadeddata[_tool][_cylc][_lint] + except KeyError: + if DEPR_LINT_SECTION in loadeddata: + LOG.warning( + f"The [{DEPR_LINT_SECTION}] section in pyproject.toml is " + f"deprecated. Use [{LINT_SECTION}] instead." + ) + data = loadeddata.get(DEPR_LINT_SECTION, {}) + tomldata.update(data) + validate_toml_items(tomldata) + return tomldata @@ -1318,7 +1341,7 @@ def get_option_parser() -> COP: ( COP_DOC + NOQA.replace('``', '"') - + TOMLDOC.format('', str(LINT_SECTIONS)) + + TOMLDOC ), argdoc=[ COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC) diff --git a/tests/functional/cylc-lint/01.lint-toml.t b/tests/functional/cylc-lint/01.lint-toml.t index 183b69beaed..fbdd243e859 100644 --- a/tests/functional/cylc-lint/01.lint-toml.t +++ b/tests/functional/cylc-lint/01.lint-toml.t @@ -58,21 +58,20 @@ named_grep_ok "it returns a 728 upgrade code" "^\[U" "${TESTOUT}" # Add a pyproject.toml file cat > pyproject.toml <<__HERE__ -[cylc-lint] - # Check against these rules - rulesets = [ - "style" - ] - # do not check for these errors - ignore = [ - "S004" - ] - # do not lint files matching - # these globs: - exclude = [ - "sites/*.cylc", - ] - +[tool.cylc.lint] +# Check against these rules +rulesets = [ + "style" +] +# do not check for these errors +ignore = [ + "S004" +] +# do not lint files matching +# these globs: +exclude = [ + "sites/*.cylc", +] __HERE__ # Test that results are different: @@ -102,19 +101,19 @@ named_grep_ok "${TEST_NAME}-line-too-long-message" \ TEST_NAME="it_does_not_fail_if_max-line-length_set_but_ignored" cat > pyproject.toml <<__HERE__ -[cylc-lint] - # Check against these rules - rulesets = [ - "style" - ] - # do not check for these errors - ignore = [ - "${LINE_LEN_NO}" - ] - exclude = [ - "sites/*.cylc", - ] - max-line-length = 1 +[tool.cylc.lint] +# Check against these rules +rulesets = [ + "style" +] +# do not check for these errors +ignore = [ + "${LINE_LEN_NO}" +] +exclude = [ + "sites/*.cylc", +] +max-line-length = 1 __HERE__ run_ok "${TEST_NAME}" cylc lint grep_ok "rules and found no issues" "${TEST_NAME}.stdout" diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py index f3009a6ddb3..f5f4bb8cc98 100644 --- a/tests/unit/scripts/test_lint.py +++ b/tests/unit/scripts/test_lint.py @@ -16,15 +16,18 @@ # along with this program. If not, see . """Tests `cylc lint` CLI Utility.""" +import logging from pathlib import Path from pprint import pformat import re +from textwrap import dedent from types import SimpleNamespace import pytest from pytest import param from cylc.flow.scripts.lint import ( + LINT_SECTION, MANUAL_DEPRECATIONS, get_cylc_files, get_pyproject_toml, @@ -230,7 +233,6 @@ def assert_contains(items, contains, instances=None): 16: 3, } - @pytest.mark.parametrize( # 11 won't be tested because there is no jinja2 shebang 'number', set(range(1, len(MANUAL_DEPRECATIONS) + 1)) - {11} @@ -473,52 +475,85 @@ def test_get_upg_info(fixture_get_deprecations, findme): @pytest.mark.parametrize( - 'expect', + 'settings, expected', [ - param({ - 'rulesets': ['style'], - 'ignore': ['S004'], - 'exclude': ['sites/*.cylc']}, - id="it returns what we want" + param( + """ + rulesets = ['style'] + ignore = ['S004'] + exclude = ['sites/*.cylc'] + """, + { + 'rulesets': ['style'], + 'ignore': ['S004'], + 'exclude': ['sites/*.cylc'], + 'max-line-length': None, + }, + id="returns what we want" ), - param({ - 'northgate': ['sites/*.cylc'], - 'mons-meg': 42}, - id="it only returns requested sections" + param( + """ + northgate = ['sites/*.cylc'] + mons-meg = 42 + """, + (CylcError, ".*northgate"), + id="invalid settings fail validation" ), - param({ - 'max-line-length': 22}, - id='it sets max line length' + param( + "max-line-length = 22", + { + 'exclude': [], + 'ignore': [], + 'rulesets': [], + 'max-line-length': 22, + }, + id='sets max line length' ) ] ) -def test_get_pyproject_toml(tmp_path, expect): +def test_get_pyproject_toml(tmp_path, settings, expected): """It returns only the lists we want from the toml file.""" - tomlcontent = "[cylc-lint]" - permitted_keys = ['rulesets', 'ignore', 'exclude', 'max-line-length'] - - for section, value in expect.items(): - tomlcontent += f'\n{section} = {value}' + tomlcontent = "[tool.cylc.lint]\n" + dedent(settings) (tmp_path / 'pyproject.toml').write_text(tomlcontent) - tomldata = get_pyproject_toml(tmp_path) - control = {} - for key in permitted_keys: - control[key] = expect.get(key, []) - assert tomldata == control + if isinstance(expected, tuple): + exc, match = expected + with pytest.raises(exc, match=match): + get_pyproject_toml(tmp_path) + else: + assert get_pyproject_toml(tmp_path) == expected -@pytest.mark.parametrize('tomlfile', [None, '', '[cylc-lint]']) +@pytest.mark.parametrize( + 'tomlfile', + [None, '', '[tool.cylc.lint]', '[cylc-lint]'] +) def test_get_pyproject_toml_returns_blank(tomlfile, tmp_path): if tomlfile is not None: tfile = (tmp_path / 'pyproject.toml') tfile.write_text(tomlfile) - expect = {k: [] for k in { - 'exclude', 'ignore', 'max-line-length', 'rulesets' - }} + expect = { + 'exclude': [], 'ignore': [], 'max-line-length': None, 'rulesets': [] + } assert get_pyproject_toml(tmp_path) == expect +def test_get_pyproject_toml__depr( + tmp_path: Path, caplog: pytest.LogCaptureFixture +): + """It warns if the section is deprecated.""" + file = tmp_path / 'pyproject.toml' + caplog.set_level(logging.WARNING) + + file.write_text(f'[{LINT_SECTION}]\nmax-line-length=14') + assert get_pyproject_toml(tmp_path)['max-line-length'] == 14 + assert not caplog.text + + file.write_text('[cylc-lint]\nmax-line-length=17') + assert get_pyproject_toml(tmp_path)['max-line-length'] == 17 + assert "[cylc-lint] section in pyproject.toml is deprecated" in caplog.text + + @pytest.mark.parametrize( 'input_, error', [ From 810f3befa6c639b6a70ced3f32971459a9a5b755 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Fri, 2 Feb 2024 14:49:02 +0000 Subject: [PATCH 11/20] Apply suggestions from code review Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> --- tests/unit/test_config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index fd07b2c8c18..629ca144739 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -148,7 +148,7 @@ def test_xfunction_import_error(self, mock_glbl_cfg, tmp_path): WorkflowConfig( workflow="caiman_workflow", fpath=flow_file, - options=SimpleNamespace(spec=[]) + options=SimpleNamespace() ) assert "not found" in str(excinfo.value) @@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path): flow_file.write_text(flow_config) with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig(workflow="capybara_workflow", fpath=flow_file, - options=SimpleNamespace(spec=[])) + options=SimpleNamespace() assert "not found" in str(excinfo.value) def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): @@ -211,7 +211,7 @@ def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): WorkflowConfig( workflow="workflow_with_not_callable", fpath=flow_file, - options=SimpleNamespace(spec=[]) + options=SimpleNamespace() ) assert "callable" in str(excinfo.value) @@ -1185,7 +1185,7 @@ def test_check_circular(opt, monkeypatch, caplog, tmp_flow_config): # ----- Setup ----- caplog.set_level(logging.WARNING, CYLC_LOG) - options = SimpleNamespace(spec=[], is_validate=True) + options = SimpleNamespace(is_validate=True) if opt: setattr(options, opt, True) @@ -1754,7 +1754,7 @@ def test_cylc_env_at_parsing( # Parse the workflow config then check the environment. WorkflowConfig( - workflow="name", fpath=flow_file, options=SimpleNamespace(spec=[]), + workflow="name", fpath=flow_file, options=SimpleNamespace(), run_dir=run_dir ) From 28878195d06e729912342e3cf68859e9678d4c4d Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Fri, 2 Feb 2024 16:28:13 +0000 Subject: [PATCH 12/20] Update tests/unit/test_config.py Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> --- tests/unit/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 629ca144739..494dd98e62b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path): flow_file.write_text(flow_config) with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig(workflow="capybara_workflow", fpath=flow_file, - options=SimpleNamespace() + options=SimpleNamespace()) assert "not found" in str(excinfo.value) def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): From 57fe4ae7e1a65d82ac650c596a2f4fdb4b49abf1 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:59:48 +0000 Subject: [PATCH 13/20] Don't record satisfied wall clock triggers in the DB (#5923) Don't record satisfied xtriggers in the DB * Closes https://github.com/cylc/cylc-flow/issues/5911 * Test that no wall_clock xtriggers are saved to the DB for a retry. --------- Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Co-authored-by: Oliver Sanders --- cylc/flow/workflow_db_mgr.py | 7 +- tests/integration/conftest.py | 86 ++++++++++++++++++++++- tests/integration/test_workflow_db_mgr.py | 66 +++++++++++++++-- 3 files changed, 147 insertions(+), 12 deletions(-) diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py index 7d6968b137c..354fff76b1e 100644 --- a/cylc/flow/workflow_db_mgr.py +++ b/cylc/flow/workflow_db_mgr.py @@ -405,9 +405,10 @@ def put_task_event_timers(self, task_events_mgr): def put_xtriggers(self, sat_xtrig): """Put statements to update external triggers table.""" for sig, res in sat_xtrig.items(): - self.db_inserts_map[self.TABLE_XTRIGGERS].append({ - "signature": sig, - "results": json.dumps(res)}) + if not sig.startswith('wall_clock('): + self.db_inserts_map[self.TABLE_XTRIGGERS].append({ + "signature": sig, + "results": json.dumps(res)}) def put_update_task_state(self, itask): """Update task_states table for current state of itask. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f2f24b09ab5..bd69bbaeed9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -20,11 +20,11 @@ from pathlib import Path import pytest from shutil import rmtree +from time import time from typing import List, TYPE_CHECKING, Set, Tuple, Union from cylc.flow.config import WorkflowConfig from cylc.flow.option_parsers import Options -from cylc.flow.network.client import WorkflowRuntimeClient from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.rundb import CylcWorkflowDAO from cylc.flow.scripts.validate import ValidateOptions @@ -34,6 +34,7 @@ ) from cylc.flow.wallclock import get_current_time_string from cylc.flow.workflow_files import infer_latest_run_from_id +from cylc.flow.workflow_status import StopMode from .utils import _rm_if_empty from .utils.flow_tools import ( @@ -47,6 +48,7 @@ if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler from cylc.flow.task_proxy import TaskProxy + from cylc.flow.network.client import WorkflowRuntimeClient InstallOpts = Options(install_gop()) @@ -323,7 +325,7 @@ def _inner( def gql_query(): """Execute a GraphQL query given a workflow runtime client.""" async def _gql_query( - client: WorkflowRuntimeClient, query_str: str + client: 'WorkflowRuntimeClient', query_str: str ) -> object: ret = await client.async_request( 'graphql', { @@ -473,3 +475,83 @@ def _inner(source, **kwargs): workflow_id = infer_latest_run_from_id(workflow_id) return workflow_id yield _inner + + +@pytest.fixture +def complete(): + """Wait for the workflow, or tasks within it to complete. + + Args: + schd: + The scheduler to await. + tokens_list: + If specified, this will wait for the tasks represented by these + tokens to be marked as completed by the task pool. + stop_mode: + If tokens_list is not provided, this will wait for the scheduler + to be shutdown with the specified mode (default = AUTO, i.e. + workflow completed normally). + timeout: + Max time to wait for the condition to be met. + + Note, if you need to increase this, you might want to rethink your + test. + + Note, use this timeout rather than wrapping the complete call with + async_timeout (handles shutdown logic more cleanly). + + """ + async def _complete( + schd, + *tokens_list, + stop_mode=StopMode.AUTO, + timeout=60, + ): + start_time = time() + tokens_list = [tokens.task for tokens in tokens_list] + + # capture task completion + remove_if_complete = schd.pool.remove_if_complete + + def _remove_if_complete(itask): + ret = remove_if_complete(itask) + if ret and itask.tokens.task in tokens_list: + tokens_list.remove(itask.tokens.task) + return ret + + schd.pool.remove_if_complete = _remove_if_complete + + # capture workflow shutdown + set_stop = schd._set_stop + has_shutdown = False + + def _set_stop(mode=None): + nonlocal has_shutdown, stop_mode + if mode == stop_mode: + has_shutdown = True + return set_stop(mode) + else: + set_stop(mode) + raise Exception(f'Workflow bailed with stop mode = {mode}') + + schd._set_stop = _set_stop + + # determine the completion condition + if tokens_list: + condition = lambda: bool(tokens_list) + else: + condition = lambda: bool(not has_shutdown) + + # wait for the condition to be met + while condition(): + # allow the main loop to advance + await asyncio.sleep(0) + if time() - start_time > timeout: + raise Exception( + f'Timeout waiting for {", ".join(map(str, tokens_list))}' + ) + + # restore regular shutdown logic + schd._set_stop = set_stop + + return _complete diff --git a/tests/integration/test_workflow_db_mgr.py b/tests/integration/test_workflow_db_mgr.py index fa15c9165b0..cf4fca7e064 100644 --- a/tests/integration/test_workflow_db_mgr.py +++ b/tests/integration/test_workflow_db_mgr.py @@ -14,10 +14,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio import pytest import sqlite3 +from typing import TYPE_CHECKING -from cylc.flow.scheduler import Scheduler +from cylc.flow.cycling.iso8601 import ISO8601Point + +if TYPE_CHECKING: + from cylc.flow.scheduler import Scheduler async def test_restart_number( @@ -29,7 +34,7 @@ async def test_restart_number( async def test(expected_restart_num: int, do_reload: bool = False): """(Re)start the workflow and check the restart number is as expected. """ - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) async with start(schd) as log: if do_reload: schd.command_reload_workflow() @@ -52,7 +57,7 @@ async def test(expected_restart_num: int, do_reload: bool = False): await test(expected_restart_num=3) -def db_remove_column(schd: Scheduler, table: str, column: str) -> None: +def db_remove_column(schd: 'Scheduler', table: str, column: str) -> None: """Remove a column from a scheduler DB table. ALTER TABLE DROP COLUMN is not supported by sqlite yet, so we have to copy @@ -82,7 +87,7 @@ async def test_db_upgrade_pre_803( id_ = flow(one_conf) # Run a scheduler to create a DB. - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) async with start(schd): assert ('n_restart', '0') in db_select(schd, False, 'workflow_params') @@ -90,7 +95,7 @@ async def test_db_upgrade_pre_803( db_remove_column(schd, "task_states", "is_manual_submit") db_remove_column(schd, "task_jobs", "flow_nums") - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) # Restart should fail due to the missing column. with pytest.raises(sqlite3.OperationalError): @@ -98,7 +103,7 @@ async def test_db_upgrade_pre_803( pass assert ('n_restart', '1') in db_select(schd, False, 'workflow_params') - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) # Run the DB upgrader for version 8.0.2 # (8.0.2 requires upgrade) @@ -117,7 +122,7 @@ async def test_workflow_param_rapid_toggle( https://github.com/cylc/cylc-flow/issues/5593 """ - schd: Scheduler = scheduler(flow(one_conf), paused_start=False) + schd: 'Scheduler' = scheduler(flow(one_conf), paused_start=False) async with run(schd): assert schd.is_paused is False schd.pause_workflow() @@ -127,3 +132,50 @@ async def test_workflow_param_rapid_toggle( w_params = dict(schd.workflow_db_mgr.pri_dao.select_workflow_params()) assert w_params['is_paused'] == '0' + + +async def test_record_only_non_clock_triggers( + flow, run, scheduler, complete, db_select +): + """Database does not record wall_clock xtriggers. + + https://github.com/cylc/cylc-flow/issues/5911 + + Includes: + - Not in DB: A normal wall clock xtrigger (wall_clock). + - In DB: An xrandom mis-labelled as wall_clock trigger DB). + - Not in DB: An execution retry xtrigger. + + @TODO: Refactor to use simulation mode to speedup after Simulation + mode upgrade bugfixes: This should speed this test up considerably. + """ + rawpoint = '1348' + id_ = flow({ + "scheduler": { + 'cycle point format': '%Y', + 'allow implicit tasks': True + }, + "scheduling": { + "initial cycle point": rawpoint, + "xtriggers": { + "another": "xrandom(100)", + "wall_clock": "xrandom(100, _=Not a real wall clock trigger)", + "real_wall_clock": "wall_clock()" + }, + "graph": { + "R1": """ + @another & @wall_clock & @real_wall_clock => foo + @real_wall_clock => bar + """ + } + }, + }) + schd = scheduler(id_, paused_start=False, run_mode='simulation') + + async with run(schd): + await complete(schd, timeout=20) + + # Assert that (only) the real clock trigger is not in the db: + assert db_select(schd, False, 'xtriggers', 'signature') == [ + ('xrandom(100)',), + ('xrandom(100, _=Not a real wall clock trigger)',)] From 28e68fec1ec8cc0a0148d2da3884f40464b2acca Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:08:53 +0000 Subject: [PATCH 14/20] Tidy --- cylc/flow/scripts/lint.py | 149 ++++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 72 deletions(-) diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index a0f52da9f33..0626a407f32 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -19,7 +19,7 @@ Checks code style, deprecated syntax and other issues. """ -# NOTE: docstring needed for `cylc help all` output +# NOTE: docstring needed for `cylc help all` output and docs # (if editing check this still comes out as expected) COP_DOC = """cylc lint [OPTIONS] ARGS @@ -88,6 +88,10 @@ from cylc.flow.terminal import cli_function if TYPE_CHECKING: + # BACK COMPAT: typing_extensions.Literal + # FROM: Python 3.7 + # TO: Python 3.8 + from typing_extensions import Literal from optparse import Values LINT_TABLE = ['tool', 'cylc', 'lint'] @@ -104,6 +108,11 @@ # 8.4.0 ? DEPR_LINT_SECTION = 'cylc-lint' +IGNORE = 'ignore' +EXCLUDE = 'exclude' +RULESETS = 'rulesets' +MAX_LINE_LENGTH = 'max-line-length' + DEPRECATED_ENV_VARS = { 'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST', 'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER', @@ -602,7 +611,9 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: }, 'U008': { 'short': 'Suicide triggers are not required at Cylc 8.', - 'url': '', + 'url': ( + 'https://cylc.github.io/cylc-doc/stable/html/7-to-8' + '/major-changes/suicide-triggers.html'), 'kwargs': True, FUNCTION: functools.partial( check_for_suicide_triggers, @@ -616,9 +627,7 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: }, 'U010': { 'short': 'rose suite-hook is deprecated at Rose 2,', - 'url': ( - 'https://cylc.github.io/cylc-doc/stable/html/7-to-8' - '/major-changes/suicide-triggers.html'), + 'url': '', FUNCTION: lambda line: 'rose suite-hook' in line, }, 'U011': { @@ -711,28 +720,34 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: list_wrapper, check=CHECK_FOR_OLD_VARS.findall), }, } -RULESETS = ['728', 'style', 'all'] +ALL_RULESETS = ['728', 'style', 'all'] EXTRA_TOML_VALIDATION = { - 'ignore': { + IGNORE: { lambda x: re.match(r'[A-Z]\d\d\d', x): '{item} not valid: Ignore codes should be in the form X001', lambda x: x in parse_checks(['728', 'style']): '{item} is a not a known linter code.' }, - 'rulesets': { - lambda item: item in RULESETS: + RULESETS: { + lambda item: item in ALL_RULESETS: '{item} not valid: Rulesets can be ' '\'728\', \'style\' or \'all\'.' }, - 'max-line-length': { + MAX_LINE_LENGTH: { lambda x: isinstance(x, int): 'max-line-length must be an integer.' }, # consider checking that item is file? - 'exclude': {} + EXCLUDE: {} } +def parse_ruleset_option(ruleset: str) -> List[str]: + if ruleset in {'all', ''}: + return ['728', 'style'] + return [ruleset] + + def get_url(check_meta: Dict) -> str: """Get URL from check data. @@ -774,7 +789,7 @@ def validate_toml_items(tomldata): f'Only {[*EXTRA_TOML_VALIDATION.keys()]} ' f'allowed as toml sections but you used "{key}"' ) - if key != 'max-line-length': + if key != MAX_LINE_LENGTH: # Item should be a list... if not isinstance(items, list): raise CylcError( @@ -798,11 +813,11 @@ def get_pyproject_toml(dir_: Path) -> Dict[str, Any]: """if a pyproject.toml file is present open it and return settings. """ tomlfile = dir_ / 'pyproject.toml' - tomldata = { - 'rulesets': [], - 'ignore': [], - 'exclude': [], - 'max-line-length': None, + tomldata: Dict[str, Union[List[str], int, None]] = { + RULESETS: [], + IGNORE: [], + EXCLUDE: [], + MAX_LINE_LENGTH: None, } if tomlfile.is_file(): try: @@ -838,20 +853,14 @@ def merge_cli_with_tomldata(target: Path, options: 'Values') -> Dict[str, Any]: _merge_cli_with_tomldata to keep the testing of file-system touching and pure logic separate. """ - ruleset_default = False - if options.linter == 'all': - options.linter = ['728', 'style'] - elif options.linter == '': - options.linter = ['728', 'style'] - ruleset_default = True - else: - options.linter = [options.linter] + ruleset_default = (options.ruleset == '') + options.ruleset = parse_ruleset_option(options.ruleset) tomlopts = get_pyproject_toml(target) return _merge_cli_with_tomldata( { - 'exclude': [], - 'ignore': options.ignores, - 'rulesets': options.linter + EXCLUDE: [], + IGNORE: options.ignores, + RULESETS: options.ruleset }, tomlopts, ruleset_default @@ -886,30 +895,30 @@ def _merge_cli_with_tomldata( >>> result['exclude'] ['*.bk'] """ - if isinstance(clidata['rulesets'][0], list): - clidata['rulesets'] = clidata['rulesets'][0] + if isinstance(clidata[RULESETS][0], list): + clidata[RULESETS] = clidata[RULESETS][0] output = {} # Combine 'ignore' sections: - output['ignore'] = sorted(set(clidata['ignore'] + tomldata['ignore'])) + output[IGNORE] = sorted(set(clidata[IGNORE] + tomldata[IGNORE])) - # Replace 'rulesets from toml with those from CLI if they exist: + # Replace 'rulesets' from toml with those from CLI if they exist: if override_cli_default_rules: - output['rulesets'] = ( - tomldata['rulesets'] if tomldata['rulesets'] - else clidata['rulesets'] + output[RULESETS] = ( + tomldata[RULESETS] if tomldata[RULESETS] + else clidata[RULESETS] ) else: - output['rulesets'] = ( - clidata['rulesets'] if clidata['rulesets'] - else tomldata['rulesets'] + output[RULESETS] = ( + clidata[RULESETS] if clidata[RULESETS] + else tomldata[RULESETS] ) # Return 'exclude' and 'max-line-length' for the tomldata: - output['exclude'] = tomldata['exclude'] - output['max-line-length'] = tomldata.get('max-line-length', None) + output[EXCLUDE] = tomldata[EXCLUDE] + output[MAX_LINE_LENGTH] = tomldata.get(MAX_LINE_LENGTH, None) return output @@ -1239,29 +1248,18 @@ def get_cylc_files( yield path -REFERENCE_TEMPLATES = { - 'section heading': '\n{title}\n{underline}\n', - 'issue heading': { - 'text': '\n{check}:\n {summary}\n {url}\n\n', - 'rst': '\n{url}_\n{underline}\n{summary}\n\n', - }, - 'auto gen message': ( - 'U998 and U999 represent automatically generated' - ' sets of deprecations and upgrades.' - ), -} - - -def get_reference(linter, output_type): +def get_reference(ruleset: str, output_type: 'Literal["text", "rst"]') -> str: """Fill out a template with all the issues Cylc Lint looks for. """ - if linter in {'all', ''}: - rulesets = ['728', 'style'] - else: - rulesets = [linter] - checks = parse_checks(rulesets, reference=True) + checks = parse_checks( + parse_ruleset_option(ruleset), + reference=True + ) - issue_heading_template = REFERENCE_TEMPLATES['issue heading'][output_type] + issue_heading_template = ( + '\n{url}_\n{underline}\n{summary}\n\n' if output_type == 'rst' else + '\n{check}:\n {summary}\n {url}\n\n' + ) output = '' current_checkset = '' for index, meta in checks.items(): @@ -1270,11 +1268,15 @@ def get_reference(linter, output_type): if meta['purpose'] != current_checkset: current_checkset = meta['purpose'] title = CHECKS_DESC[meta["purpose"]] - output += REFERENCE_TEMPLATES['section heading'].format( - title=title, underline="-" * len(title)) + output += '\n{title}\n{underline}\n'.format( + title=title, underline="-" * len(title) + ) if current_checkset == 'A': - output += REFERENCE_TEMPLATES['auto gen message'] + output += ( + 'U998 and U999 represent automatically generated' + ' sets of deprecations and upgrades.' + ) # Fill a template with info about the issue. if output_type == 'rst': @@ -1321,18 +1323,18 @@ def target_version_check( disabled thatr with --exit-zero. """ cylc8 = (target / 'flow.cylc').exists() - if not cylc8 and mergedopts['rulesets'] == ['728']: + if not cylc8 and mergedopts[RULESETS] == ['728']: LOG.error( f'{target} not a Cylc 8 workflow: ' 'Lint after renaming ' '"suite.rc" to "flow.cylc"' ) sys.exit(not quiet) - elif not cylc8 and '728' in mergedopts['rulesets']: - check_names = mergedopts['rulesets'] + elif not cylc8 and '728' in mergedopts[RULESETS]: + check_names = mergedopts[RULESETS] check_names.remove('728') else: - check_names = mergedopts['rulesets'] + check_names = mergedopts[RULESETS] return check_names @@ -1364,7 +1366,7 @@ def get_option_parser() -> COP: ), default='', choices=["728", "style", "all", ''], - dest='linter' + dest='ruleset' ) parser.add_option( '--list-codes', @@ -1399,8 +1401,11 @@ def get_option_parser() -> COP: @cli_function(get_option_parser) def main(parser: COP, options: 'Values', target=None) -> None: + if cylc.flow.flags.verbosity < 2: + set_timestamps(LOG, False) + if options.ref_mode: - print(get_reference(options.linter, 'text')) + print(get_reference(options.ruleset, 'text')) sys.exit(0) # If target not given assume we are looking at PWD: @@ -1426,13 +1431,13 @@ def main(parser: COP, options: 'Values', target=None) -> None: # Get the checks object. checks = parse_checks( check_names, - ignores=mergedopts['ignore'], - max_line_len=mergedopts['max-line-length'] + ignores=mergedopts[IGNORE], + max_line_len=mergedopts[MAX_LINE_LENGTH] ) # Check each file matching a pattern: counter: Dict[str, int] = {} - for file in get_cylc_files(target, mergedopts['exclude']): + for file in get_cylc_files(target, mergedopts[EXCLUDE]): LOG.debug(f'Checking {file}') check_cylc_file( file, From f301d547ae67a6946435da67a94b02863e04aa4b Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:26:43 +0000 Subject: [PATCH 15/20] Changelog --- changes.d/5956.break.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes.d/5956.break.md diff --git a/changes.d/5956.break.md b/changes.d/5956.break.md new file mode 100644 index 00000000000..642c805814d --- /dev/null +++ b/changes.d/5956.break.md @@ -0,0 +1 @@ +`cylc lint`: deprecated `[cylc-lint]` section in favour of `[tool.cylc.lint]` in `pyproject.toml` From b1822a607fe1080cee9106c99394ae02e8140cb6 Mon Sep 17 00:00:00 2001 From: Mark Dawson Date: Thu, 8 Feb 2024 10:11:09 +0000 Subject: [PATCH 16/20] added suite state xtrigger for back-compat (#5864) added suite state xtrigger for back commpat extended back compat a unit test added deprecation log message Replace icky unit test with functional test (#4) --------- Co-authored-by: Mark Dawson --- changes.d/5864.feat.md | 1 + cylc/flow/xtriggers/suite_state.py | 81 +++++++++++++++++++++ setup.cfg | 1 + tests/functional/xtriggers/04-suite_state.t | 50 +++++++++++++ tests/unit/xtriggers/test_workflow_state.py | 10 ++- 5 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 changes.d/5864.feat.md create mode 100644 cylc/flow/xtriggers/suite_state.py create mode 100644 tests/functional/xtriggers/04-suite_state.t diff --git a/changes.d/5864.feat.md b/changes.d/5864.feat.md new file mode 100644 index 00000000000..905b6b9dadd --- /dev/null +++ b/changes.d/5864.feat.md @@ -0,0 +1 @@ +Reimplemented the `suite-state` xtrigger for interoperability with Cylc 7. diff --git a/cylc/flow/xtriggers/suite_state.py b/cylc/flow/xtriggers/suite_state.py new file mode 100644 index 00000000000..dc8351a69df --- /dev/null +++ b/cylc/flow/xtriggers/suite_state.py @@ -0,0 +1,81 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cylc.flow import LOG +import cylc.flow.flags +from cylc.flow.xtriggers.workflow_state import workflow_state + +if not cylc.flow.flags.cylc7_back_compat: + LOG.warning( + "The suite_state xtrigger is deprecated. " + "Please use the workflow_state xtrigger instead." + ) + + +def suite_state(suite, task, point, offset=None, status='succeeded', + message=None, cylc_run_dir=None, debug=False): + """Suite state xtrigger, required for interoperability with Cylc 7. + + * The suite_state xtrigger was renamed to workflow_state, this breaks Cylc 7-8 interoperability. + * This suite_state xtrigger replicates workflow_state - ensuring back-support. + + Arguments: + suite: + The workflow to interrogate. + task: + The name of the task to query. + point: + The cycle point. + offset: + The offset between the cycle this xtrigger is used in and the one + it is querying for as an ISO8601 time duration. + e.g. PT1H (one hour). + status: + The task status required for this xtrigger to be satisfied. + message: + The custom task output required for this xtrigger to be satisfied. + .. note:: + + This cannot be specified in conjunction with ``status``. + + cylc_run_dir: + The directory in which the workflow to interrogate. + + .. note:: + + This only needs to be supplied if the workflow is running in a + different location to what is specified in the global + configuration (usually ``~/cylc-run``). + + Returns: + tuple: (satisfied, results) + + satisfied: + True if ``satisfied`` else ``False``. + results: + Dictionary containing the args / kwargs which were provided + to this xtrigger. + + """ + return workflow_state( + workflow=suite, + task=task, + point=point, + offset=offset, + status=status, + message=message, + cylc_run_dir=cylc_run_dir + ) diff --git a/setup.cfg b/setup.cfg index 7db9eb8bf4e..849f4ae71b8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -220,6 +220,7 @@ cylc.xtriggers = echo = cylc.flow.xtriggers.echo:echo wall_clock = cylc.flow.xtriggers.wall_clock:wall_clock workflow_state = cylc.flow.xtriggers.workflow_state:workflow_state + suite_state = cylc.flow.xtriggers.suite_state:suite_state xrandom = cylc.flow.xtriggers.xrandom:xrandom [bdist_rpm] diff --git a/tests/functional/xtriggers/04-suite_state.t b/tests/functional/xtriggers/04-suite_state.t new file mode 100644 index 00000000000..ece9cbe715e --- /dev/null +++ b/tests/functional/xtriggers/04-suite_state.t @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Test that deprecation warnings are printed appropriately for the suite_state +# xtrigger. + +. "$(dirname "$0")/test_header" + +set_test_number 4 + +init_workflow "$TEST_NAME_BASE" << __FLOW_CONFIG__ +[scheduling] + initial cycle point = 2000 + [[dependencies]] + [[[R1]]] + graph = @upstream => foo + [[xtriggers]] + upstream = suite_state(suite=thorin/oin/gloin, task=mithril, point=1) +[runtime] + [[foo]] +__FLOW_CONFIG__ + +msg='WARNING - The suite_state xtrigger is deprecated' + +TEST_NAME="${TEST_NAME_BASE}-val" +run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME" + +grep_ok "$msg" "${TEST_NAME}.stderr" + +# Rename flow.cylc to suite.rc: +mv "${WORKFLOW_RUN_DIR}/flow.cylc" "${WORKFLOW_RUN_DIR}/suite.rc" + +TEST_NAME="${TEST_NAME_BASE}-val-2" +run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME" + +grep_fail "$msg" "${TEST_NAME}.stderr" diff --git a/tests/unit/xtriggers/test_workflow_state.py b/tests/unit/xtriggers/test_workflow_state.py index b3d25737cc2..ed02750f04a 100644 --- a/tests/unit/xtriggers/test_workflow_state.py +++ b/tests/unit/xtriggers/test_workflow_state.py @@ -42,7 +42,7 @@ def test_inferred_run(tmp_run_dir: Callable, monkeymock: MonkeyMock): assert results['workflow'] == expected_workflow_id -def test_back_compat(tmp_run_dir): +def test_back_compat(tmp_run_dir, caplog): """Test workflow_state xtrigger backwards compatibility with Cylc 7 database.""" id_ = 'celebrimbor' @@ -80,7 +80,15 @@ def test_back_compat(tmp_run_dir): finally: conn.close() + # Test workflow_state function satisfied, _ = workflow_state(id_, task='mithril', point='2012') assert satisfied satisfied, _ = workflow_state(id_, task='arkenstone', point='2012') assert not satisfied + + # Test back-compat (old suite_state function) + from cylc.flow.xtriggers.suite_state import suite_state + satisfied, _ = suite_state(suite=id_, task='mithril', point='2012') + assert satisfied + satisfied, _ = suite_state(suite=id_, task='arkenstone', point='2012') + assert not satisfied From 381c691d9d5ecd7f7c369ad1ee2833b4793e73fc Mon Sep 17 00:00:00 2001 From: Mark Dawson Date: Thu, 8 Feb 2024 12:34:24 +0000 Subject: [PATCH 17/20] fixed flake8 (#5962) Co-authored-by: Mark Dawson --- cylc/flow/xtriggers/suite_state.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cylc/flow/xtriggers/suite_state.py b/cylc/flow/xtriggers/suite_state.py index dc8351a69df..45a8418a832 100644 --- a/cylc/flow/xtriggers/suite_state.py +++ b/cylc/flow/xtriggers/suite_state.py @@ -29,8 +29,10 @@ def suite_state(suite, task, point, offset=None, status='succeeded', message=None, cylc_run_dir=None, debug=False): """Suite state xtrigger, required for interoperability with Cylc 7. - * The suite_state xtrigger was renamed to workflow_state, this breaks Cylc 7-8 interoperability. - * This suite_state xtrigger replicates workflow_state - ensuring back-support. + * The suite_state xtrigger was renamed to workflow_state - + this breaks Cylc 7-8 interoperability. + * This suite_state xtrigger replicates workflow_state - + ensuring back-support. Arguments: suite: From 8ab57ad0943abeea5b297f7d3b19d3cd8a055175 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:57:02 +0000 Subject: [PATCH 18/20] fix tui tests broken by urwid update (#5967) --- .../tui/screenshots/test_auto_expansion.later-time.html | 2 +- .../tui/screenshots/test_auto_expansion.on-load.html | 2 +- tests/integration/tui/screenshots/test_errors.open-error.html | 2 +- tests/integration/tui/screenshots/test_job_logs.01-job.out.html | 2 +- tests/integration/tui/screenshots/test_job_logs.02-job.out.html | 2 +- .../screenshots/test_navigation.cursor-at-bottom-of-screen.html | 2 +- .../tui/screenshots/test_navigation.family-A-collapsed.html | 2 +- tests/integration/tui/screenshots/test_navigation.on-load.html | 2 +- .../tui/screenshots/test_navigation.workflow-expanded.html | 2 +- .../test_offline_mutation.clean-mutation-selected.html | 2 +- .../test_offline_mutation.stop-all-mutation-selected.html | 2 +- .../test_online_mutation.hold-mutation-selected.html | 2 +- .../tui/screenshots/test_online_mutation.task-selected.html | 2 +- .../screenshots/test_restart_reconnect.1-workflow-running.html | 2 +- .../screenshots/test_restart_reconnect.2-workflow-stopped.html | 2 +- .../test_restart_reconnect.3-workflow-restarted.html | 2 +- .../tui/screenshots/test_scheduler_logs.log-file-selection.html | 2 +- .../tui/screenshots/test_scheduler_logs.scheduler-log-file.html | 2 +- .../test_scheduler_logs.workflow-configuration-file.html | 2 +- .../tui/screenshots/test_subscribe_unsubscribe.subscribed.html | 2 +- .../screenshots/test_subscribe_unsubscribe.unsubscribed.html | 2 +- .../tui/screenshots/test_task_logs.latest-job.err.html | 2 +- .../tui/screenshots/test_task_logs.latest-job.out.html | 2 +- .../tui/screenshots/test_tui_basics.test-rakiura-enter.html | 2 +- .../tui/screenshots/test_tui_basics.test-rakiura.html | 2 +- .../tui/screenshots/test_workflow_states.filter-active.html | 2 +- .../screenshots/test_workflow_states.filter-starts-with-t.html | 2 +- .../test_workflow_states.filter-stopped-or-paused.html | 2 +- .../tui/screenshots/test_workflow_states.filter-stopped.html | 2 +- .../tui/screenshots/test_workflow_states.unfiltered.html | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html index f5a19fd428d..ca1a8a73f0c 100644 --- a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html +++ b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html @@ -1,7 +1,7 @@ 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ b                                                                  
       - ̿○ 2                                                                     
diff --git a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
index df3c9f5c41b..d8f3ca712b8 100644
--- a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
+++ b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
          - ̿○ A                                                                  
               ̿○ a                                                               
diff --git a/tests/integration/tui/screenshots/test_errors.open-error.html b/tests/integration/tui/screenshots/test_errors.open-error.html
index 142d0d88c72..31d842ca75b 100644
--- a/tests/integration/tui/screenshots/test_errors.open-error.html
+++ b/tests/integration/tui/screenshots/test_errors.open-error.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Error: Something went wrong :(                                              │
 │                                                                              │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │                                                                              │
 │                                                                              │
diff --git a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
index c1c767b98cf..6f6f87fdce8 100644
--- a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
+++ b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/01                                                                 │
 │  this is a job log                                                           │
diff --git a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
index 0eb94051201..e14a77b99e5 100644
--- a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
+++ b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/02                                                                 │
 │  this is a job log                                                           │
diff --git a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
index bf5e3812008..c25e28ebd11 100644
--- a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
+++ b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
@@ -10,7 +10,7 @@
                  ̿○ b12                                                          
             - ̿○ B2                                                              
                  ̿○ b21                                                          
-                 ̿○ b22                                                          
+                 ̿○ b22                                                          
                                                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
index cefab5264f4..4c988d41792 100644
--- a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
+++ b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
@@ -3,7 +3,7 @@
 - ~cylc                                                                         
    - one - paused                                                               
       - ̿○ 1                                                                     
-         + ̿○ A                                                                  
+         + ̿○ A                                                                  
          - ̿○ B                                                                  
             - ̿○ B1                                                              
                  ̿○ b11                                                          
diff --git a/tests/integration/tui/screenshots/test_navigation.on-load.html b/tests/integration/tui/screenshots/test_navigation.on-load.html
index a0bd107742b..ea7d6577c9b 100644
--- a/tests/integration/tui/screenshots/test_navigation.on-load.html
+++ b/tests/integration/tui/screenshots/test_navigation.on-load.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - paused                                                               
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
index 6b26ced563e..e91225fef23 100644
--- a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
+++ b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
          - ̿○ A                                                                  
               ̿○ a1                                                              
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
index f28cced0714..5590254a28d 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
@@ -4,7 +4,7 @@
    + one - stop│  Action                                        │               
                │  < (cancel)                                 >  │               
                │                                                │               
-               │  < clean                                    >  │               
+               │  < clean                                    >  │               
                │  < log                                      >  │               
                │  < play                                     >  │               
                │  < reinstall-reload                         >  │               
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
index c2355597f78..eb04278317b 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
@@ -4,7 +4,7 @@
    + one - stop│  Action                                        │               
                │  < (cancel)                                 >  │               
                │                                                │               
-               │  < stop-all                                 >  │               
+               │  < stop-all                                 >  │               
                │                                                │               
                │                                                │               
                │                                                │               
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
index 34be2ffa0ce..b8a802948f3 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -4,7 +4,7 @@
    - one - paus│  Action                                        │               
       - ̿○ 1    │  < (cancel)                                 >  │               
            ̿○ on│                                                │               
-               │  < hold                                     >  │               
+               │  < hold                                     >  │               
                │  < kill                                     >  │               
                │  < log                                      >  │               
                │  < poll                                     >  │               
diff --git a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
index 7d94d5e43dd..df7a917ff60 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
@@ -3,7 +3,7 @@
 - ~cylc                                                                         
    - one - paused                                                               
       - ̿○ 1                                                                     
-           ̿○ one                                                                
+           ̿○ one                                                                
                                                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
index 74c02508239..aabddcc9cf6 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ one                                                                
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
index 09c3bbd7fb0..ee75b87558d 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - stopped                                                              
+   - one - stopped                                                              
         Workflow is not running                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
index 74c02508239..aabddcc9cf6 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ one                                                                
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
index f88e1b0124d..14be08c1066 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
@@ -14,7 +14,7 @@
 │                     ┌──────────────────────────────────────┐                 │
 │                     │  Select File                         │                 │
 │                     │                                      │                 │
-│                     │  < config/01-start-01.cylc        >  │                 │
+│                     │  < config/01-start-01.cylc        >  │                 │
 │                     │  < config/flow-processed.cylc     >  │                 │
 │                     │  < scheduler/01-start-01.log      >  │                 │
 │                     │                                      │                 │
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
index 68dbcc10f9c..57d6e4066b1 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  this is the                                                                 │
 │  scheduler log file                                                          │
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
index 011734bc410..c7ab1e925ec 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  [scheduling]                                                                │
 │      [[graph]]                                                               │
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
index 019184ec897..7d190cb6f4d 100644
--- a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ one                                                                
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
index 8fa0f4329a1..39c2f247c97 100644
--- a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - paused                                                               
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
index 4814892df7a..481d427be6f 100644
--- a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/02                                                                 │
 │  this is a job error                                                         │
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
index 0eb94051201..e14a77b99e5 100644
--- a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/02                                                                 │
 │  this is a job log                                                           │
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
index d54d9538d26..34601749413 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
@@ -12,7 +12,7 @@
                │  id: ~cylc/root                                │               
                │                                                │               
                │  Action                                        │               
-               │  < (cancel)                                 >  │               
+               │  < (cancel)                                 >  │               
                │                                                │               
                │  < stop-all                                 >  │               
                │                                                │               
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
index 7f80031804b..d507b83d799 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
                                                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
index 282f76735ed..994d00e911e 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - stopping                                                             
    + two - paused                                                               
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
index 8c26ce6ccc9..65b11fe4411 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + tre - stopped                                                              
    + two - paused                                                               
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
index 8c26ce6ccc9..65b11fe4411 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + tre - stopped                                                              
    + two - paused                                                               
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
index 1ff602df101..3913180f312 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + tre - stopped                                                              
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
index 0651eedec30..d8cf9ff39fd 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - stopping                                                             
    + tre - stopped                                                              
    + two - paused                                                               

From 4eb7d82ab8fba56247baedb3a8975d6c86a57f2d Mon Sep 17 00:00:00 2001
From: Oliver Sanders 
Date: Thu, 15 Feb 2024 12:37:37 +0000
Subject: [PATCH 19/20] remote_init: remove trailing space from symlink paths
 (#5969)

---
 cylc/flow/task_remote_mgr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py
index 541bc1265a0..2797672617b 100644
--- a/cylc/flow/task_remote_mgr.py
+++ b/cylc/flow/task_remote_mgr.py
@@ -247,7 +247,7 @@ def remote_init(
         dirs_to_symlink = get_dirs_to_symlink(install_target, self.workflow)
         for key, value in dirs_to_symlink.items():
             if value is not None:
-                cmd.append(f"{key}={quote(value)} ")
+                cmd.append(f"{key}={quote(value)}")
         # Create the ssh command
         try:
             host = get_host_from_platform(

From ca6c64d930fa28a24d0e86d3f50d35ffa815099f Mon Sep 17 00:00:00 2001
From: Oliver Sanders 
Date: Thu, 15 Feb 2024 14:15:24 +0000
Subject: [PATCH 20/20] doc: link workflow event configs to user guide section
 (#5954)

---
 cylc/flow/cfgspec/globalcfg.py | 64 ++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 2d39ad74829..713db726660 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -165,9 +165,17 @@
 
         Template variables can be used to configure handlers. For a full list
         of supported variables see :ref:`workflow_event_template_variables`.
+
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
     ''',
     'handler events': '''
         Specify the events for which workflow event handlers should be invoked.
+
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
     ''',
     'mail events': '''
         Specify the workflow events for which notification emails should
@@ -176,6 +184,10 @@
     'startup handlers': f'''
         Handlers to run at scheduler startup.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``startup handler``.
@@ -183,6 +195,10 @@
     'shutdown handlers': f'''
         Handlers to run at scheduler shutdown.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``shutdown handler``.
@@ -191,6 +207,10 @@
         Handlers to run if the scheduler shuts down with error status due to
         a configured timeout or a fatal error condition.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``aborted handler``.
@@ -199,22 +219,38 @@
         Workflow timeout interval. The timer starts counting down at scheduler
         startup. It resets on workflow restart.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.0.0
     ''',
     'workflow timeout handlers': '''
         Handlers to run if the workflow timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.0.0
     ''',
     'abort on workflow timeout': '''
         Whether the scheduler should shut down immediately with error status if
         the workflow timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.0.0
     ''',
     'stall handlers': f'''
         Handlers to run if the scheduler stalls.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``stalled handler``.
@@ -222,6 +258,10 @@
     'stall timeout': f'''
         The length of a timer which starts if the scheduler stalls.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``timeout``.
@@ -229,6 +269,10 @@
     'stall timeout handlers': f'''
         Handlers to run if the stall timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``timeout handler``.
@@ -237,6 +281,10 @@
         Whether the scheduler should shut down immediately with error status if
         the stall timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``abort on timeout``.
@@ -245,6 +293,10 @@
         Scheduler inactivity timeout interval. The timer resets when any
         workflow activity occurs.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES} ``inactivity``.
@@ -252,6 +304,10 @@
     'inactivity timeout handlers': f'''
         Handlers to run if the inactivity timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``inactivity handler``.
@@ -260,6 +316,10 @@
         Whether the scheduler should shut down immediately with error status if
         the inactivity timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``abort on inactivity``.
@@ -268,6 +328,10 @@
         How long to wait for intervention on restarting a completed workflow.
         The timer stops if any task is triggered.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.2.0
 
     '''