diff --git a/README.md b/README.md index b3976ff265d..7943f069716 100644 --- a/README.md +++ b/README.md @@ -29,28 +29,14 @@ domains. # install cylc conda install cylc-flow -# write your first workflow -mkdir -p ~/cylc-src/example -cat > ~/cylc-src/example/flow.cylc <<__CONFIG__ -[scheduling] - initial cycle point = 1 - cycling mode = integer - [[graph]] - P1 = """ - a => b => c & d - b[-P1] => b - """ -[runtime] - [[a, b, c, d]] - script = echo "Hello $CYLC_TASK_NAME" -__CONFIG__ +# extract an example to run +cylc get-resources examples/integer-cycling # install and run it -cylc install example -cylc play example +cylc vip integer-cycling # vip = validate, install and play # watch it run -cylc tui example +cylc tui integer-cycling ``` ### The Cylc Ecosystem diff --git a/cylc/flow/etc/examples/README.md b/cylc/flow/etc/examples/README.md new file mode 100644 index 00000000000..ae4e1391d23 --- /dev/null +++ b/cylc/flow/etc/examples/README.md @@ -0,0 +1,27 @@ +# Examples + +These examples are intended to illustrate the major patterns for implementing +Cylc workflows. The hope is that users can find a workflow which fits their +pattern, make a copy and fill in the details. Keep the examples minimal and +abstract. We aren't trying to document every Cylc feature here, just the +major design patterns. + +These examples are auto-documented in cylc-doc which looks for an `index.rst` +file in each example. + +Users can extract them using `cylc get-resources` which will put them into the +configured Cylc source directory (`~/cylc-src` by default). They can then be +run using the directory name, e.g. `cylc vip hello-world`. + +Files: + +* `index.rst` + This file is used to generate a page in the documentation for the example. + This file is excluded when the user extracts the example. +* `.validate` + This is a test file, it gets detected and run automatically. + This file is excluded when the user extracts the example. +* `README.rst` + Examples can include a README file, to save duplication, you can + `.. include::` this in the `index.rst` file (hence using ReStructuredText + rather than Markdown). diff --git a/cylc/flow/etc/examples/converging-workflow/flow.cylc b/cylc/flow/etc/examples/converging-workflow/flow.cylc new file mode 100644 index 00000000000..782033ca51e --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/flow.cylc @@ -0,0 +1,38 @@ +[meta] + title = Converging Workflow + description = """ + A workflow which runs a pattern of tasks over and over until a + convergence condition has been met. + """ + +[scheduling] + cycling mode = integer + initial cycle point = 1 + [[graph]] + P1 = """ + # run "increment" then check the convergence condition + increment => check_convergence + + # if the workflow has not converged yet, then run another cycle + check_convergence:not_converged? => increment[+P1] + + # if the workflow has converged, then do nothing + check_convergence:converged? + """ + +[runtime] + [[increment]] + # a task which evolves the data + [[check_convergence]] + # a task which checks whether the convergence condition has been met + script = """ + if (( CYLC_TASK_CYCLE_POINT == 4 )); then + # for the purpose of example, assume convergence at cycle point 4 + cylc message -- convergence condition met + else + cylc message -- convergence condition not met + fi + """ + [[[outputs]]] + converged = 'convergence condition met' + not_converged = 'convergence condition not met' diff --git a/cylc/flow/etc/examples/converging-workflow/index.rst b/cylc/flow/etc/examples/converging-workflow/index.rst new file mode 100644 index 00000000000..fa7de1aa731 --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/index.rst @@ -0,0 +1,39 @@ +Converging Workflow +=================== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/converging-workflow + +A workflow which runs a pattern of tasks over and over until a convergence +condition has been met. + +* The ``increment`` task runs some kind of model or process which increments + us toward the solutiuon. +* The ``check_convergence`` task, checks if the convergence condition has been + met. + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip converging-workflow + +.. admonition:: Example - Genetic algorithms + :class: hint + + .. _genetic algorithm: https://en.wikipedia.org/wiki/Genetic_algorithm + + An example of a converging workflow might be a `genetic algorithm`_, where you + "breed" entities, then test their "fitness", and breed again, over and over + until you end up with an entity which is able to satisfy the requirement. + + .. digraph:: Example + + random_seed -> breed -> test_fitness + test_fitness -> breed + test_fitness -> stop diff --git a/cylc/flow/etc/examples/datetime-cycling/.validate b/cylc/flow/etc/examples/datetime-cycling/.validate new file mode 100755 index 00000000000..32c2a51908b --- /dev/null +++ b/cylc/flow/etc/examples/datetime-cycling/.validate @@ -0,0 +1,28 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip \ + --check-circular \ + --no-detach \ + --no-run-name \ + --final-cycle-point "$(isodatetime now --format 'CCYYMMDD')T00" \ + --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/datetime-cycling/flow.cylc b/cylc/flow/etc/examples/datetime-cycling/flow.cylc new file mode 100644 index 00000000000..1102c3b8e96 --- /dev/null +++ b/cylc/flow/etc/examples/datetime-cycling/flow.cylc @@ -0,0 +1,44 @@ +[meta] + title = Datetime Cycling + description = """ + A basic cycling workflow which runs the same set of tasks over + and over. Each cycle will be given a datetime identifier. + + The task "a" will wait until the real-world (or wallclock) time passes + the cycle time. + + Try changing the "initial cycle point" to "previous(00T00) - P1D" to + see how this works. + """ + +[scheduling] + # start this workflow today at midnight + initial cycle point = previous(T00) + + [[graph]] + # repeat this with a "P"eriod of "1" "D"ay -> P1D + P1D = """ + # this is the workflow we want to repeat: + a => b => c & d + + # this is an "inter-cycle dependency", it makes the task "b" + # wait until its previous instance has successfully completed: + b[-P1D] => b + + # this makes the task "a" wait until its cycle point matches + # the real world time - i.e. it prevents the workflow from getting + # ahead of the clock. If the workflow is running behind (e.g. after + # a delay, or from an earlier initial cycle point) it will catch + # until the clock-trigger constrains it again. To run entirely in + # "simulated time" remove this line: + @wall_clock => a + """ + +[runtime] + [[root]] + # all tasks will "inherit" the configuration in the "root" section + script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!" + [[a]] + [[b]] + [[c]] + [[d]] diff --git a/cylc/flow/etc/examples/datetime-cycling/index.rst b/cylc/flow/etc/examples/datetime-cycling/index.rst new file mode 100644 index 00000000000..8b2482cef9b --- /dev/null +++ b/cylc/flow/etc/examples/datetime-cycling/index.rst @@ -0,0 +1,16 @@ +Date-Time Cycling +================= + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/datetime-cycling + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip datetime-cycling diff --git a/cylc/flow/etc/examples/event-driven-cycling/.validate b/cylc/flow/etc/examples/event-driven-cycling/.validate new file mode 100755 index 00000000000..ef224cd9e2c --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/.validate @@ -0,0 +1,47 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + +# start the workflow +cylc vip --check-circular --no-run-name --workflow-name "$ID" +sleep 1 # give it a reasonable chance to start up + +# kick off the first cycle +./bin/trigger "$ID" WORLD=earth + +# wait for it to complete +cylc workflow-state "$ID" \ + --task=run \ + --point=1 \ + --status=succeeded \ + --max-polls=60 \ + --interval=1 + +# check the job received the environment variable we provided +grep 'Hello earth' "$HOME/cylc-run/$ID/log/job/1/run/NN/job.out" + +# stop the workflow +cylc stop --kill --max-polls=10 --interval=2 "$ID" + +# lint +cylc lint "$ID" + +# clean up +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/event-driven-cycling/README.rst b/cylc/flow/etc/examples/event-driven-cycling/README.rst new file mode 100644 index 00000000000..152ce3be42c --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/README.rst @@ -0,0 +1,49 @@ +Cylc is good at orchestrating tasks to a schedule, e.g: + +* ``PT1H`` - every hour +* ``P1D`` - every day +* ``P1M`` - every month +* ``PT1H ! (T00, T12)`` - every hour, except midnight and midday. + +But sometimes the things you want to run don't have a schedule. + +This example uses ``cylc ext-trigger`` to establish a pattern where Cylc waits +for an external signal and starts a new cycle every time a signal is recieved. + +The signal can carry data using the ext-trigger ID, this example sets the ID +as a file path containing some data that we want to make available to the tasks +that run in the cycle it triggers. + +To use this example, first start the workflow as normal:: + + cylc vip event-driven-cycling + +Then, when you're ready, kick off a new cycle, specifying any +environment variables you want to configure this cycle with:: + + ./bin/trigger WORLD=earth + +Replacing ```` with the ID you installed this workflow as. + +.. admonition:: Example - CI/CD + :class: hint + + This pattern is good for CI/CD type workflows where you're waiting on + external events. This pattern is especially powerful when used with + sub-workflows where it provides a solution to two-dimensional cycling + problems. + +.. admonition:: Example - Polar satellite data processing + :class: hint + + Polar satellites pass overhead at irregular intervals. This makes it tricky + to schedule data processing because you don't know when the satellite will + pass over the receiver station. With the event driven cycling approach you + could start a new cycle every time data arrives. + +.. note:: + + * The number of parallel cycles can be adjusted by changing the + :cylc:conf:`[scheduling]runahead limit`. + * To avoid hitting the runahead limit ensure failures are handled in the + graph. diff --git a/cylc/flow/etc/examples/event-driven-cycling/bin/trigger b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger new file mode 100755 index 00000000000..09cdaff4dbf --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu + +if [[ $# -lt 1 ]]; then + echo 'Usage ./trigger WORKFLOW_ID [KEY=VALUE ...]' >&2 + echo + echo 'Trigger a new cycle in the target workflow.' + echo 'Any environment variable KEY=VALUE pairs will be broadcasted to' + echo 'all tasks in the cycle.' + exit 1 +fi + +# determine the workflow +WORKFLOW_ID="$1" +shift +WORKFLOW_RUN_DIR="${HOME}/cylc-run/${WORKFLOW_ID}" +EXT_TRIGGER_DIR="${WORKFLOW_RUN_DIR}/triggers" +mkdir -p "$EXT_TRIGGER_DIR" + +# pick a trigger-id +TRIGGER_ID="$(date --iso-8601=seconds)" + +# write environment variables to a broadcast fuke +TRIGGER_FILE="${EXT_TRIGGER_DIR}/${TRIGGER_ID}.cylc" +echo '[environment]' >"$TRIGGER_FILE" +for env in "$@"; do + echo " $env" >> "$TRIGGER_FILE" +done + +# issue the xtrigger +cylc ext-trigger "$WORKFLOW_ID" 'trigger' "$TRIGGER_ID" diff --git a/cylc/flow/etc/examples/event-driven-cycling/flow.cylc b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc new file mode 100644 index 00000000000..7711c2ba258 --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc @@ -0,0 +1,32 @@ +[scheduling] + cycling mode = integer + initial cycle point = 1 + runahead limit = P5 # max number of cycles which can run in parallel + [[special tasks]] + # register the external trigger, it must be given a name, + # here, 'trigger' is used as a placeholder, the bash script will + # need to be updated if this is changed + external-trigger = configure("trigger") + [[graph]] + P1 = """ + # use a "?" to prevent failures causing runahead stalls + configure? => run + """ + +[runtime] + [[configure]] + # this task reads in the broadcast file the trigger wrote + # and broadcasts any variables set to all tasks in this cycle + script = """ + echo "received new ext-trigger ID=$CYLC_EXT_TRIGGER_ID" + TRIGGER_FILE="${CYLC_WORKFLOW_RUN_DIR}/triggers/${CYLC_EXT_TRIGGER_ID}.cylc" + cylc broadcast "${CYLC_WORKFLOW_ID}" \ + -p "${CYLC_TASK_CYCLE_POINT}" \ + -F "${TRIGGER_FILE}" + """ + + [[run]] + # this task could be a sub-workflow + script = """ + echo "Hello $WORLD!" + """ diff --git a/cylc/flow/etc/examples/event-driven-cycling/index.rst b/cylc/flow/etc/examples/event-driven-cycling/index.rst new file mode 100644 index 00000000000..5980591f0fd --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/index.rst @@ -0,0 +1,14 @@ +Event Driven Cycling +==================== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/event-driven-cycling + +.. include:: README.rst + +.. literalinclude:: flow.cylc + :language: cylc diff --git a/cylc/flow/etc/examples/hello-world/.validate b/cylc/flow/etc/examples/hello-world/.validate new file mode 100755 index 00000000000..968a7b349cb --- /dev/null +++ b/cylc/flow/etc/examples/hello-world/.validate @@ -0,0 +1,23 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip --check-circular --no-detach --no-run-name --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/hello-world/flow.cylc b/cylc/flow/etc/examples/hello-world/flow.cylc new file mode 100644 index 00000000000..92510d1928b --- /dev/null +++ b/cylc/flow/etc/examples/hello-world/flow.cylc @@ -0,0 +1,13 @@ +[meta] + title = Hello World + description = """ + A simple workflow which runs a single task (hello_world) once. + """ + +[scheduling] + [[graph]] + R1 = hello_world + +[runtime] + [[hello_world]] + script = echo "Hello World!" diff --git a/cylc/flow/etc/examples/hello-world/index.rst b/cylc/flow/etc/examples/hello-world/index.rst new file mode 100644 index 00000000000..683df7ab5f8 --- /dev/null +++ b/cylc/flow/etc/examples/hello-world/index.rst @@ -0,0 +1,20 @@ +Hello World +----------- + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/hello-world + +In the time honoured tradition, this is the minimal Cylc workflow: + +.. literalinclude:: flow.cylc + :language: cylc + +It writes the phrase "Hello World!" to standard output (captured to the `job.out` log file). + +Run it with:: + + $ cylc vip hello-world diff --git a/cylc/flow/etc/examples/integer-cycling/.validate b/cylc/flow/etc/examples/integer-cycling/.validate new file mode 100755 index 00000000000..054f2662862 --- /dev/null +++ b/cylc/flow/etc/examples/integer-cycling/.validate @@ -0,0 +1,23 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip --check-circular --no-detach --no-run-name --final-cycle-point=1 --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/integer-cycling/flow.cylc b/cylc/flow/etc/examples/integer-cycling/flow.cylc new file mode 100644 index 00000000000..b8d2f2f05ce --- /dev/null +++ b/cylc/flow/etc/examples/integer-cycling/flow.cylc @@ -0,0 +1,28 @@ +[meta] + title = Integer Cycling + description = """ + A basic cycling workflow which runs the same set of tasks over + and over. Each cycle will be given an integer number. + """ + +[scheduling] + # tell Cylc to count cycles as numbers starting from the number 1 + cycling mode = integer + initial cycle point = 1 + [[graph]] + P1 = """ + # this is the workflow we want to repeat: + a => b => c & d + # this is an "inter-cycle dependency", it makes the task "b" + # wait until its previous instance has completed: + b[-P1] => b + """ + +[runtime] + [[root]] + # all tasks will "inherit" the configuration in the "root" section + script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!" + [[a]] + [[b]] + [[c]] + [[d]] diff --git a/cylc/flow/etc/examples/integer-cycling/index.rst b/cylc/flow/etc/examples/integer-cycling/index.rst new file mode 100644 index 00000000000..89be2bb0871 --- /dev/null +++ b/cylc/flow/etc/examples/integer-cycling/index.rst @@ -0,0 +1,16 @@ +Integer Cycling +=============== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/integer-cycling + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip integer-cycling diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/.validate b/cylc/flow/etc/examples/inter-workflow-triggers/.validate new file mode 100755 index 00000000000..bdd414e275d --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/.validate @@ -0,0 +1,57 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +UPID=inter-workflow-triggers/upstream +DOID=inter-workflow-triggers/downstream + +ICP="$(isodatetime now --format=CCYYMMDDThh --offset=-PT2H)" + +# run the workflows +cylc vip \ + --check-circular \ + --no-run-name \ + --final-cycle-point="$ICP" \ + --workflow-name "$UPID" \ + ./upstream +cylc vip \ + --check-circular \ + --no-run-name \ + --final-cycle-point="$ICP" \ + --workflow-name "$DOID" \ + ./downstream + +# wait for the first task in the downstream to succeed +cylc workflow-state "$DOID" \ + --task=process \ + --point="$ICP" \ + --status=succeeded \ + --max-polls=60 \ + --interval=1 + +# stop the workflows +cylc stop --kill --max-polls=10 --interval=2 "$UPID" +cylc stop --kill --max-polls=10 --interval=2 "$DOID" + +# lint'em +cylc lint "$UPID" +cylc lint "$DOID" + +# clean up +cylc clean "$UPID" +cylc clean "$DOID" diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/README.rst b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst new file mode 100644 index 00000000000..8aa8d7d7b6f --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst @@ -0,0 +1,14 @@ +This example shows how one workflow can "trigger off" of tasks in another +workflow. + +In this example, there are two workflows: + +* "upstream" writes a file. +* "downstream" reads this file. + +Run both workflows simultaneously to see this in action: + +.. code-block:: console + + $ cylc vip inter-workflow-triggers/upstream + $ cylc vip inter-workflow-triggers/downstream diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc new file mode 100644 index 00000000000..dcd8be94dd9 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc @@ -0,0 +1,25 @@ +[meta] + title = Downstream Workflow + description = """ + This workflow uses the data provided by the upstream workflow. + """ + +[scheduling] + # start two hours before the current time + initial cycle point = previous(T-00) - PT2H + [[xtriggers]] + # this is an "xtrigger" it will wait for the task "b" in the same cycle + # from the workflow "upstream" + upstream = workflow_state(workflow="inter-workflow-triggers/upstream", task="b", point="%(point)s") + [[graph]] + PT1H = """ + @upstream => process + """ + +[runtime] + [[process]] + script = echo "The random number is: $(cat "$file")" + [[[environment]]] + # this is where the data should be written to in the upstream workflow + # Note: "runN" will point to the most recent run of a workflow + file = $HOME/cylc-run/upstream/runN/share/$CYLC_TASK_CYCLE_POINT diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/index.rst b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst new file mode 100644 index 00000000000..185b8b70e54 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst @@ -0,0 +1,28 @@ +Inter-Workflow Triggering +========================= + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/inter-workflow-triggers + +.. include:: README.rst + +.. literalinclude:: upstream/flow.cylc + :language: cylc + :caption: Upstream Workflow + +.. literalinclude:: downstream/flow.cylc + :language: cylc + :caption: Downstream Workflow + +.. admonition:: Example - Decoupled workflows + :class: hint + + This pattern is useful where you have workflows that you want to keep decoupled + from one another, but still need to exchange data. E.G. in operational + meteorology we might have a global model (covering the whole Earth) and a + regional model (just covering a little bit of of) where the regional model + obtains its boundary condition from the global model. diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc new file mode 100644 index 00000000000..217a8e79994 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc @@ -0,0 +1,27 @@ +[meta] + title = Upstream Workflow + description = """ + This is the workflow which is providing the data that the downstream + workflow wants to use. + """ + +[scheduling] + # start two hours before the current time + initial cycle point = previous(T-00) - PT2H + [[graph]] + PT1H = """ + # wait for the "real world" time before running "a": + @wall_clock => a + + # then run task "b" + a => b + """ + +[runtime] + [[a]] + [[b]] + # write a random number to ~/cylc-run//share/ + # for the downstream workflow to use + script = echo "$RANDOM" > "$file" + [[[environment]]] + file = ${CYLC_WORKFLOW_SHARE_DIR}/${CYLC_TASK_CYCLE_POINT} diff --git a/cylc/flow/resources.py b/cylc/flow/resources.py index 79884e27476..d81b7f7211b 100644 --- a/cylc/flow/resources.py +++ b/cylc/flow/resources.py @@ -16,6 +16,7 @@ """Extract named resources from the cylc.flow package.""" +from contextlib import suppress from pathlib import Path from random import shuffle import shutil @@ -31,6 +32,7 @@ RESOURCE_DIR = Path(cylc.flow.__file__).parent / 'etc' TUTORIAL_DIR = RESOURCE_DIR / 'tutorial' +EXAMPLE_DIR = RESOURCE_DIR / 'examples' # {resource: brief description} @@ -52,6 +54,11 @@ def list_resources(write=print, headers=True): for path in TUTORIAL_DIR.iterdir() if path.is_dir() ] + examples = [ + path.relative_to(RESOURCE_DIR) + for path in EXAMPLE_DIR.iterdir() + if path.is_dir() + ] if headers: write('Resources:') max_len = max(len(res) for res in RESOURCE_NAMES) @@ -62,15 +69,21 @@ def list_resources(write=print, headers=True): for tutorial in tutorials: write(f' {tutorial}') write(f' {API_KEY}') + if headers: + write('\nExamples:') + for example in examples: + write(f' {example}') -def path_is_tutorial(src: Path) -> bool: - """Returns True if the src path is in the tutorial directory.""" - try: +def path_is_source_workflow(src: Path) -> bool: + """Returns True if the src path is a Cylc workflow.""" + with suppress(ValueError): src.relative_to(TUTORIAL_DIR) - except ValueError: - return False - return True + return True + with suppress(ValueError): + src.relative_to(EXAMPLE_DIR) + return True + return False def get_resources(resource: str, tgt_dir: Optional[str]): @@ -95,11 +108,11 @@ def get_resources(resource: str, tgt_dir: Optional[str]): '\nRun `cylc get-resources --list` for resource names.' ) - is_tutorial = path_is_tutorial(src) + is_source_workflow = path_is_source_workflow(src) # get the target path if not tgt_dir: - if is_tutorial: + if is_source_workflow: # this is a tutorial => use the primary source dir _tgt_dir = Path(glbl_cfg().get(['install', 'source dirs'])[0]) else: @@ -113,8 +126,8 @@ def get_resources(resource: str, tgt_dir: Optional[str]): tgt = tgt.resolve() # extract resources - extract_resource(src, tgt, is_tutorial) - if is_tutorial: + extract_resource(src, tgt, is_source_workflow) + if is_source_workflow: set_api_key(tgt) @@ -131,16 +144,32 @@ def _backup(tgt: Path) -> None: shutil.move(str(tgt), str(backup)) -def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None: +def extract_resource( + src: Path, + tgt: Path, + is_source_workflow: bool = False, +) -> None: """Extract src into tgt. NOTE: src can be a dir or a file. """ LOG.info(f"Extracting {src.relative_to(RESOURCE_DIR)} to {tgt}") - if is_tutorial and tgt.exists(): + if is_source_workflow and tgt.exists(): # target exists, back up the old copy _backup(tgt) + # files to exclude + if is_source_workflow: + excludes = [ + # test files + '.validate', + 'reftest', + # documentation files + 'index.rst', + ] + else: + excludes = [] + # create the target directory try: tgt.parent.mkdir(parents=True, exist_ok=True) @@ -151,6 +180,10 @@ def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None: shutil.copytree(str(src), str(tgt)) else: shutil.copyfile(str(src), str(tgt)) + for exclude in excludes: + path = tgt / exclude + if path.exists(): + path.unlink() except IsADirectoryError as exc: LOG.error( f'Cannot extract file {exc.filename} as there is an ' diff --git a/etc/bin/run-validate-tutorials b/etc/bin/run-validate-tutorials index e9b7be1ecde..44baf2fc9ca 100755 --- a/etc/bin/run-validate-tutorials +++ b/etc/bin/run-validate-tutorials @@ -19,11 +19,13 @@ set -eu cd "$(dirname "$0")" -for FILE in $(echo ../../cylc/flow/etc/tutorial/*/.validate) ; do - echo "running $FILE" - ( - cd "$(dirname "$FILE")" - ./.validate - ) +for DIR in tutorial examples; do + echo "# Running tests for: $DIR" + for FILE in $(echo "../../cylc/flow/etc/$DIR/"*/.validate); do + echo "## Running test: $FILE" + ( + cd "$(dirname "$FILE")" + ./.validate + ) + done done -