diff --git a/.codecov.yml b/.codecov.yml index ad4d7104f80..5b3d02564ba 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -17,18 +17,19 @@ coverage: project: default: # commits below this threshold will be marked as failed - target: '85%' + target: '92%' # how much we allow the coverage to drop threshold: '2%' patch: default: - target: '95%' + target: '97%' threshold: '5%' # files to ignore ignore: - "tests/**" - "ws_messages_pb2.py" + - "cylc/flow/scripts/report_timings.py" flag_management: default_rules: diff --git a/.coveragerc b/.coveragerc index 1e2a2fb5157..5a4b396f19c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -16,6 +16,7 @@ omit = tests/* */cylc/flow/*_pb2.py cylc/flow/etc/* + cylc/flow/scripts/report_timings.py parallel = True source = ./cylc timid = False diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml index f94b8d60873..10a700b7ab4 100644 --- a/.github/workflows/1_create_release_pr.yml +++ b/.github/workflows/1_create_release_pr.yml @@ -21,7 +21,7 @@ jobs: uses: cylc/release-actions/stage-1/sanitize-inputs@v1 - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.BASE_REF }} fetch-depth: 0 # need to fetch all commits to check contributors @@ -30,7 +30,7 @@ jobs: uses: cylc/release-actions/check-shortlog@v1 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml index 99aac4991d6..f76f8afde2d 100644 --- a/.github/workflows/2_auto_publish_release.yml +++ b/.github/workflows/2_auto_publish_release.yml @@ -22,12 +22,12 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.MERGE_SHA }} - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' @@ -38,7 +38,7 @@ jobs: uses: cylc/release-actions/build-python-package@v1 - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.11 with: user: __token__ # uses the API token feature of PyPI - least permissions possible password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml index cf0e49c362b..eb34e3ad76d 100644 --- a/.github/workflows/bash.yml +++ b/.github/workflows/bash.yml @@ -44,7 +44,7 @@ jobs: - '5.0' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run Docker container run: | @@ -85,7 +85,7 @@ jobs: - name: Upload artifact if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: Upload cylc-run artifact + name: 'cylc-run (bash-${{ matrix.bash-version }})' path: cylc-run diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 041b6e50a70..d1d76b4e4d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,10 +24,10 @@ jobs: python: '3.7' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/shortlog.yml b/.github/workflows/shortlog.yml index 7998d92ad3f..25e2b0427e3 100644 --- a/.github/workflows/shortlog.yml +++ b/.github/workflows/shortlog.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # need to fetch all commits to check contributors diff --git a/.github/workflows/test_conda-build.yml b/.github/workflows/test_conda-build.yml index a077e798366..91d1ac2ad6b 100644 --- a/.github/workflows/test_conda-build.yml +++ b/.github/workflows/test_conda-build.yml @@ -19,7 +19,7 @@ jobs: timeout-minutes: 10 steps: - name: checkout cylc-flow - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: build conda env run: | diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml index 53d6c475bc3..66f08ce61ac 100644 --- a/.github/workflows/test_fast.yml +++ b/.github/workflows/test_fast.yml @@ -17,27 +17,30 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: - fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs + fail-fast: false # don't stop on first failure matrix: os: ['ubuntu-latest'] - python-version: ['3.7', '3.8', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.10', '3.11', '3'] include: # mac os test - os: 'macos-11' python-version: '3.7' # oldest supported version + # non-utc timezone test - os: 'ubuntu-latest' python-version: '3.9' # not the oldest, not the most recent version time-zone: 'XXX-09:35' + env: TZ: ${{ matrix.time-zone }} PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes + steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -45,7 +48,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get update - sudo apt-get install -y shellcheck sqlite3 + sudo apt-get install -y sqlite3 - name: Install run: | @@ -54,37 +57,10 @@ jobs: - name: Configure git # Needed by the odd test uses: cylc/release-actions/configure-git@v1 - - name: Check changelog - if: startsWith(matrix.os, 'ubuntu') - run: towncrier build --draft - - - name: Style - if: startsWith(matrix.os, 'ubuntu') - run: | - flake8 - etc/bin/shellchecker - - # note: exclude python 3.10+ from mypy checks as these produce false - # positives in installed libraries for python 3.7 - - name: Typing - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9) - run: mypy - - - name: Doctests - timeout-minutes: 4 - run: | - pytest cylc/flow - - name: Unit Tests - timeout-minutes: 4 + timeout-minutes: 5 run: | - pytest tests/unit - - - name: Bandit - if: ${{ matrix.python-version == '3.7' }} - # https://github.com/PyCQA/bandit/issues/658 - run: | - bandit -r --ini .bandit cylc/flow + pytest cylc/flow tests/unit - name: Integration Tests timeout-minutes: 6 @@ -93,7 +69,7 @@ jobs: - name: Upload failed tests artifact if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cylc-run (${{ matrix.os }} py-${{ matrix.python-version }}) path: ~/cylc-run/ @@ -104,15 +80,53 @@ jobs: coverage report - name: Upload coverage artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }} path: coverage.xml retention-days: 7 + lint: + runs-on: 'ubuntu-latest' + timeout-minutes: 10 + steps: + - name: Apt-Get Install + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + + - name: Checkout + uses: actions/checkout@v4 + + # note: exclude python 3.10+ from mypy checks as these produce false + # positives in installed libraries for python 3.7 + - name: Configure Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install + run: | + pip install -e ."[tests]" + + - name: Flake8 + run: flake8 + + - name: Bandit + run: | + bandit -r --ini .bandit cylc/flow + + - name: Shellchecker + run: etc/bin/shellchecker + + - name: MyPy + run: mypy + + - name: Towncrier + run: towncrier build --draft + - name: Linkcheck - if: startsWith(matrix.python-version, '3.10') - run: pytest -m linkcheck --dist=load tests/unit + run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py codecov: needs: test @@ -120,10 +134,10 @@ jobs: timeout-minutes: 2 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download coverage artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Codecov upload uses: codecov/codecov-action@v3 diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml index b1afd98cfe3..ed411c2eae8 100644 --- a/.github/workflows/test_functional.yml +++ b/.github/workflows/test_functional.yml @@ -46,9 +46,9 @@ jobs: # NOTE: includes must define ALL of the matrix values include: # latest python - - name: 'py-3.11' + - name: 'py-3-latest' os: 'ubuntu-latest' - python-version: '3.11' + python-version: '3' test-base: 'tests/f' chunk: '1/4' platform: '_local_background*' @@ -96,10 +96,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -108,7 +108,7 @@ jobs: run: | # install system deps brew update - brew install bash coreutils gnu-sed + brew install bash coreutils gnu-sed grep # add GNU coreutils and sed to the user PATH # (see instructions in brew install output) @@ -118,6 +118,9 @@ jobs: echo \ "/usr/local/opt/gnu-sed/libexec/gnubin" \ >> "${GITHUB_PATH}" + echo \ + "/usr/local/opt/grep/libexec/gnubin" \ + >> "${GITHUB_PATH}" # add coreutils to the bashrc too (for jobs) cat >> "${HOME}/.bashrc" <<__HERE__ @@ -248,6 +251,7 @@ jobs: -exec echo '====== {} ======' \; -exec cat '{}' \; - name: Set artifact upload name + if: always() id: uploadname run: | # artifact name cannot contain '/' characters @@ -256,7 +260,7 @@ jobs: - name: Upload failed tests artifact if: failure() && steps.test.outcome == 'failure' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cylc-run (${{ steps.uploadname.outputs.uploadname }}) path: ~/cylc-run/ @@ -294,7 +298,7 @@ jobs: coverage report - name: Upload coverage artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage_${{ steps.uploadname.outputs.uploadname }} path: coverage.xml @@ -306,10 +310,10 @@ jobs: timeout-minutes: 2 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download coverage artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Codecov upload uses: codecov/codecov-action@v3 diff --git a/.github/workflows/test_manylinux.yml b/.github/workflows/test_manylinux.yml index e3d21cee291..84e1e041283 100644 --- a/.github/workflows/test_manylinux.yml +++ b/.github/workflows/test_manylinux.yml @@ -38,13 +38,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure git # Needed by the odd test uses: cylc/release-actions/configure-git@v1 - name: Configure Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test_tutorial_workflow.yml b/.github/workflows/test_tutorial_workflow.yml index c83c850ad5d..7859b8588e2 100644 --- a/.github/workflows/test_tutorial_workflow.yml +++ b/.github/workflows/test_tutorial_workflow.yml @@ -21,12 +21,12 @@ jobs: test: strategy: matrix: - python-version: ['3.7', '3.11'] + python-version: ['3.7', '3'] runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: configure python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/update_copyright.yml b/.github/workflows/update_copyright.yml index 603973f4d83..878412a1a27 100644 --- a/.github/workflows/update_copyright.yml +++ b/.github/workflows/update_copyright.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure git uses: cylc/release-actions/configure-git@v1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6fbc4283968..a1bf42e6215 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ requests_). - Prasanna Challuri - David Matthews - Tim Whitcomb - - (Scott Wales) + - Scott Wales - Tomek Trzeciak - Thomas Coleman - Bruno Kinoshita @@ -92,6 +92,7 @@ requests_). - John Haiducek - (Andrew Huang) - Cheng Da + - Mark Dawson (All contributors are identifiable with email addresses in the git version diff --git a/changes.d/5571.feat.md b/changes.d/5571.feat.md new file mode 100644 index 00000000000..4bda6c6af4b --- /dev/null +++ b/changes.d/5571.feat.md @@ -0,0 +1 @@ +Make workflow `CYLC_` variables available to the template processor during parsing. diff --git a/changes.d/5600.break.md b/changes.d/5600.break.md new file mode 100644 index 00000000000..fd11f650ff5 --- /dev/null +++ b/changes.d/5600.break.md @@ -0,0 +1,3 @@ +The `cylc dump` command now only shows active tasks (e.g. running & queued +tasks). This restores its behaviour of only showing the tasks which currently +exist in the pool as it did in Cylc 7 and earlier versions of Cylc 8. diff --git a/changes.d/5709.feat.md b/changes.d/5709.feat.md new file mode 100644 index 00000000000..11aeabcf81d --- /dev/null +++ b/changes.d/5709.feat.md @@ -0,0 +1 @@ +Forward arbitrary environment variables over SSH connections diff --git a/changes.d/5727.break.md b/changes.d/5727.break.md new file mode 100644 index 00000000000..06cb196216d --- /dev/null +++ b/changes.d/5727.break.md @@ -0,0 +1 @@ +Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`. \ No newline at end of file diff --git a/changes.d/5731.feat.md b/changes.d/5731.feat.md new file mode 100644 index 00000000000..b0c28a01ac1 --- /dev/null +++ b/changes.d/5731.feat.md @@ -0,0 +1 @@ +Major upgrade to `cylc tui` which now supports larger workflows and can browse installed workflows. diff --git a/changes.d/5794.break.md b/changes.d/5794.break.md new file mode 100644 index 00000000000..53c5315b013 --- /dev/null +++ b/changes.d/5794.break.md @@ -0,0 +1 @@ +Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12. \ No newline at end of file diff --git a/changes.d/5803.feat.md b/changes.d/5803.feat.md new file mode 100644 index 00000000000..a4bc0f1b898 --- /dev/null +++ b/changes.d/5803.feat.md @@ -0,0 +1 @@ +Updated 'reinstall' functionality to support multiple workflows \ No newline at end of file diff --git a/changes.d/5831.feat.md b/changes.d/5831.feat.md new file mode 100644 index 00000000000..daecc5e7a87 --- /dev/null +++ b/changes.d/5831.feat.md @@ -0,0 +1 @@ +Add capability to install xtriggers via a new cylc.xtriggers entry point diff --git a/changes.d/5836.break.md b/changes.d/5836.break.md new file mode 100644 index 00000000000..8c14b101f63 --- /dev/null +++ b/changes.d/5836.break.md @@ -0,0 +1 @@ +Removed the 'CYLC_TASK_DEPENDENCIES' environment variable \ No newline at end of file diff --git a/changes.d/5872.feat.md b/changes.d/5872.feat.md new file mode 100644 index 00000000000..d88b0dd8116 --- /dev/null +++ b/changes.d/5872.feat.md @@ -0,0 +1 @@ +Improvements to `cylc clean` remote timeout handling. diff --git a/changes.d/5879.feat.md b/changes.d/5879.feat.md new file mode 100644 index 00000000000..be4c7e14e94 --- /dev/null +++ b/changes.d/5879.feat.md @@ -0,0 +1 @@ +`cylc lint` now warns of use of old templated items such as `%(suite)s` diff --git a/changes.d/5890.feat.md b/changes.d/5890.feat.md new file mode 100644 index 00000000000..5e6fce66c05 --- /dev/null +++ b/changes.d/5890.feat.md @@ -0,0 +1,2 @@ +Lint: Warn users that setting ``CYLC_VERSION``, ``ROSE_VERSION`` or +``FCM_VERSION`` in the workflow config is deprecated. \ No newline at end of file diff --git a/conda-environment.yml b/conda-environment.yml index 248e4dd21c0..93f5cda68fc 100644 --- a/conda-environment.yml +++ b/conda-environment.yml @@ -10,13 +10,13 @@ dependencies: # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat - jinja2 >=3.0,<3.1 - metomi-isodatetime >=1!3.0.0, <1!3.2.0 + - packaging # Constrain protobuf version for compatible Scheduler-UIS comms across hosts - - protobuf >=4.21.2,<4.22.0 + - protobuf >=4.24.4,<4.25.0 - psutil >=5.6.0 - python - pyzmq >=22 - - setuptools >=49,!=67.* - - importlib_metadata # [py<3.8] + - importlib_metadata >=5.0 # [py<3.12] - urwid >=2,<3 - tomli >=2 # [py<3.11] diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py index f6e90da3c53..d4192f72766 100644 --- a/cylc/flow/__init__.py +++ b/cylc/flow/__init__.py @@ -53,16 +53,24 @@ def environ_init(): environ_init() -__version__ = '8.2.4.dev' +__version__ = '8.3.0.dev' def iter_entry_points(entry_point_name): """Iterate over Cylc entry points.""" - import pkg_resources + import sys + if sys.version_info[:2] > (3, 11): + from importlib.metadata import entry_points + else: + # BACK COMPAT: importlib_metadata + # importlib.metadata was added in Python 3.8. The required interfaces + # were completed by 3.12. For lower versions we must use the + # importlib_metadata backport. + # FROM: Python 3.7 + # TO: Python: 3.12 + from importlib_metadata import entry_points yield from ( entry_point - for entry_point in pkg_resources.iter_entry_points(entry_point_name) - # Filter out the cylc namespace as it should be empty. - # All cylc packages should take the form cylc- - if entry_point.dist.key != 'cylc' + # for entry_point in entry_points()[entry_point_name] + for entry_point in entry_points().select(group=entry_point_name) ) diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 3c01ff52729..2d39ad74829 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -22,7 +22,7 @@ from typing import List, Optional, Tuple, Any, Union from contextlib import suppress -from pkg_resources import parse_version +from packaging.version import Version from cylc.flow import LOG from cylc.flow import __version__ as CYLC_VERSION @@ -610,6 +610,26 @@ def default_for( Prior to Cylc 8, ``global.cylc`` was named ``global.rc``, but that name is no longer supported. ''') as SPEC: + with Conf('hub', desc=''' + Configure the public URL of Jupyter Hub. + + If configured, the ``cylc gui`` command will open a web browser at this + location rather than starting a standalone server when called. + + + .. seealso:: + + * The cylc hub :ref:`architecture-reference` for fuller details. + * :ref:`UI_Server_config` for practical details. + + '''): + Conf('url', VDR.V_STRING, '', desc=''' + .. versionadded:: 8.3.0 + + Where Jupyter Hub is used a url can be provided for routing on + execution of ``cylc gui`` command. + ''') + with Conf('scheduler', desc=( default_for(SCHEDULER_DESCR, "[scheduler]", section=True) )): @@ -1202,6 +1222,9 @@ def default_for( {PLATFORM_REPLACES.format("[job]batch system")} ''') + replaces = PLATFORM_REPLACES.format( + "[job]batch submit command template" + ) Conf('job runner command template', VDR.V_STRING, desc=f''' Set the command used by the chosen job runner. @@ -1210,9 +1233,7 @@ def default_for( .. versionadded:: 8.0.0 - {PLATFORM_REPLACES.format( - "[job]batch submit command template" - )} + {replaces} ''') Conf('shell', VDR.V_STRING, '/bin/bash', desc=''' @@ -1445,6 +1466,8 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs command``. ''') + replaces = PLATFORM_REPLACES.format( + "[remote]retrieve job logs max size") Conf('retrieve job logs max size', VDR.V_STRING, desc=f''' {LOG_RETR_SETTINGS['retrieve job logs max size']} @@ -1452,9 +1475,10 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs max size``. - {PLATFORM_REPLACES.format( - "[remote]retrieve job logs max size")} + {replaces} ''') + replaces = PLATFORM_REPLACES.format( + "[remote]retrieve job logs retry delays") Conf('retrieve job logs retry delays', VDR.V_INTERVAL_LIST, desc=f''' {LOG_RETR_SETTINGS['retrieve job logs retry delays']} @@ -1463,8 +1487,7 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs retry delays``. - {PLATFORM_REPLACES.format( - "[remote]retrieve job logs retry delays")} + {replaces} ''') Conf('tail command template', VDR.V_STRING, 'tail -n +1 --follow=name %(filename)s', @@ -1632,6 +1655,14 @@ def default_for( .. versionadded:: 8.0.0 ''') + Conf('ssh forward environment variables', VDR.V_STRING_LIST, '', + desc=''' + A list containing the names of the environment variables to + forward with SSH connections to the workflow host from + the host running 'cylc play' + + .. versionadded:: 8.3.0 + ''') with Conf('selection', desc=''' How to select a host from the list of platform hosts. @@ -1838,8 +1869,7 @@ def get_version_hierarchy(version: str) -> List[str]: ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev'] """ - smart_ver: Any = parse_version(version) - # (No type anno. yet for Version in pkg_resources.extern.packaging.version) + smart_ver = Version(version) base = [str(i) for i in smart_ver.release] hierarchy = [''] hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)] diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index 101cc7132aa..de919c27c0f 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -401,7 +401,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): # differentiate between not set vs set to empty default = None elif item.endswith("handlers"): - desc = desc + '\n\n' + dedent(rf''' + desc = desc + '\n\n' + dedent(f''' Examples: .. code-block:: cylc @@ -413,9 +413,9 @@ def get_script_common_text(this: str, example: Optional[str] = None): {item} = echo %(workflow)s # configure multiple event handlers - {item} = \ - 'echo %(workflow)s, %(event)s', \ - 'my_exe %(event)s %(message)s' \ + {item} = \\ + 'echo %(workflow)s, %(event)s', \\ + 'my_exe %(event)s %(message)s' \\ 'curl -X PUT -d event=%(event)s host:port' ''') elif item.startswith("abort on"): @@ -1260,10 +1260,17 @@ def get_script_common_text(this: str, example: Optional[str] = None): - ``all`` - all instance of the task will fail - ``2017-08-12T06, 2017-08-12T18`` - these instances of the task will fail + + If you set :cylc:conf:`[..][..]execution retry delays` + the second attempt will succeed unless you set + :cylc:conf:`[..]fail try 1 only = False`. ''') Conf('fail try 1 only', VDR.V_BOOLEAN, True, desc=''' If ``True`` only the first run of the task instance will fail, otherwise retries will fail too. + + Task instances must be set to fail by + :cylc:conf:`[..]fail cycle points`. ''') Conf('disable task event handlers', VDR.V_BOOLEAN, True, desc=''' @@ -1849,7 +1856,7 @@ def upg(cfg, descr): ['scheduling', 'max active cycle points'], ['scheduling', 'runahead limit'], cvtr=converter( - lambda x: f'P{int(x)-1}' if x != '' else '', + lambda x: f'P{int(x) - 1}' if x != '' else '', '"{old}" -> "{new}"' ), silent=cylc.flow.flags.cylc7_back_compat, diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py index 81259e12a25..c4434dc9244 100644 --- a/cylc/flow/clean.py +++ b/cylc/flow/clean.py @@ -187,7 +187,7 @@ def init_clean(id_: str, opts: 'Values') -> None: if platform_names and platform_names != {'localhost'}: remote_clean( - id_, platform_names, opts.rm_dirs, opts.remote_timeout + id_, platform_names, opts.remote_timeout, opts.rm_dirs ) if not opts.remote_only: @@ -338,8 +338,8 @@ def _clean_using_glob( def remote_clean( id_: str, platform_names: Iterable[str], + timeout: str, rm_dirs: Optional[List[str]] = None, - timeout: str = '120' ) -> None: """Run subprocesses to clean a workflow on its remote install targets (skip localhost), given a set of platform names to look up. @@ -348,8 +348,9 @@ def remote_clean( id_: Workflow name. platform_names: List of platform names to look up in the global config, in order to determine the install targets to clean on. + timeout: ISO 8601 duration or number of seconds to wait before + cancelling. rm_dirs: Sub dirs to remove instead of the whole run dir. - timeout: Number of seconds to wait before cancelling. """ try: install_targets_map = ( @@ -358,6 +359,7 @@ def remote_clean( raise PlatformLookupError( f"Cannot clean {id_} on remote platforms as the workflow database " f"is out of date/inconsistent with the global config - {exc}") + queue: Deque[RemoteCleanQueueTuple] = deque() remote_clean_cmd = partial( _remote_clean_cmd, id_=id_, rm_dirs=rm_dirs, timeout=timeout @@ -376,7 +378,7 @@ def remote_clean( remote_clean_cmd(platform=platforms[0]), target, platforms ) ) - failed_targets: Dict[str, PlatformError] = {} + failed_targets: Dict[str, Union[PlatformError, str]] = {} # Handle subproc pool results almost concurrently: while queue: item = queue.popleft() @@ -387,7 +389,12 @@ def remote_clean( out, err = item.proc.communicate() if out: LOG.info(f"[{item.install_target}]\n{out}") - if ret_code: + if ret_code == 124: + failed_targets[item.install_target] = ( + f"cylc clean timed out after {timeout}s. You can increase " + "this timeout using the --timeout option." + ) + elif ret_code: this_platform = item.platforms.pop(0) excp = PlatformError( PlatformError.MSG_TIDY, @@ -415,9 +422,9 @@ def remote_clean( LOG.debug(f"[{item.install_target}]\n{err}") sleep(0.2) if failed_targets: - for target, excp in failed_targets.items(): + for target, info in failed_targets.items(): LOG.error( - f"Could not clean {id_} on install target: {target}\n{excp}" + f"Could not clean {id_} on install target: {target}\n{info}" ) raise CylcError(f"Remote clean failed for {id_}") diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 79d305da57a..d80456266bf 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -68,7 +68,7 @@ import cylc.flow.flags from cylc.flow.graph_parser import GraphParser from cylc.flow.listify import listify -from cylc.flow.option_parsers import verbosity_to_env +from cylc.flow.log_level import verbosity_to_env from cylc.flow.graphnode import GraphNodeParser from cylc.flow.param_expand import NameExpander from cylc.flow.parsec.exceptions import ItemNotFoundError @@ -79,8 +79,8 @@ get_cylc_run_dir, is_relative_to, ) -from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM from cylc.flow.print_tree import print_tree +from cylc.flow.simulation import configure_sim_modes from cylc.flow.subprocctx import SubFuncContext from cylc.flow.task_events_mgr import ( EventData, @@ -521,7 +521,8 @@ def __init__( self.process_runahead_limit() if self.run_mode('simulation', 'dummy'): - self.configure_sim_modes() + configure_sim_modes( + self.taskdefs.values(), self.run_mode()) self.configure_workflow_state_polling_tasks() @@ -1340,68 +1341,6 @@ def configure_workflow_state_polling_tasks(self): script = "echo " + comstr + "\n" + comstr rtc['script'] = script - def configure_sim_modes(self): - """Adjust task defs for simulation and dummy mode.""" - for tdef in self.taskdefs.values(): - # Compute simulated run time by scaling the execution limit. - rtc = tdef.rtconfig - limit = rtc['execution time limit'] - speedup = rtc['simulation']['speedup factor'] - if limit and speedup: - sleep_sec = (DurationParser().parse( - str(limit)).get_seconds() / speedup) - else: - sleep_sec = DurationParser().parse( - str(rtc['simulation']['default run length']) - ).get_seconds() - rtc['execution time limit'] = ( - sleep_sec + DurationParser().parse(str( - rtc['simulation']['time limit buffer'])).get_seconds() - ) - rtc['job']['simulated run length'] = sleep_sec - - # Generate dummy scripting. - rtc['init-script'] = "" - rtc['env-script'] = "" - rtc['pre-script'] = "" - rtc['post-script'] = "" - scr = "sleep %d" % sleep_sec - # Dummy message outputs. - for msg in rtc['outputs'].values(): - scr += "\ncylc message '%s'" % msg - if rtc['simulation']['fail try 1 only']: - arg1 = "true" - else: - arg1 = "false" - arg2 = " ".join(rtc['simulation']['fail cycle points']) - scr += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2) - rtc['script'] = scr - - # Dummy mode jobs should run on platform localhost - # All Cylc 7 config items which conflict with platform are removed. - for section, keys in FORBIDDEN_WITH_PLATFORM.items(): - if section in rtc: - for key in keys: - if key in rtc[section]: - rtc[section][key] = None - - rtc['platform'] = 'localhost' - - # Disable environment, in case it depends on env-script. - rtc['environment'] = {} - - # Simulation mode tasks should fail in which cycle points? - f_pts = [] - f_pts_orig = rtc['simulation']['fail cycle points'] - if 'all' in f_pts_orig: - # None for "fail all points". - f_pts = None - else: - # (And [] for "fail no points".) - for point_str in f_pts_orig: - f_pts.append(get_point(point_str).standardise()) - rtc['simulation']['fail cycle points'] = f_pts - def get_parent_lists(self): return self.runtime['parents'] diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py index 655c06c2881..a66ce3f5ba0 100644 --- a/cylc/flow/cycling/iso8601.py +++ b/cylc/flow/cycling/iso8601.py @@ -88,7 +88,9 @@ def from_nonstandard_string(cls, point_string): def add(self, other): """Add an Interval to self.""" - return ISO8601Point(self._iso_point_add(self.value, other.value)) + return ISO8601Point(self._iso_point_add( + self.value, other.value, CALENDAR.mode + )) def standardise(self): """Reformat self.value into a standard representation.""" @@ -106,25 +108,27 @@ def standardise(self): def sub(self, other): """Subtract a Point or Interval from self.""" if isinstance(other, ISO8601Point): - return ISO8601Interval( - self._iso_point_sub_point(self.value, other.value)) - return ISO8601Point( - self._iso_point_sub_interval(self.value, other.value)) + return ISO8601Interval(self._iso_point_sub_point( + self.value, other.value, CALENDAR.mode + )) + return ISO8601Point(self._iso_point_sub_interval( + self.value, other.value, CALENDAR.mode + )) @staticmethod @lru_cache(10000) - def _iso_point_add(point_string, interval_string): + def _iso_point_add(point_string, interval_string, _calendar_mode): """Add the parsed point_string to the parsed interval_string.""" point = point_parse(point_string) interval = interval_parse(interval_string) return str(point + interval) def _cmp(self, other: 'ISO8601Point') -> int: - return self._iso_point_cmp(self.value, other.value) + return self._iso_point_cmp(self.value, other.value, CALENDAR.mode) @staticmethod @lru_cache(10000) - def _iso_point_cmp(point_string, other_point_string): + def _iso_point_cmp(point_string, other_point_string, _calendar_mode): """Compare the parsed point_string to the other one.""" point = point_parse(point_string) other_point = point_parse(other_point_string) @@ -132,7 +136,7 @@ def _iso_point_cmp(point_string, other_point_string): @staticmethod @lru_cache(10000) - def _iso_point_sub_interval(point_string, interval_string): + def _iso_point_sub_interval(point_string, interval_string, _calendar_mode): """Return the parsed point_string minus the parsed interval_string.""" point = point_parse(point_string) interval = interval_parse(interval_string) @@ -140,7 +144,7 @@ def _iso_point_sub_interval(point_string, interval_string): @staticmethod @lru_cache(10000) - def _iso_point_sub_point(point_string, other_point_string): + def _iso_point_sub_point(point_string, other_point_string, _calendar_mode): """Return the difference between the two parsed point strings.""" point = point_parse(point_string) other_point = point_parse(other_point_string) diff --git a/cylc/flow/data_messages.proto b/cylc/flow/data_messages.proto index e448d51e5b1..6068bb1c5df 100644 --- a/cylc/flow/data_messages.proto +++ b/cylc/flow/data_messages.proto @@ -132,6 +132,7 @@ message PbRuntime { // Nodes message PbJob { + reserved 29; /* see https://github.com/cylc/cylc-flow/pull/5672 */ optional string stamp = 1; optional string id = 2; optional int32 submit_num = 3; @@ -145,7 +146,6 @@ message PbJob { optional float execution_time_limit = 14; optional string platform = 15; optional string job_log_dir = 17; - repeated string extra_logs = 29; optional string name = 30; /* filter item */ optional string cycle_point = 31; /* filter item */ repeated string messages = 32; diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py index 311a84f8958..82c620bcacf 100644 --- a/cylc/flow/data_messages_pb2.py +++ b/cylc/flow/data_messages_pb2.py @@ -3,10 +3,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: data_messages.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -14,87 +14,87 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\xab\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x12\n\nextra_logs\x18\x1d \x03(\t\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtime\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _PBWORKFLOW_STATETOTALSENTRY._options = None - _PBWORKFLOW_STATETOTALSENTRY._serialized_options = b'8\001' - _PBWORKFLOW_LATESTSTATETASKSENTRY._options = None - _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_options = b'8\001' - _PBTASKPROXY_OUTPUTSENTRY._options = None - _PBTASKPROXY_OUTPUTSENTRY._serialized_options = b'8\001' - _PBTASKPROXY_EXTERNALTRIGGERSENTRY._options = None - _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_options = b'8\001' - _PBTASKPROXY_XTRIGGERSENTRY._options = None - _PBTASKPROXY_XTRIGGERSENTRY._serialized_options = b'8\001' - _PBFAMILYPROXY_STATETOTALSENTRY._options = None - _PBFAMILYPROXY_STATETOTALSENTRY._serialized_options = b'8\001' - _PBMETA._serialized_start=24 - _PBMETA._serialized_end=174 - _PBTIMEZONE._serialized_start=177 - _PBTIMEZONE._serialized_end=347 - _PBTASKPROXYREFS._serialized_start=349 - _PBTASKPROXYREFS._serialized_end=388 - _PBWORKFLOW._serialized_start=391 - _PBWORKFLOW._serialized_end=2011 - _PBWORKFLOW_STATETOTALSENTRY._serialized_start=1441 - _PBWORKFLOW_STATETOTALSENTRY._serialized_end=1491 - _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_start=1493 - _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_end=1566 - _PBRUNTIME._serialized_start=2014 - _PBRUNTIME._serialized_end=2839 - _PBJOB._serialized_start=2842 - _PBJOB._serialized_end=3525 - _PBTASK._serialized_start=3528 - _PBTASK._serialized_end=3882 - _PBPOLLTASK._serialized_start=3885 - _PBPOLLTASK._serialized_end=4101 - _PBCONDITION._serialized_start=4104 - _PBCONDITION._serialized_end=4307 - _PBPREREQUISITE._serialized_start=4310 - _PBPREREQUISITE._serialized_end=4460 - _PBOUTPUT._serialized_start=4463 - _PBOUTPUT._serialized_end=4603 - _PBTRIGGER._serialized_start=4606 - _PBTRIGGER._serialized_end=4771 - _PBTASKPROXY._serialized_start=4774 - _PBTASKPROXY._serialized_end=5815 - _PBTASKPROXY_OUTPUTSENTRY._serialized_start=5425 - _PBTASKPROXY_OUTPUTSENTRY._serialized_end=5482 - _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_start=5484 - _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_end=5551 - _PBTASKPROXY_XTRIGGERSENTRY._serialized_start=5553 - _PBTASKPROXY_XTRIGGERSENTRY._serialized_end=5613 - _PBFAMILY._serialized_start=5818 - _PBFAMILY._serialized_end=6146 - _PBFAMILYPROXY._serialized_start=6149 - _PBFAMILYPROXY._serialized_end=6963 - _PBFAMILYPROXY_STATETOTALSENTRY._serialized_start=1441 - _PBFAMILYPROXY_STATETOTALSENTRY._serialized_end=1491 - _PBEDGE._serialized_start=6966 - _PBEDGE._serialized_end=7154 - _PBEDGES._serialized_start=7156 - _PBEDGES._serialized_end=7279 - _PBENTIREWORKFLOW._serialized_start=7282 - _PBENTIREWORKFLOW._serialized_end=7524 - _EDELTAS._serialized_start=7527 - _EDELTAS._serialized_end=7702 - _FDELTAS._serialized_start=7705 - _FDELTAS._serialized_end=7884 - _FPDELTAS._serialized_start=7887 - _FPDELTAS._serialized_end=8077 - _JDELTAS._serialized_start=8080 - _JDELTAS._serialized_end=8253 - _TDELTAS._serialized_start=8256 - _TDELTAS._serialized_end=8431 - _TPDELTAS._serialized_start=8434 - _TPDELTAS._serialized_end=8620 - _WDELTAS._serialized_start=8623 - _WDELTAS._serialized_end=8818 - _ALLDELTAS._serialized_start=8821 - _ALLDELTAS._serialized_end=9030 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._options = None + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_options = b'8\001' + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._options = None + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_options = b'8\001' + _globals['_PBTASKPROXY_OUTPUTSENTRY']._options = None + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_options = b'8\001' + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._options = None + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_options = b'8\001' + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._options = None + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_options = b'8\001' + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._options = None + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_options = b'8\001' + _globals['_PBMETA']._serialized_start=24 + _globals['_PBMETA']._serialized_end=174 + _globals['_PBTIMEZONE']._serialized_start=177 + _globals['_PBTIMEZONE']._serialized_end=347 + _globals['_PBTASKPROXYREFS']._serialized_start=349 + _globals['_PBTASKPROXYREFS']._serialized_end=388 + _globals['_PBWORKFLOW']._serialized_start=391 + _globals['_PBWORKFLOW']._serialized_end=2011 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_start=1441 + _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_end=1491 + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=1493 + _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=1566 + _globals['_PBRUNTIME']._serialized_start=2014 + _globals['_PBRUNTIME']._serialized_end=2839 + _globals['_PBJOB']._serialized_start=2842 + _globals['_PBJOB']._serialized_end=3511 + _globals['_PBTASK']._serialized_start=3514 + _globals['_PBTASK']._serialized_end=3868 + _globals['_PBPOLLTASK']._serialized_start=3871 + _globals['_PBPOLLTASK']._serialized_end=4087 + _globals['_PBCONDITION']._serialized_start=4090 + _globals['_PBCONDITION']._serialized_end=4293 + _globals['_PBPREREQUISITE']._serialized_start=4296 + _globals['_PBPREREQUISITE']._serialized_end=4446 + _globals['_PBOUTPUT']._serialized_start=4449 + _globals['_PBOUTPUT']._serialized_end=4589 + _globals['_PBTRIGGER']._serialized_start=4592 + _globals['_PBTRIGGER']._serialized_end=4757 + _globals['_PBTASKPROXY']._serialized_start=4760 + _globals['_PBTASKPROXY']._serialized_end=5801 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5411 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5468 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5470 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5537 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5539 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5599 + _globals['_PBFAMILY']._serialized_start=5804 + _globals['_PBFAMILY']._serialized_end=6132 + _globals['_PBFAMILYPROXY']._serialized_start=6135 + _globals['_PBFAMILYPROXY']._serialized_end=6949 + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1441 + _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1491 + _globals['_PBEDGE']._serialized_start=6952 + _globals['_PBEDGE']._serialized_end=7140 + _globals['_PBEDGES']._serialized_start=7142 + _globals['_PBEDGES']._serialized_end=7265 + _globals['_PBENTIREWORKFLOW']._serialized_start=7268 + _globals['_PBENTIREWORKFLOW']._serialized_end=7510 + _globals['_EDELTAS']._serialized_start=7513 + _globals['_EDELTAS']._serialized_end=7688 + _globals['_FDELTAS']._serialized_start=7691 + _globals['_FDELTAS']._serialized_end=7870 + _globals['_FPDELTAS']._serialized_start=7873 + _globals['_FPDELTAS']._serialized_end=8063 + _globals['_JDELTAS']._serialized_start=8066 + _globals['_JDELTAS']._serialized_end=8239 + _globals['_TDELTAS']._serialized_start=8242 + _globals['_TDELTAS']._serialized_end=8417 + _globals['_TPDELTAS']._serialized_start=8420 + _globals['_TPDELTAS']._serialized_end=8606 + _globals['_WDELTAS']._serialized_start=8609 + _globals['_WDELTAS']._serialized_end=8804 + _globals['_ALLDELTAS']._serialized_start=8807 + _globals['_ALLDELTAS']._serialized_end=9016 # @@protoc_insertion_point(module_scope) diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py index 94354880349..ef77105b3a4 100644 --- a/cylc/flow/data_store_mgr.py +++ b/cylc/flow/data_store_mgr.py @@ -1587,7 +1587,7 @@ def insert_job(self, name, cycle_point, status, job_conf): name=tproxy.name, cycle_point=tproxy.cycle_point, execution_time_limit=job_conf.get('execution_time_limit'), - platform=job_conf.get('platform')['name'], + platform=job_conf['platform']['name'], job_runner_name=job_conf.get('job_runner_name'), ) # Not all fields are populated with some submit-failures, @@ -1603,7 +1603,6 @@ def insert_job(self, name, cycle_point, status, job_conf): # Add in log files. j_buf.job_log_dir = get_task_job_log( self.schd.workflow, tproxy.cycle_point, tproxy.name, sub_num) - j_buf.extra_logs.extend(job_conf.get('logfiles', [])) self.added[JOBS][j_id] = j_buf getattr(self.updated[WORKFLOW], JOBS).append(j_id) diff --git a/cylc/flow/etc/syntax/cylc.lang b/cylc/flow/etc/syntax/cylc.lang index 2de9106f360..0270c7f3cfc 100644 --- a/cylc/flow/etc/syntax/cylc.lang +++ b/cylc/flow/etc/syntax/cylc.lang @@ -87,35 +87,37 @@ --> xtriggers + workflow timeout handlers + workflow timeout workflow state polling work sub-directory - warning handler + warning handlers verbose mode user to title - workflow timeout handler - workflow timeout time limit buffer templates task parameters task event batch interval - succeeded handler - submitted handler - submission timeout handler + succeeded handlers + submitted handlers + submission timeout handlers submission timeout - submission retry handler + submission retry handlers submission retry delays submission polling intervals - submission failed handler + submission failed handlers stop after cycle point - startup handler - started handler - stall handler + startup handlers + started handlers + stall timeout handlers + stall timeout + stall handlers speedup factor special tasks simulation - shutdown handler + shutdown handlers sequential script scheduling @@ -123,10 +125,11 @@ runtime runahead limit run-dir - retry handler + retry handlers retrieve job logs retry delays retrieve job logs max size retrieve job logs + restart timeout remote queues pre-script @@ -144,7 +147,7 @@ mail limit late offset - late handler + late handlers job interval install @@ -153,7 +156,7 @@ init-script inherit include - inactivity timeout handler + inactivity timeout handlers inactivity timeout host hold after cycle point @@ -166,14 +169,14 @@ flow.cylc final cycle point constraints final cycle point - failed handler + failed handlers fail try 1 only fail cycle points external-trigger - expired handler + expired handlers expected task failures exit-script - execution timeout handler + execution timeout handlers execution timeout execution time limit execution retry delays @@ -184,7 +187,6 @@ environment filter environment env-script - edge penwidth disable task event handlers directives description @@ -194,17 +196,17 @@ cycle point time zone cycle point num expanded year digits cycle point format - custom handler - critical handler + custom handlers + critical handlers clock-trigger clock-expire batch system batch submit command template allow implicit tasks - aborted handler abort on workflow timeout abort on stall timeout abort on inactivity timeout + abort handlers UTC mode URL diff --git a/cylc/flow/etc/syntax/cylc.xml b/cylc/flow/etc/syntax/cylc.xml index 9f3a90287e8..da11c1215e1 100644 --- a/cylc/flow/etc/syntax/cylc.xml +++ b/cylc/flow/etc/syntax/cylc.xml @@ -14,35 +14,37 @@ The sort is in reverse order to allow longer keywords to take precedence over sub-sets. --> + + - + - - - - - + + + - + - + - - - + + + + + - + @@ -50,10 +52,11 @@ - + + @@ -71,7 +74,7 @@ - + @@ -80,8 +83,8 @@ - - + + @@ -93,14 +96,14 @@ - + - + - + @@ -120,17 +123,17 @@ - - + + - - + + diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings index 15de1f8ea13..53b4aa2c17a 100644 --- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings +++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings @@ -7,6 +7,5 @@ [[[environment]]] # These environment variables ensure that tasks can # run in the same environment as the workflow: - {% from "sys" import path, executable %} - PYTHONPATH = {{':'.join(path)}} + {% from "sys" import executable %} PATH = $(dirname {{executable}}):$PATH diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py index 79a726d7bbe..d1a459f996b 100644 --- a/cylc/flow/exceptions.py +++ b/cylc/flow/exceptions.py @@ -26,11 +26,14 @@ Tuple, Type, Union, + TYPE_CHECKING, ) -from cylc.flow.subprocctx import SubFuncContext from cylc.flow.util import format_cmd +if TYPE_CHECKING: + from cylc.flow.subprocctx import SubFuncContext + class CylcError(Exception): """Generic exception for Cylc errors. @@ -198,7 +201,7 @@ def __init__( message: str, platform_name: str, *, - ctx: Optional[SubFuncContext] = None, + ctx: 'Optional[SubFuncContext]' = None, cmd: Optional[Union[str, Iterable]] = None, ret_code: Optional[int] = None, out: Optional[str] = None, diff --git a/cylc/flow/flow_mgr.py b/cylc/flow/flow_mgr.py index 148adb8213f..41aec947a80 100644 --- a/cylc/flow/flow_mgr.py +++ b/cylc/flow/flow_mgr.py @@ -16,11 +16,14 @@ """Manage flow counter and flow metadata.""" -from typing import Dict, Set, Optional +from typing import Dict, Set, Optional, TYPE_CHECKING import datetime from cylc.flow import LOG -from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager + + +if TYPE_CHECKING: + from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager FlowNums = Set[int] diff --git a/cylc/flow/id.py b/cylc/flow/id.py index b8f34fb217f..222d16f82b6 100644 --- a/cylc/flow/id.py +++ b/cylc/flow/id.py @@ -497,7 +497,7 @@ def duplicate( )? (?: # cycle/task/job - { RELATIVE_PATTERN } + {RELATIVE_PATTERN} )? )? )? diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py index 0d1e8a8a102..b47ba739bea 100644 --- a/cylc/flow/id_cli.py +++ b/cylc/flow/id_cli.py @@ -31,11 +31,6 @@ upgrade_legacy_ids, ) from cylc.flow.pathutil import EXPLICIT_RELATIVE_PATH_REGEX -from cylc.flow.network.scan import ( - filter_name, - is_active, - scan, -) from cylc.flow.workflow_files import ( check_flow_file, detect_both_flow_and_suite, @@ -493,6 +488,12 @@ async def _expand_workflow_tokens_impl(tokens, match_active=True): 'currently supported.' ) + # import only when needed to avoid slowing CLI unnecessarily + from cylc.flow.network.scan import ( + filter_name, + is_active, + scan, + ) # construct the pipe pipe = scan | filter_name(fnmatch.translate(tokens['workflow'])) if match_active is not None: diff --git a/cylc/flow/job_file.py b/cylc/flow/job_file.py index cc8576dad37..930331dc5a4 100644 --- a/cylc/flow/job_file.py +++ b/cylc/flow/job_file.py @@ -25,16 +25,9 @@ from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.job_runner_mgr import JobRunnerManager import cylc.flow.flags -from cylc.flow.option_parsers import verbosity_to_env +from cylc.flow.log_level import verbosity_to_env from cylc.flow.config import interpolate_template, ParamExpandError -# the maximum number of task dependencies which Cylc will list before -# omitting the CYLC_TASK_DEPENDENCIES environment variable -# see: https://github.com/cylc/cylc-flow/issues/5551 -# NOTE: please update `src/reference/job-script-vars/var-list.txt` -# in cylc-doc if changing this value -MAX_CYLC_TASK_DEPENDENCIES_LEN = 50 - class JobFileWriter: @@ -227,18 +220,6 @@ def _write_task_environment(self, handle, job_conf): handle.write( '\n export CYLC_TASK_NAMESPACE_HIERARCHY="%s"' % ' '.join(job_conf['namespace_hierarchy'])) - if len(job_conf['dependencies']) <= MAX_CYLC_TASK_DEPENDENCIES_LEN: - handle.write( - '\n export CYLC_TASK_DEPENDENCIES="%s"' % - ' '.join(job_conf['dependencies'])) - else: - # redact the CYLC_TASK_DEPENDENCIES variable but leave a note - # explaining why - # see: https://github.com/cylc/cylc-flow/issues/5551 - handle.write( - '\n # CYLC_TASK_DEPENDENCIES=disabled' - f' (more than {MAX_CYLC_TASK_DEPENDENCIES_LEN} dependencies)' - ) handle.write( '\n export CYLC_TASK_TRY_NUMBER=%s' % job_conf['try_num']) handle.write( diff --git a/cylc/flow/job_runner_handlers/documentation.py b/cylc/flow/job_runner_handlers/documentation.py index 735b1801b16..a6f6af983be 100644 --- a/cylc/flow/job_runner_handlers/documentation.py +++ b/cylc/flow/job_runner_handlers/documentation.py @@ -22,13 +22,16 @@ not intended to be subclassed. """ -import re from typing import ( Iterable, List, Tuple, + TYPE_CHECKING, ) +if TYPE_CHECKING: + import re + class ExampleHandler(): """Documentation for writing job runner handlers. @@ -110,7 +113,6 @@ class MyHandler(): * ``job_file_path`` * ``job_runner_command_template`` * ``job_runner_name`` - * ``logfiles`` * ``namespace_hierarchy`` * ``param_var`` * ``platform`` @@ -258,7 +260,7 @@ class QSUBHandler(PBSHandler): """ - REC_ID_FROM_SUBMIT_OUT: re.Pattern + REC_ID_FROM_SUBMIT_OUT: 're.Pattern' """Regular expression to extract job IDs from submission stderr. A regular expression (compiled) to extract the job "id" from the standard @@ -266,7 +268,7 @@ class QSUBHandler(PBSHandler): """ - REC_ID_FROM_SUBMIT_ERR: re.Pattern + REC_ID_FROM_SUBMIT_ERR: 're.Pattern' """Regular expression to extract job IDs from submission stderr. See :py:attr:`ExampleHandler.REC_ID_FROM_SUBMIT_OUT`. diff --git a/cylc/flow/log_level.py b/cylc/flow/log_level.py new file mode 100644 index 00000000000..cda3c4493ad --- /dev/null +++ b/cylc/flow/log_level.py @@ -0,0 +1,116 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Utilities for configuring logging level via the CLI.""" + +import logging +from typing import List, Dict, Union, TYPE_CHECKING + +if TYPE_CHECKING: + import os + + +def verbosity_to_log_level(verb: int) -> int: + """Convert Cylc verbosity to log severity level.""" + if verb < 0: + return logging.WARNING + if verb > 0: + return logging.DEBUG + return logging.INFO + + +def log_level_to_verbosity(lvl: int) -> int: + """Convert log severity level to Cylc verbosity. + + Examples: + >>> log_level_to_verbosity(logging.NOTSET) + 2 + >>> log_level_to_verbosity(logging.DEBUG) + 2 + >>> log_level_to_verbosity(logging.INFO) + 0 + >>> log_level_to_verbosity(logging.WARNING) + -1 + >>> log_level_to_verbosity(logging.ERROR) + -1 + """ + if lvl <= logging.DEBUG: + return 2 + if lvl < logging.INFO: + return 1 + if lvl == logging.INFO: + return 0 + return -1 + + +def verbosity_to_opts(verb: int) -> List[str]: + """Convert Cylc verbosity to the CLI opts required to replicate it. + + Examples: + >>> verbosity_to_opts(0) + [] + >>> verbosity_to_opts(-2) + ['-q', '-q'] + >>> verbosity_to_opts(2) + ['-v', '-v'] + + """ + return [ + '-q' + for _ in range(verb, 0) + ] + [ + '-v' + for _ in range(0, verb) + ] + + +def verbosity_to_env(verb: int) -> Dict[str, str]: + """Convert Cylc verbosity to the env vars required to replicate it. + + Examples: + >>> verbosity_to_env(0) + {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'} + >>> verbosity_to_env(1) + {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'} + >>> verbosity_to_env(2) + {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'} + + """ + return { + 'CYLC_VERBOSE': str((verb > 0)).lower(), + 'CYLC_DEBUG': str((verb > 1)).lower(), + } + + +def env_to_verbosity(env: 'Union[Dict, os._Environ]') -> int: + """Extract verbosity from environment variables. + + Examples: + >>> env_to_verbosity({}) + 0 + >>> env_to_verbosity({'CYLC_VERBOSE': 'true'}) + 1 + >>> env_to_verbosity({'CYLC_DEBUG': 'true'}) + 2 + >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'}) + 2 + + """ + return ( + 2 if env.get('CYLC_DEBUG', '').lower() == 'true' + else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true' + else 0 + ) diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py index f8b499332e9..b3955d3b058 100644 --- a/cylc/flow/network/client.py +++ b/cylc/flow/network/client.py @@ -143,6 +143,10 @@ def timeout_handler(self) -> None: f'It has moved to {contact_host}:{contact_port}' ) + if os.getenv('CYLC_TASK_COMMS_METHOD'): + # don't attempt to clean up old contact files in task messages + return + # Cannot connect, perhaps workflow is no longer running and is leaving # behind a contact file? try: @@ -299,7 +303,9 @@ async def async_request( raise ClientTimeout( 'Timeout waiting for server response.' ' This could be due to network or server issues.' - ' Check the workflow log.' + '\n* You might want to increase the timeout using the' + ' --comms-timeout option;' + '\n* or check the workflow log.' ) if msg['command'] in PB_METHOD_MAP: diff --git a/cylc/flow/network/multi.py b/cylc/flow/network/multi.py index 86c89c5f71d..a11842d6391 100644 --- a/cylc/flow/network/multi.py +++ b/cylc/flow/network/multi.py @@ -107,4 +107,4 @@ def _report_single(report, workflow, result): def _report(_): - print('Done') + print('Command submitted; the scheduler will log any problems.') diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index 54222a510c7..c2202f3f31e 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -50,10 +50,8 @@ import re from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union -from pkg_resources import ( - parse_requirements, - parse_version -) +from packaging.version import parse as parse_version +from packaging.specifiers import SpecifierSet from cylc.flow import LOG from cylc.flow.async_util import ( @@ -354,11 +352,7 @@ async def validate_contact_info(flow): def parse_requirement(requirement_string): """Parse a requirement from a requirement string.""" - # we have to give the requirement a name but what we call it doesn't - # actually matter - for req in parse_requirements(f'x {requirement_string}'): - # there should only be one requirement - return (req,), {} + return (SpecifierSet(requirement_string),), {} @pipe(preproc=parse_requirement) @@ -373,7 +367,7 @@ async def cylc_version(flow, requirement): flow (dict): Flow information dictionary, provided by scan through the pipe. requirement (str): - Requirement specifier in pkg_resources format e.g. ``> 8, < 9`` + Requirement specifier in PEP 440 format e.g. ``> 8, < 9`` """ return parse_version(flow[ContactFileFields.VERSION]) in requirement @@ -391,7 +385,7 @@ async def api_version(flow, requirement): flow (dict): Flow information dictionary, provided by scan through the pipe. requirement (str): - Requirement specifier in pkg_resources format e.g. ``> 8, < 9`` + Requirement specifier in PEP 440 format e.g. ``> 8, < 9`` """ return parse_version(flow[ContactFileFields.API]) in requirement diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index 458fd1f8168..f9ed95c7158 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -893,11 +893,6 @@ class Meta: job_log_dir = String( description="The path to the job's log directory.", ) - extra_logs = graphene.List( - # TODO: remove. see https://github.com/cylc/cylc-flow/issues/5610 - String, - description='Obsolete, do not use.', - ) messages = graphene.List( String, description='The list of task messages generated by this job.', diff --git a/cylc/flow/network/server.py b/cylc/flow/network/server.py index f6847dcef59..5c070472025 100644 --- a/cylc/flow/network/server.py +++ b/cylc/flow/network/server.py @@ -21,7 +21,6 @@ from time import sleep from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union -from graphql.execution import ExecutionResult from graphql.execution.executors.asyncio import AsyncioExecutor import zmq from zmq.auth.thread import ThreadAuthenticator @@ -41,6 +40,7 @@ if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler + from graphql.execution import ExecutionResult # maps server methods to the protobuf message (for client/UIS import) @@ -368,7 +368,7 @@ def graphql( object: Execution result, or a list with errors. """ try: - executed: ExecutionResult = schema.execute( + executed: 'ExecutionResult' = schema.execute( request_string, variable_values=variables, context_value={ diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py index 3bf59720d21..b83ff45aab9 100644 --- a/cylc/flow/option_parsers.py +++ b/cylc/flow/option_parsers.py @@ -33,7 +33,7 @@ import sys from textwrap import dedent -from typing import Any, Dict, Optional, List, Tuple, Union +from typing import Any, Dict, Optional, List, Tuple from cylc.flow import LOG from cylc.flow.terminal import supports_color, DIM @@ -42,8 +42,13 @@ CylcLogFormatter, setup_segregated_log_streams, ) +from cylc.flow.log_level import ( + env_to_verbosity, + verbosity_to_log_level +) WORKFLOW_ID_ARG_DOC = ('WORKFLOW', 'Workflow ID') +OPT_WORKFLOW_ID_ARG_DOC = ('[WORKFLOW]', 'Workflow ID') WORKFLOW_ID_MULTI_ARG_DOC = ('WORKFLOW ...', 'Workflow ID(s)') WORKFLOW_ID_OR_PATH_ARG_DOC = ('WORKFLOW | PATH', 'Workflow ID or path') ID_MULTI_ARG_DOC = ('ID ...', 'Workflow/Cycle/Family/Task ID(s)') @@ -177,99 +182,6 @@ def format_help_headings(string): ) -def verbosity_to_log_level(verb: int) -> int: - """Convert Cylc verbosity to log severity level.""" - if verb < 0: - return logging.WARNING - if verb > 0: - return logging.DEBUG - return logging.INFO - - -def log_level_to_verbosity(lvl: int) -> int: - """Convert log severity level to Cylc verbosity. - - Examples: - >>> log_level_to_verbosity(logging.NOTSET) - 2 - >>> log_level_to_verbosity(logging.DEBUG) - 2 - >>> log_level_to_verbosity(logging.INFO) - 0 - >>> log_level_to_verbosity(logging.WARNING) - -1 - >>> log_level_to_verbosity(logging.ERROR) - -1 - """ - if lvl <= logging.DEBUG: - return 2 - if lvl < logging.INFO: - return 1 - if lvl == logging.INFO: - return 0 - return -1 - - -def verbosity_to_opts(verb: int) -> List[str]: - """Convert Cylc verbosity to the CLI opts required to replicate it. - - Examples: - >>> verbosity_to_opts(0) - [] - >>> verbosity_to_opts(-2) - ['-q', '-q'] - >>> verbosity_to_opts(2) - ['-v', '-v'] - - """ - return [ - '-q' - for _ in range(verb, 0) - ] + [ - '-v' - for _ in range(0, verb) - ] - - -def verbosity_to_env(verb: int) -> Dict[str, str]: - """Convert Cylc verbosity to the env vars required to replicate it. - - Examples: - >>> verbosity_to_env(0) - {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'} - >>> verbosity_to_env(1) - {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'} - >>> verbosity_to_env(2) - {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'} - - """ - return { - 'CYLC_VERBOSE': str((verb > 0)).lower(), - 'CYLC_DEBUG': str((verb > 1)).lower(), - } - - -def env_to_verbosity(env: Union[Dict, os._Environ]) -> int: - """Extract verbosity from environment variables. - - Examples: - >>> env_to_verbosity({}) - 0 - >>> env_to_verbosity({'CYLC_VERBOSE': 'true'}) - 1 - >>> env_to_verbosity({'CYLC_DEBUG': 'true'}) - 2 - >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'}) - 2 - - """ - return ( - 2 if env.get('CYLC_DEBUG', '').lower() == 'true' - else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true' - else 0 - ) - - class CylcOption(Option): """Optparse option which adds a decrement action.""" diff --git a/cylc/flow/parsec/empysupport.py b/cylc/flow/parsec/empysupport.py index 7bc8a69e3c4..b4164894e0f 100644 --- a/cylc/flow/parsec/empysupport.py +++ b/cylc/flow/parsec/empysupport.py @@ -24,6 +24,7 @@ import typing as t from cylc.flow.parsec.exceptions import EmPyError +from cylc.flow.parsec.fileparse import get_cylc_env_vars def empyprocess( @@ -52,6 +53,12 @@ def empyprocess( ftempl = StringIO('\n'.join(flines)) xtempl = StringIO() interpreter = em.Interpreter(output=em.UncloseableFile(xtempl)) + + # Add `CYLC_` environment variables to the global namespace. + interpreter.updateGlobals( + get_cylc_env_vars() + ) + try: interpreter.file(ftempl, '