Merge branch 'master' into 8.2.x-sync

wxtim · Jan 9, 2024 · 8002595 · 8002595
2 parents b64fcd6 + df350cd
commit 8002595
Show file tree

Hide file tree

Showing 220 changed files with 7,120 additions and 2,150 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -17,18 +17,19 @@ coverage:
     project:
       default:
         # commits below this threshold will be marked as failed
-        target: '85%'
+        target: '92%'
         # how much we allow the coverage to drop
         threshold: '2%'
     patch:
       default:
-        target: '95%'
+        target: '97%'
         threshold: '5%'
 
 # files to ignore
 ignore:
   - "tests/**"
   - "ws_messages_pb2.py"
+  - "cylc/flow/scripts/report_timings.py"
 
 flag_management:
   default_rules:

diff --git a/.coveragerc b/.coveragerc
@@ -16,6 +16,7 @@ omit =
     tests/*
     */cylc/flow/*_pb2.py
     cylc/flow/etc/*
+    cylc/flow/scripts/report_timings.py
 parallel = True
 source = ./cylc
 timid = False

diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml
@@ -21,7 +21,7 @@ jobs:
         uses: cylc/release-actions/stage-1/sanitize-inputs@v1
 
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE_REF }}
           fetch-depth: 0  # need to fetch all commits to check contributors
@@ -30,7 +30,7 @@ jobs:
         uses: cylc/release-actions/check-shortlog@v1
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
@@ -22,12 +22,12 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ env.MERGE_SHA }}
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
 
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/[email protected].8
+      uses: pypa/[email protected].11
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml
@@ -44,7 +44,7 @@ jobs:
           - '5.0'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Docker container
         run: |
@@ -85,7 +85,7 @@ jobs:
 
       - name: Upload artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: Upload cylc-run artifact
+          name: 'cylc-run (bash-${{ matrix.bash-version }})'
           path: cylc-run
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,10 +24,10 @@ jobs:
             python: '3.7'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 

diff --git a/.github/workflows/shortlog.yml b/.github/workflows/shortlog.yml
@@ -14,7 +14,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # need to fetch all commits to check contributors
 

diff --git a/.github/workflows/test_conda-build.yml b/.github/workflows/test_conda-build.yml
@@ -19,7 +19,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: checkout cylc-flow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: build conda env
         run: |

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
@@ -17,35 +17,38 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
-      fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
+      fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.10', '3.11', '3']
         include:
           # mac os test
           - os: 'macos-11'
             python-version: '3.7'  # oldest supported version
+
           # non-utc timezone test
           - os: 'ubuntu-latest'
             python-version: '3.9'  # not the oldest, not the most recent version
             time-zone: 'XXX-09:35'
+
     env:
       TZ: ${{ matrix.time-zone }}
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Apt-Get Install
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo apt-get update
-          sudo apt-get install -y shellcheck sqlite3
+          sudo apt-get install -y sqlite3
 
       - name: Install
         run: |
@@ -54,37 +57,10 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: Check changelog
-        if: startsWith(matrix.os, 'ubuntu')
-        run: towncrier build --draft
-
-      - name: Style
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          flake8
-          etc/bin/shellchecker
-
-      # note: exclude python 3.10+ from mypy checks as these produce false
-      # positives in installed libraries for python 3.7
-      - name: Typing
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9)
-        run: mypy
-
-      - name: Doctests
-        timeout-minutes: 4
-        run: |
-          pytest cylc/flow
-
       - name: Unit Tests
-        timeout-minutes: 4
+        timeout-minutes: 5
         run: |
-          pytest tests/unit
-
-      - name: Bandit
-        if: ${{ matrix.python-version == '3.7' }}
-        # https://github.com/PyCQA/bandit/issues/658
-        run: |
-          bandit -r --ini .bandit cylc/flow
+          pytest cylc/flow tests/unit
 
       - name: Integration Tests
         timeout-minutes: 6
@@ -93,7 +69,7 @@ jobs:
 
       - name: Upload failed tests artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cylc-run (${{ matrix.os }} py-${{ matrix.python-version }})
           path: ~/cylc-run/
@@ -104,26 +80,64 @@ jobs:
           coverage report
 
       - name: Upload coverage artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }}
           path: coverage.xml
           retention-days: 7
 
+  lint:
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 10
+    steps:
+      - name: Apt-Get Install
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # note: exclude python 3.10+ from mypy checks as these produce false
+      # positives in installed libraries for python 3.7
+      - name: Configure Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install
+        run: |
+          pip install -e ."[tests]"
+
+      - name: Flake8
+        run: flake8
+
+      - name: Bandit
+        run: |
+          bandit -r --ini .bandit cylc/flow
+
+      - name: Shellchecker
+        run: etc/bin/shellchecker
+
+      - name: MyPy
+        run: mypy
+
+      - name: Towncrier
+        run: towncrier build --draft
+
       - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+        run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py
 
   codecov:
     needs: test
     runs-on: ubuntu-latest
     timeout-minutes: 2
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download coverage artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       - name: Codecov upload
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
@@ -46,9 +46,9 @@ jobs:
         # NOTE: includes must define ALL of the matrix values
         include:
           # latest python
-          - name: 'py-3.11'
+          - name: 'py-3-latest'
             os: 'ubuntu-latest'
-            python-version: '3.11'
+            python-version: '3'
             test-base: 'tests/f'
             chunk: '1/4'
             platform: '_local_background*'
@@ -96,10 +96,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -108,7 +108,7 @@ jobs:
         run: |
           # install system deps
           brew update
-          brew install bash coreutils gnu-sed
+          brew install bash coreutils gnu-sed grep
 
           # add GNU coreutils and sed to the user PATH
           # (see instructions in brew install output)
@@ -118,6 +118,9 @@ jobs:
           echo \
             "/usr/local/opt/gnu-sed/libexec/gnubin" \
             >> "${GITHUB_PATH}"
+          echo \
+            "/usr/local/opt/grep/libexec/gnubin" \
+            >> "${GITHUB_PATH}"
 
           # add coreutils to the bashrc too (for jobs)
           cat >> "${HOME}/.bashrc" <<__HERE__
@@ -248,6 +251,7 @@ jobs:
               -exec echo '====== {} ======' \; -exec cat '{}' \;
 
       - name: Set artifact upload name
+        if: always()
         id: uploadname
         run: |
           # artifact name cannot contain '/' characters
@@ -256,7 +260,7 @@ jobs:
 
       - name: Upload failed tests artifact
         if: failure() && steps.test.outcome == 'failure'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cylc-run (${{ steps.uploadname.outputs.uploadname }})
           path: ~/cylc-run/
@@ -294,7 +298,7 @@ jobs:
           coverage report
 
       - name: Upload coverage artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_${{ steps.uploadname.outputs.uploadname }}
           path: coverage.xml
@@ -306,10 +310,10 @@ jobs:
     timeout-minutes: 2
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download coverage artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       - name: Codecov upload
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/test_manylinux.yml b/.github/workflows/test_manylinux.yml
@@ -38,13 +38,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test_tutorial_workflow.yml b/.github/workflows/test_tutorial_workflow.yml
@@ -21,12 +21,12 @@ jobs:
   test:
     strategy:
       matrix:
-        python-version: ['3.7', '3.11']
+        python-version: ['3.7', '3']
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
       - name: configure python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}