Update MacOS version on GH Actions (cylc#6187)

* actions: patch DNS issues on Mac OS runners * Small fix for host info logic * GH Actions: update MacOS version * Reduce test flakiness --------- Co-authored-by: Oliver Sanders <[email protected]>
wxtim · Jul 8, 2024 · 6ff547e · 6ff547e
1 parent ba57e88
commit 6ff547e
Show file tree

Hide file tree

Showing 17 changed files with 91 additions and 73 deletions.
diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
@@ -23,8 +23,8 @@ jobs:
         python-version: ['3.7', '3.8', '3.10', '3.11', '3']
         include:
           # mac os test
-          - os: 'macos-11'
-            python-version: '3.7'  # oldest supported version
+          - os: 'macos-latest'
+            python-version: '3.9'  # oldest supported version
           # non-utc timezone test
           - os: 'ubuntu-latest'
             python-version: '3.9'  # not the oldest, not the most recent version
@@ -49,6 +49,9 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y sqlite3
 
+      - name: Patch DNS
+        uses: cylc/release-actions/patch-dns@v1
+
       - name: Install
         run: |
           pip install -e ."[all]"

diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
@@ -74,14 +74,14 @@ jobs:
             platform: '_remote_background_indep_tcp _remote_at_indep_tcp'
           # macos
           - name: 'macos 1/5'
-            os: 'macos-11'
-            python-version: '3.7'
+            os: 'macos-latest'
+            python-version: '3.9'
             test-base: 'tests/f'
             chunk: '1/5'
             platform: '_local_background*'
           - name: 'macos 2/5'
-            os: 'macos-11'
-            python-version: '3.7'
+            os: 'macos-latest'
+            python-version: '3.9'
             test-base: 'tests/f'
             chunk: '2/5'
             platform: '_local_background*'
@@ -103,6 +103,29 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Create global config
+        run: |
+          CONF_PATH="$HOME/.cylc/flow/8"
+          mkdir -p "$CONF_PATH"
+          touch "$CONF_PATH/global.cylc"
+          ln -s "$CONF_PATH/global.cylc" "$CONF_PATH/global-tests.cylc"
+          echo "GLOBAL_CFG_PATH=${CONF_PATH}/global.cylc" >> "$GITHUB_ENV"
+
+      - name: Patch DNS
+        uses: cylc/release-actions/patch-dns@v1
+
+      - name: Add localhost entries to global config
+        if: startsWith(runner.os, 'macos')
+        run: |
+          cat >> "$GLOBAL_CFG_PATH" <<__HERE__
+          [platforms]
+            [[localhost, $(hostname -f), $(hostname -s)]]
+              hosts = localhost
+              install target = localhost
+              ssh command = ssh -oBatchMode=yes -oConnectTimeout=8 -oStrictHostKeyChecking=no
+          __HERE__
+          cat "$GLOBAL_CFG_PATH"
+
       - name: Brew Install
         if: startsWith(matrix.os, 'macos')
         run: |
@@ -112,19 +135,15 @@ jobs:
 
           # add GNU coreutils and sed to the user PATH
           # (see instructions in brew install output)
-          echo \
-            "$(brew --prefix)/opt/coreutils/libexec/gnubin" \
-            >> "${GITHUB_PATH}"
-          echo \
-            "/usr/local/opt/gnu-sed/libexec/gnubin" \
-            >> "${GITHUB_PATH}"
-          echo \
-            "/usr/local/opt/grep/libexec/gnubin" \
-            >> "${GITHUB_PATH}"
+          echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" >> "${GITHUB_PATH}"
+          echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> "${GITHUB_PATH}"
+          echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> "${GITHUB_PATH}"
 
           # add coreutils to the bashrc too (for jobs)
           cat >> "${HOME}/.bashrc" <<__HERE__
-          PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:/usr/local/opt/gnu-sed/libexec/gnubin:$PATH"
+          PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:$PATH"
+          PATH="$(brew --prefix)/opt/grep/libexec/gnubin:$PATH"
+          PATH="$(brew --prefix)/opt/gnu-sed/libexec/gnubin:$PATH"
           export PATH
           __HERE__
 
@@ -146,16 +165,13 @@ jobs:
       - name: Configure Atrun
         if: contains(matrix.platform, '_local_at')
         run: |
-          PTH="$HOME/.cylc/flow/"
-          mkdir -p "${PTH}"
-          cat > "${PTH}/global.cylc" << __HERE__
+          cat >> "$GLOBAL_CFG_PATH" << __HERE__
             [platforms]
                 [[_local_at_indep_tcp]]
                     hosts = localhost
                     install target = localhost
                     job runner = at
           __HERE__
-          cp "${PTH}/global.cylc" "${PTH}/global-tests.cylc"
 
       - name: Swarm Configure
         run: |
@@ -244,11 +260,11 @@ jobs:
         timeout-minutes: 1
         run: |
           find "$HOME/cylc-run" -name '*.err' -type f \
-              -exec echo '====== {} ======' \; -exec cat '{}' \;
+            -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \;
           find "$HOME/cylc-run" -name '*.log' -type f \
-              -exec echo '====== {} ======' \; -exec cat '{}' \;
+            -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \;
           find "${TMPDIR:-/tmp}/${USER}/cylctb-"* -type f \
-              -exec echo '====== {} ======' \; -exec cat '{}' \;
+            -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \;
 
       - name: Set artifact upload name
         if: always()

diff --git a/cylc/flow/hostuserutil.py b/cylc/flow/hostuserutil.py
@@ -50,6 +50,7 @@
 import socket
 import sys
 from time import time
+from typing import List, Optional, Tuple
 
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 
@@ -113,25 +114,24 @@ def get_host_ip_by_name(target):
         """Return internal IP address of target."""
         return socket.gethostbyname(target)
 
-    def _get_host_info(self, target=None):
+    def _get_host_info(
+        self, target: Optional[str] = None
+    ) -> Tuple[str, List[str], List[str]]:
         """Return the extended info of the current host."""
+        if target is None:
+            target = socket.getfqdn()
+        if IS_MAC_OS and target in {
+            '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.'
+            '0.0.0.0.0.0.ip6.arpa',
+            '1.0.0.127.in-addr.arpa',
+        }:
+            # Python's socket bindings don't play nicely with mac os
+            # so by default we get the above ip6.arpa address from
+            # socket.getfqdn, note this does *not* match `hostname -f`.
+            # https://github.com/cylc/cylc-flow/issues/2689
+            # https://github.com/cylc/cylc-flow/issues/3595
+            target = socket.gethostname()
         if target not in self._host_exs:
-            if target is None:
-                target = socket.getfqdn()
-            if (
-                IS_MAC_OS
-                and target in {
-                    '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.'
-                    '0.0.0.0.0.0.ip6.arpa',
-                    '1.0.0.127.in-addr.arpa'
-                }
-            ):
-                # Python's socket bindings don't play nicely with mac os
-                # so by default we get the above ip6.arpa address from
-                # socket.getfqdn, note this does *not* match `hostname -f`.
-                # https://github.com/cylc/cylc-flow/issues/2689
-                # https://github.com/cylc/cylc-flow/issues/3595
-                target = socket.gethostname()
             try:
                 self._host_exs[target] = socket.gethostbyname_ex(target)
             except IOError as exc:

diff --git a/tests/functional/cylc-set/00-set-succeeded/flow.cylc b/tests/functional/cylc-set/00-set-succeeded/flow.cylc
@@ -15,23 +15,23 @@
     [[graph]]
         R1 = """
             foo & bar => post<m>
-            setter
+            foo:started & bar:started => setter
         """
 [runtime]
     [[post<m>]]
     [[foo, bar]]
          script = false
     [[setter]]
-         script = """
-             # wait for foo and bar to fail.
-             for TASK in foo bar
-             do
-                 cylc workflow-state \
+        script = """
+            # wait for foo and bar to fail.
+            for TASK in foo bar
+            do
+                cylc workflow-state \
                     ${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/${TASK}:failed \
-                    --max-polls=10 --interval=1
-             done
-             # set foo succeeded (via --output)
-             cylc set -o succeeded $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/foo
-             # set bar succeeded (via default)
-             cylc set $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/bar
-         """
+                    --max-polls=20 --interval=1
+            done
+            # set foo succeeded (via --output)
+            cylc set -o succeeded $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/foo
+            # set bar succeeded (via default)
+            cylc set $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/bar
+        """
diff --git a/tests/functional/cylc-set/00-set-succeeded/reference.log b/tests/functional/cylc-set/00-set-succeeded/reference.log
@@ -1,4 +1,4 @@
-1/setter -triggered off [] in flow 1
+1/setter -triggered off ['1/bar', '1/foo'] in flow 1
 1/foo -triggered off [] in flow 1
 1/bar -triggered off [] in flow 1
 1/post_m1 -triggered off ['1/bar', '1/foo'] in flow 1

diff --git a/tests/functional/cylc-set/03-set-failed/flow.cylc b/tests/functional/cylc-set/03-set-failed/flow.cylc
@@ -2,7 +2,7 @@
 
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 
 [scheduling]

diff --git a/tests/functional/cylc-set/04-switch/flow.cylc b/tests/functional/cylc-set/04-switch/flow.cylc
@@ -2,7 +2,7 @@
 
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
         stall timeout = PT0S
         abort on stall timeout = True

diff --git a/tests/functional/cylc-set/05-expire/flow.cylc b/tests/functional/cylc-set/05-expire/flow.cylc
@@ -2,7 +2,7 @@
 
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
         stall timeout = PT0S
         abort on stall timeout = True

diff --git a/tests/functional/cylc-trigger/06-already-active/flow.cylc b/tests/functional/cylc-trigger/06-already-active/flow.cylc
@@ -1,7 +1,7 @@
 # test triggering an already active task
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 [scheduling]
     [[graph]]
@@ -19,4 +19,3 @@
             cylc__job__poll_grep_workflow_log \
                 "1/triggeree.* ignoring trigger - already active" -E
         """
-
diff --git a/tests/functional/job-submission/15-garbage-platform-command-2/flow.cylc b/tests/functional/job-submission/15-garbage-platform-command-2/flow.cylc
@@ -1,6 +1,6 @@
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
         stall timeout = PT20S
         abort on stall timeout = True

diff --git a/tests/functional/restart/50-two-flows/flow.cylc b/tests/functional/restart/50-two-flows/flow.cylc
@@ -3,7 +3,7 @@
 
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 [scheduling]
     [[graph]]
@@ -20,10 +20,10 @@
     [[b, d]]
     [[c]]
        script = """
-           if ((CYLC_TASK_FLOW_NUMBERS == 1)); then
-               cylc trigger --flow=new --meta="cheese wizard" \
-                   "$CYLC_WORKFLOW_ID//1/a"
-               cylc__job__poll_grep_workflow_log -E "\[1/a/02\(flows=2\):submitted\] => running"
-               cylc stop $CYLC_WORKFLOW_ID
-           fi
+            if ((CYLC_TASK_FLOW_NUMBERS == 1)); then
+                cylc trigger --flow=new --meta="cheese wizard" \
+                    "$CYLC_WORKFLOW_ID//1/a"
+                cylc__job__poll_grep_workflow_log -E "\[1/a/02\(flows=2\):submitted\] => running"
+                cylc stop $CYLC_WORKFLOW_ID
+            fi
        """
diff --git a/tests/functional/restart/59-retart-timeout/flow.cylc b/tests/functional/restart/59-retart-timeout/flow.cylc
@@ -1,6 +1,6 @@
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 [scheduling]
     [[graph]]

diff --git a/tests/functional/runahead/default-future/flow.cylc b/tests/functional/runahead/default-future/flow.cylc
@@ -3,7 +3,7 @@
     UTC mode = True
     allow implicit tasks = True
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 [scheduling]
     initial cycle point = 20100101T00

diff --git a/tests/functional/runahead/no_final/flow.cylc b/tests/functional/runahead/no_final/flow.cylc
@@ -5,7 +5,7 @@
         abort on stall timeout = True
         stall timeout = PT0S
         abort on inactivity timeout = True
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
 [scheduling]
     runahead limit = P1
     initial cycle point = 20100101T00

diff --git a/tests/functional/shutdown/08-now1.t b/tests/functional/shutdown/08-now1.t
@@ -22,7 +22,7 @@ set_test_number 5
 
 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
 run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}"
-workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "${WORKFLOW_NAME}"
+workflow_run_ok "${TEST_NAME_BASE}-run" cylc play -v --no-detach "${WORKFLOW_NAME}"
 LOGD="$RUN_DIR/${WORKFLOW_NAME}/log"
 grep_ok 'INFO - Workflow shutting down - REQUEST(NOW)' "${LOGD}/scheduler/log"
 JLOGD="${LOGD}/job/1/t1/01"

diff --git a/tests/functional/spawn-on-demand/11-hold-not-spawned/flow.cylc b/tests/functional/spawn-on-demand/11-hold-not-spawned/flow.cylc
@@ -1,7 +1,7 @@
 # Test holding a task that hasn't spawned yet.
 [scheduler]
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 [scheduling]
     [[graph]]

diff --git a/tests/functional/spawn-on-demand/15-stop-flow-3/flow.cylc b/tests/functional/spawn-on-demand/15-stop-flow-3/flow.cylc
@@ -8,7 +8,7 @@
 [scheduler]
     allow implicit tasks = True
     [[events]]
-        inactivity timeout = PT20S
+        inactivity timeout = PT1M
         abort on inactivity timeout = True
 [scheduling]
     [[xtriggers]]