Move emulator/simulator webview tests to own workflows (#1716)

* Move android emulator webview tests * Move ios simulator webview tests * Exclude webview from default ios workflow
leancodepl · Sep 19, 2023 · 699bdeb · 699bdeb
1 parent 57f1585
commit 699bdeb
Show file tree

Hide file tree

Showing 5 changed files with 268 additions and 2 deletions.
diff --git a/.github/workflows/test-android-emulator-webview.yaml b/.github/workflows/test-android-emulator-webview.yaml
@@ -0,0 +1,108 @@
+name: test android emulator webview
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */24 * * *'
+
+jobs:
+  run_tests:
+    name: Flutter ${{ matrix.flutter_version }} webview on emulator.wtf
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    outputs:
+      SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter_version }} webview on emulator.wtf
+      TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }}
+      URL_TO_DETAILS: ${{ steps.tests_step.outputs.URL_TO_DETAILS }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        flutter_version: ['3.13.x']
+
+    defaults:
+      run:
+        working-directory: packages/patrol/example
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Gradle cache
+        uses: gradle/gradle-build-action@v2
+        with:
+          generate-job-summary: false
+
+      - name: Set up Flutter
+        uses: subosito/flutter-action@v2
+        with:
+          flutter-version: ${{ matrix.flutter_version }}
+
+      - name: Preload Flutter artifacts
+        run: flutter precache
+
+      - name: Set up Patrol CLI
+        working-directory: packages/patrol_cli
+        run: dart pub global activate --source path . && patrol
+
+      - name: Generate Gradle wrapper
+        run: flutter build apk --debug --flavor=does-not-exist || true
+
+      - name: Install ew-cli
+        run: |
+          mkdir -p "$HOME/bin"
+          curl "https://maven.emulator.wtf/releases/ew-cli" -o "$HOME/bin/ew-cli"
+          chmod a+x "$HOME/bin/ew-cli"
+          echo "$HOME/bin" >> $GITHUB_PATH
+          echo "EW_API_TOKEN=${{ secrets.EW_API_TOKEN }}" >> $GITHUB_ENV
+
+      - name: Set tests to include
+        run: |
+          TESTS_TO_INCLUDE="webview_hackernews_test,\
+          webview_leancode_test,\
+          webview_stackoverflow_test"
+
+          target_paths=""
+          for target in $(echo $TESTS_TO_INCLUDE | tr ',' '\n'); do
+              target_paths+="integration_test/${target}.dart,"
+          done
+
+          echo "INCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
+
+      - name: patrol build android
+        run: patrol build android --target ${{ env.INCLUDED_TESTS }} --verbose
+
+      - name: Upload APKs to emulator.wtf and wait for tests to finish
+        id: tests_step
+        run: |
+          set +e
+          output="$(ew-cli \
+            emulatorwtf.yaml:ci_webview --json \
+            --display-name "Patrol example app (${GITHUB_SHA::7})")"
+          TESTS_EXIT_CODE=$?
+          set -e
+
+          # Extract the results URL and write it to Github Summmary
+          link="$(echo "$output" | jq -r ".resultsUrl")"
+          echo "[Test details on emulator.wtf]($link) (LeanCode members only)" >> "$GITHUB_STEP_SUMMARY"
+
+          echo "URL_TO_DETAILS=$link" >> "$GITHUB_OUTPUT"
+          echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT"
+          exit $TESTS_EXIT_CODE
+
+  call_send_slack_message:
+    name: Notify on Slack
+    uses: ./.github/workflows/send-slack-message.yaml
+    needs: run_tests
+    if: always()
+    with:
+      TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }}
+      SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }}
+      URL_TO_DETAILS: ${{ needs.run_tests.outputs.URL_TO_DETAILS }}
+    secrets: inherit
diff --git a/.github/workflows/test-android-emulator.yaml b/.github/workflows/test-android-emulator.yaml
@@ -68,7 +68,10 @@ jobs:
         run: |
           TESTS_TO_EXCLUDE="service_airplane_mode_test,\
           service_bluetooth_test,\
-          notifications_test"
+          notifications_test,\
+          webview_hackernews_test,\
+          webview_leancode_test,\
+          webview_stackoverflow_test"
 
           target_paths=""
           for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do

diff --git a/.github/workflows/test-ios-simulator-webview.yaml b/.github/workflows/test-ios-simulator-webview.yaml
@@ -0,0 +1,145 @@
+name: test ios simulator webview
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */24 * * *'
+
+jobs:
+  run_tests:
+    name: Flutter ${{ matrix.flutter_version }} webview on ${{ matrix.device_model }} (${{ matrix.os_version }}) simulator
+    runs-on: macos-latest
+    timeout-minutes: 40
+    outputs:
+      SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter_version }} on ${{ matrix.os }} ${{ matrix.os_version }} simulator
+      TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }}
+      FAILURE_STATUS: ${{ steps.status_step.outputs.FAILURE_STATUS }}
+      ERROR_STATUS: ${{ steps.status_step.outputs.ERROR_STATUS }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        flutter_version: ['3.13.x']
+        device_model: [iPhone 14]
+        os: [iOS]
+        os_version: ['16.2']
+
+    defaults:
+      run:
+        working-directory: packages/patrol/example
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+
+      - name: Set up Flutter
+        uses: subosito/flutter-action@v2
+        with:
+          flutter-version: ${{ matrix.flutter_version }}
+
+      - name: Preload Flutter artifacts
+        run: flutter precache
+
+      - name: Set up Patrol CLI
+        working-directory: packages/patrol_cli
+        run: dart pub global activate --source path . && patrol
+
+      - name: Start iOS simulator
+        uses: futureware-tech/simulator-action@v2
+        with:
+          model: ${{ matrix.device_model }}
+          os: ${{ matrix.os }}
+          os_version: ${{ matrix.os_version }}
+          erase_before_boot: true
+          shutdown_after_job: true
+
+      - name: Set tests to include
+        run: |
+          TESTS_TO_INCLUDE="webview_hackernews_test,\
+          webview_leancode_test,\
+          webview_stackoverflow_test"
+
+          target_paths=""
+          for target in $(echo $TESTS_TO_INCLUDE | tr ',' '\n'); do
+              target_paths+="integration_test/${target}.dart,"
+          done
+
+          echo "INCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
+
+      - name: Run tests
+        id: tests_step
+        run: |
+          xcrun simctl io booted recordVideo --codec=h264 "${{ matrix.device_model }}.mp4" &
+          recordingpid="$!"
+
+          xcrun simctl spawn booted log stream --type log --color none > all_simulator_logs.txt &
+          logpid="$!"
+
+          sleep 10 # See https://github.com/leancodepl/patrol/issues/1282
+
+          TESTS_EXIT_CODE=0
+          patrol test --target ${{ env.INCLUDED_TESTS }} --verbose || TESTS_EXIT_CODE=$?
+
+          kill -SIGINT $recordingpid
+          kill -SIGINT $logpid
+          echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT"
+          exit $TESTS_EXIT_CODE
+
+      - name: Check if something went wrong
+        id: status_step
+        if: always()
+        run: >
+          if [ -z ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} ]; then 
+            echo "ERROR_STATUS=error" >> "$GITHUB_OUTPUT";
+          elif [ ! ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} == 0 ]; then
+            echo "FAILURE_STATUS=failure" >> "$GITHUB_OUTPUT";
+          fi;
+
+      - name: Find xcresult path
+        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
+        run: |
+          brew install coreutils # to provide realpath binary
+          echo "XCRESULT_PATH=$(realpath build/ios_results_*.xcresult)" >> $GITHUB_ENV
+
+      - name: Publish test report to summary
+        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
+        uses: kishikawakatsumi/xcresulttool@v1
+        with:
+          title: Patrol tests on ${{ matrix.device_model }}
+          upload-bundles: never
+          path: |
+            ${{ env.XCRESULT_PATH }}
+
+      - name: Upload XCRESULT test result to artifacts
+        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test result from ${{ matrix.device_model }}.xcresult
+          path: ${{ env.XCRESULT_PATH }}
+
+      - name: Upload simulator logs to artifacts
+        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: Logs from ${{ matrix.device_model }}
+          path: |
+            ${{ github.workspace }}/packages/patrol/example/all_simulator_logs.txt
+
+      - name: Upload captured video to artifacts
+        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: Captured video from ${{ matrix.device_model }}.mp4
+          path: ${{ github.workspace }}/packages/patrol/example/${{ matrix.device_model }}.mp4
+
+  call_send_slack_message:
+    name: Notify on Slack
+    uses: ./.github/workflows/send-slack-message.yaml
+    needs: run_tests
+    if: always()
+    with:
+      TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }}
+      FAILURE_STATUS: ${{ needs.run_tests.outputs.FAILURE_STATUS }}
+      ERROR_STATUS: ${{ needs.run_tests.outputs.ERROR_STATUS }}
+      SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }}
+    secrets: inherit
diff --git a/.github/workflows/test-ios-simulator.yaml b/.github/workflows/test-ios-simulator.yaml
@@ -46,7 +46,6 @@ jobs:
         run: dart pub global activate --source path . && patrol
 
       - name: Start iOS simulator
-        id: start_simulator
         uses: futureware-tech/simulator-action@v2
         with:
           model: ${{ matrix.device_model }}
@@ -67,6 +66,8 @@ jobs:
           service_cellular_test,\
           service_wifi_test,\
           webview_stackoverflow_test,\
+          webview_leancode_test,\
+          webview_hackernews_test,\
           swipe_test"
 
           target_paths=""

diff --git a/packages/patrol/example/emulatorwtf.yaml b/packages/patrol/example/emulatorwtf.yaml
@@ -42,6 +42,12 @@ ci_devices:
     #   version: 27
     #   gpu: auto
 
+ci_webview_devices:
+  device:
+    - model: Pixel7
+      version: 31
+      gpu: auto
+
 config:
   use-orchestrator: true
   clear-package-data: true
@@ -63,5 +69,8 @@ config_ci:
 ci:
   include: [apps, ci_devices, config, config_ci]
 
+ci_webview:
+  include: [apps, ci_webview_devices, config, config_ci]
+
 debug:
   include: [apps, all_devices, config]