test-ios-simulator.yaml

name: test ios simulator

on:
  workflow_dispatch:
  schedule:
    - cron: "0 0 1 * *" #run this workflow once every month

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  run_tests:
    name: Flutter ${{ matrix.flutter-version }} on ${{ matrix.device_model }} (${{ matrix.os_version }}) simulator
    runs-on: macos-latest
    timeout-minutes: 40
    outputs:
      SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter-version }} on ${{ matrix.os }} ${{ matrix.os_version }} simulator
      TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }}
      FAILURE_STATUS: ${{ steps.status_step.outputs.FAILURE_STATUS }}
      ERROR_STATUS: ${{ steps.status_step.outputs.ERROR_STATUS }}
      EXCLUDED_TESTS: ${{ steps.set_excluded_tests.outputs.EXCLUDED_TESTS }}

    strategy:
      fail-fast: false
      matrix:
        flutter-version: ["3.22.x"]
        flutter-channel: ["stable"]
        device_model:
          [iPhone SE (3rd generation), iPhone 14, iPad (10th generation)]
        os: [iOS]
        os_version: ["17.2"]

    defaults:
      run:
        working-directory: dev/e2e_app

    steps:
      - name: Clone repository
        uses: actions/checkout@v4

      - name: Set up Flutter
        uses: subosito/flutter-action@v2
        with:
          flutter-version: ${{ matrix.flutter-version }}
          channel: ${{ matrix.flutter-channel }}

      - name: Preload Flutter artifacts
        run: flutter precache --ios

      - name: Set up Melos and activate workspace
        working-directory: .
        run: |
          dart pub global activate melos
          melos bootstrap

      - name: Set up Patrol CLI
        working-directory: packages/patrol_cli
        run: dart pub global activate --source path . && patrol

      - name: Start iOS simulator
        uses: futureware-tech/simulator-action@v2
        with:
          model: ${{ matrix.device_model }}
          os: ${{ matrix.os }}
          os_version: ${{ matrix.os_version }}
          erase_before_boot: true
          shutdown_after_job: true

      - name: Set simulator location
        run: xcrun simctl location booted set 52.17469,21.03193

      - name: Set tests to exclude
        id: set_excluded_tests
        run: |
          TESTS_TO_EXCLUDE="android_app_test,\
          service_airplane_mode_test,\
          service_bluetooth_test,\
          service_cellular_test,\
          service_wifi_test,\
          service_location_test,\
          webview_stackoverflow_test,\
          webview_leancode_test,\
          webview_hackernews_test"

          target_paths=""
          for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do
              target_paths+="integration_test/${target}.dart,"
          done
          target_paths="${target_paths%,}"

          echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT"
          echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"

      - name: Run UI tests
        id: tests_step
        run: |
          xcrun simctl io booted recordVideo --codec=h264 "${{ matrix.device_model }}.mp4" &
          recordingpid="$!"

          xcrun simctl spawn booted log stream --type log --color none > all_simulator_logs.txt &
          logpid="$!"

          sleep 10 # See https://github.com/leancodepl/patrol/issues/1282

          TESTS_EXIT_CODE=0
          patrol test --exclude ${{ env.EXCLUDED_TESTS }} --verbose || TESTS_EXIT_CODE=$?

          kill -SIGINT $recordingpid
          kill -SIGINT $logpid
          echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT"
          exit $TESTS_EXIT_CODE

      - name: Check if something went wrong
        id: status_step
        if: always()
        run: >
          if [ -z ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} ]; then 
            echo "ERROR_STATUS=error" >> "$GITHUB_OUTPUT";
          elif [ ! ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} == 0 ]; then
            echo "FAILURE_STATUS=failure" >> "$GITHUB_OUTPUT";
          fi;

      - name: Find xcresult path
        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
        run: |
          brew install coreutils # to provide realpath binary
          echo "XCRESULT_PATH=$(realpath build/ios_results_*.xcresult)" >> $GITHUB_ENV

      - name: Publish test report to summary
        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
        uses: kishikawakatsumi/xcresulttool@v1
        with:
          title: Patrol tests on ${{ matrix.device_model }}
          upload-bundles: never
          path: |
            ${{ env.XCRESULT_PATH }}

      - name: Upload XCRESULT test result to artifacts
        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
        uses: actions/upload-artifact@v3
        with:
          name: Test result from ${{ matrix.device_model }}.xcresult
          path: ${{ env.XCRESULT_PATH }}

      - name: Upload simulator logs to artifacts
        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
        uses: actions/upload-artifact@v3
        with:
          name: Logs from ${{ matrix.device_model }}
          path: |
            ${{ github.workspace }}/dev/e2e_app/all_simulator_logs.txt

      - name: Upload captured video to artifacts
        if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
        uses: actions/upload-artifact@v3
        with:
          name: Captured video from ${{ matrix.device_model }}.mp4
          path: ${{ github.workspace }}/dev/e2e_app/${{ matrix.device_model }}.mp4

  call_send_slack_message:
    name: Notify on Slack
    uses: ./.github/workflows/send-slack-message.yaml
    needs: run_tests
    if: always()
    with:
      TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }}
      FAILURE_STATUS: ${{ needs.run_tests.outputs.FAILURE_STATUS }}
      ERROR_STATUS: ${{ needs.run_tests.outputs.ERROR_STATUS }}
      SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }}
      EXCLUDED_TESTS: ${{ needs.run_tests.outputs.EXCLUDED_TESTS }}
    secrets: inherit