From 4211399e946fa2268527db8b02baeb7959457f25 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 21 Sep 2023 09:13:52 +0200 Subject: [PATCH] Merge master into develop (#1740) * Use iPhone 15 Pro Max on CirrusCI (#1718) * Move emulator/simulator webview tests to own workflows (#1716) * Add missing label (#1719) * Add missing label * Use flutter 3.13 and xcode 15 * Bump to 2.8.0 (#1722) * Trim trailing comma (#1735) * Fix (#1736) * remove notice about Patrol 2.0 (#1738) * patrol: bump version to 2.2.5 (#1739) --------- Co-authored-by: EthnosDev <123928164+ethnosdev@users.noreply.github.com> Co-authored-by: Filip Tuzimek <63547653+fylyppo@users.noreply.github.com> --- .cirrus.yml | 8 +- .github/labeler.yml | 2 + .github/workflows/test-android-device.yaml | 3 +- .../test-android-emulator-webview.yaml | 109 +++++++++++++ .github/workflows/test-android-emulator.yaml | 8 +- .github/workflows/test-ios-device.yaml | 1 + .../workflows/test-ios-simulator-webview.yaml | 146 ++++++++++++++++++ .github/workflows/test-ios-simulator.yaml | 4 +- docs/getting-started.mdx | 2 +- packages/patrol/example/emulatorwtf.yaml | 9 ++ packages/patrol_cli/CHANGELOG.md | 4 + .../lib/src/runner/patrol_command_runner.dart | 8 - 12 files changed, 288 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/test-android-emulator-webview.yaml create mode 100644 .github/workflows/test-ios-simulator-webview.yaml diff --git a/.cirrus.yml b/.cirrus.yml index 5940e9ce6..d71032c87 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,6 +3,7 @@ test_linux_task: required_pr_labels: - 'package: patrol_cli' - 'package: patrol' + - 'cirrusci' container: image: ghcr.io/cirruslabs/flutter:stable cpu: 6 @@ -10,7 +11,7 @@ test_linux_task: kvm: 'true' env: PATH: $HOME/.pub-cache/bin:$HOME/fvm/default/bin:$CIRRUS_WORKING_DIR/fvm:${PATH} - FLUTTER_VERSION: '3.10' + FLUTTER_VERSION: '3.13' EMULATOR_API_LEVEL: '34' EMULATOR_ABI: google_apis_playstore;x86_64 EMULATOR_IMAGE: system-images;android-${EMULATOR_API_LEVEL};${EMULATOR_ABI} @@ -71,11 +72,12 @@ test_macos_task: required_pr_labels: - 'package: patrol_cli' - 'package: patrol' + - 'cirrusci' macos_instance: image: ghcr.io/cirruslabs/macos-ventura-xcode:latest env: PATH: $HOME/.pub-cache/bin:$HOME/fvm/default/bin:${PATH} - FLUTTER_VERSION: '3.10' + FLUTTER_VERSION: '3.13' timeout_in: 30m set_up_fvm_script: | @@ -94,7 +96,7 @@ test_macos_task: setup_patrol_cli_script: - dart pub global activate --source path packages/patrol_cli && patrol setup_simulator_script: | - xcrun simctl boot "iPhone 14 Pro Max" + xcrun simctl boot "iPhone 15 Pro Max" patrol_test_script: | cd dev/cli_tests dart pub get diff --git a/.github/labeler.yml b/.github/labeler.yml index 7a12d8d8a..0a32aea1c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,6 +2,8 @@ - packages/adb/**/* 'package: patrol_cli': - packages/patrol_cli/**/* +'cirrusci': + - .cirrus.yml 'package: patrol': - packages/patrol/**/* 'docs': diff --git a/.github/workflows/test-android-device.yaml b/.github/workflows/test-android-device.yaml index 7b1ce354f..eb44186c6 100644 --- a/.github/workflows/test-android-device.yaml +++ b/.github/workflows/test-android-device.yaml @@ -47,7 +47,7 @@ jobs: uses: google-github-actions/setup-gcloud@v1 - name: Gradle cache - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v2.8.0 with: generate-job-summary: false @@ -76,6 +76,7 @@ jobs: for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do target_paths+="integration_test/${target}.dart," done + target_paths="${target_paths%,}" echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT" echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV" diff --git a/.github/workflows/test-android-emulator-webview.yaml b/.github/workflows/test-android-emulator-webview.yaml new file mode 100644 index 000000000..5a3734586 --- /dev/null +++ b/.github/workflows/test-android-emulator-webview.yaml @@ -0,0 +1,109 @@ +name: test android emulator webview + +on: + workflow_dispatch: + schedule: + - cron: '0 */24 * * *' + +jobs: + run_tests: + name: Flutter ${{ matrix.flutter_version }} webview on emulator.wtf + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter_version }} webview on emulator.wtf + TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} + URL_TO_DETAILS: ${{ steps.tests_step.outputs.URL_TO_DETAILS }} + + strategy: + fail-fast: false + matrix: + flutter_version: ['3.13.x'] + + defaults: + run: + working-directory: packages/patrol/example + + steps: + - name: Clone repository + uses: actions/checkout@v3 + + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 17 + + - name: Gradle cache + uses: gradle/gradle-build-action@v2.8.0 + with: + generate-job-summary: false + + - name: Set up Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: ${{ matrix.flutter_version }} + + - name: Preload Flutter artifacts + run: flutter precache + + - name: Set up Patrol CLI + working-directory: packages/patrol_cli + run: dart pub global activate --source path . && patrol + + - name: Generate Gradle wrapper + run: flutter build apk --debug --flavor=does-not-exist || true + + - name: Install ew-cli + run: | + mkdir -p "$HOME/bin" + curl "https://maven.emulator.wtf/releases/ew-cli" -o "$HOME/bin/ew-cli" + chmod a+x "$HOME/bin/ew-cli" + echo "$HOME/bin" >> $GITHUB_PATH + echo "EW_API_TOKEN=${{ secrets.EW_API_TOKEN }}" >> $GITHUB_ENV + + - name: Set tests to include + run: | + TESTS_TO_INCLUDE="webview_hackernews_test,\ + webview_leancode_test,\ + webview_stackoverflow_test" + + target_paths="" + for target in $(echo $TESTS_TO_INCLUDE | tr ',' '\n'); do + target_paths+="integration_test/${target}.dart," + done + target_paths="${target_paths%,}" + + echo "INCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV" + + - name: patrol build android + run: patrol build android --target ${{ env.INCLUDED_TESTS }} --verbose + + - name: Upload APKs to emulator.wtf and wait for tests to finish + id: tests_step + run: | + set +e + output="$(ew-cli \ + emulatorwtf.yaml:ci_webview --json \ + --display-name "Patrol example app (${GITHUB_SHA::7})")" + TESTS_EXIT_CODE=$? + set -e + + # Extract the results URL and write it to Github Summmary + link="$(echo "$output" | jq -r ".resultsUrl")" + echo "[Test details on emulator.wtf]($link) (LeanCode members only)" >> "$GITHUB_STEP_SUMMARY" + + echo "URL_TO_DETAILS=$link" >> "$GITHUB_OUTPUT" + echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT" + exit $TESTS_EXIT_CODE + + call_send_slack_message: + name: Notify on Slack + uses: ./.github/workflows/send-slack-message.yaml + needs: run_tests + if: always() + with: + TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }} + SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }} + URL_TO_DETAILS: ${{ needs.run_tests.outputs.URL_TO_DETAILS }} + secrets: inherit diff --git a/.github/workflows/test-android-emulator.yaml b/.github/workflows/test-android-emulator.yaml index ae8eff364..ae67ec21d 100644 --- a/.github/workflows/test-android-emulator.yaml +++ b/.github/workflows/test-android-emulator.yaml @@ -36,7 +36,7 @@ jobs: java-version: 17 - name: Gradle cache - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v2.8.0 with: generate-job-summary: false @@ -68,12 +68,16 @@ jobs: run: | TESTS_TO_EXCLUDE="service_airplane_mode_test,\ service_bluetooth_test,\ - notifications_test" + notifications_test,\ + webview_hackernews_test,\ + webview_leancode_test,\ + webview_stackoverflow_test" target_paths="" for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do target_paths+="integration_test/${target}.dart," done + target_paths="${target_paths%,}" echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT" echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV" diff --git a/.github/workflows/test-ios-device.yaml b/.github/workflows/test-ios-device.yaml index 99bee1f4b..3b0f3fa41 100644 --- a/.github/workflows/test-ios-device.yaml +++ b/.github/workflows/test-ios-device.yaml @@ -91,6 +91,7 @@ jobs: for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do target_paths+="integration_test/${target}.dart," done + target_paths="${target_paths%,}" echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT" echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV" diff --git a/.github/workflows/test-ios-simulator-webview.yaml b/.github/workflows/test-ios-simulator-webview.yaml new file mode 100644 index 000000000..dbbf59bb1 --- /dev/null +++ b/.github/workflows/test-ios-simulator-webview.yaml @@ -0,0 +1,146 @@ +name: test ios simulator webview + +on: + workflow_dispatch: + schedule: + - cron: '0 */24 * * *' + +jobs: + run_tests: + name: Flutter ${{ matrix.flutter_version }} webview on ${{ matrix.device_model }} (${{ matrix.os_version }}) simulator + runs-on: macos-latest + timeout-minutes: 40 + outputs: + SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter_version }} on ${{ matrix.os }} ${{ matrix.os_version }} simulator + TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} + FAILURE_STATUS: ${{ steps.status_step.outputs.FAILURE_STATUS }} + ERROR_STATUS: ${{ steps.status_step.outputs.ERROR_STATUS }} + + strategy: + fail-fast: false + matrix: + flutter_version: ['3.13.x'] + device_model: [iPhone 14] + os: [iOS] + os_version: ['16.2'] + + defaults: + run: + working-directory: packages/patrol/example + + steps: + - name: Clone repository + uses: actions/checkout@v3 + + - name: Set up Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: ${{ matrix.flutter_version }} + + - name: Preload Flutter artifacts + run: flutter precache + + - name: Set up Patrol CLI + working-directory: packages/patrol_cli + run: dart pub global activate --source path . && patrol + + - name: Start iOS simulator + uses: futureware-tech/simulator-action@v2 + with: + model: ${{ matrix.device_model }} + os: ${{ matrix.os }} + os_version: ${{ matrix.os_version }} + erase_before_boot: true + shutdown_after_job: true + + - name: Set tests to include + run: | + TESTS_TO_INCLUDE="webview_hackernews_test,\ + webview_leancode_test,\ + webview_stackoverflow_test" + + target_paths="" + for target in $(echo $TESTS_TO_INCLUDE | tr ',' '\n'); do + target_paths+="integration_test/${target}.dart," + done + target_paths="${target_paths%,}" + + echo "INCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV" + + - name: Run tests + id: tests_step + run: | + xcrun simctl io booted recordVideo --codec=h264 "${{ matrix.device_model }}.mp4" & + recordingpid="$!" + + xcrun simctl spawn booted log stream --type log --color none > all_simulator_logs.txt & + logpid="$!" + + sleep 10 # See https://github.com/leancodepl/patrol/issues/1282 + + TESTS_EXIT_CODE=0 + patrol test --target ${{ env.INCLUDED_TESTS }} --verbose || TESTS_EXIT_CODE=$? + + kill -SIGINT $recordingpid + kill -SIGINT $logpid + echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT" + exit $TESTS_EXIT_CODE + + - name: Check if something went wrong + id: status_step + if: always() + run: > + if [ -z ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} ]; then + echo "ERROR_STATUS=error" >> "$GITHUB_OUTPUT"; + elif [ ! ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} == 0 ]; then + echo "FAILURE_STATUS=failure" >> "$GITHUB_OUTPUT"; + fi; + + - name: Find xcresult path + if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }} + run: | + brew install coreutils # to provide realpath binary + echo "XCRESULT_PATH=$(realpath build/ios_results_*.xcresult)" >> $GITHUB_ENV + + - name: Publish test report to summary + if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }} + uses: kishikawakatsumi/xcresulttool@v1 + with: + title: Patrol tests on ${{ matrix.device_model }} + upload-bundles: never + path: | + ${{ env.XCRESULT_PATH }} + + - name: Upload XCRESULT test result to artifacts + if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }} + uses: actions/upload-artifact@v3 + with: + name: Test result from ${{ matrix.device_model }}.xcresult + path: ${{ env.XCRESULT_PATH }} + + - name: Upload simulator logs to artifacts + if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }} + uses: actions/upload-artifact@v3 + with: + name: Logs from ${{ matrix.device_model }} + path: | + ${{ github.workspace }}/packages/patrol/example/all_simulator_logs.txt + + - name: Upload captured video to artifacts + if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }} + uses: actions/upload-artifact@v3 + with: + name: Captured video from ${{ matrix.device_model }}.mp4 + path: ${{ github.workspace }}/packages/patrol/example/${{ matrix.device_model }}.mp4 + + call_send_slack_message: + name: Notify on Slack + uses: ./.github/workflows/send-slack-message.yaml + needs: run_tests + if: always() + with: + TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }} + FAILURE_STATUS: ${{ needs.run_tests.outputs.FAILURE_STATUS }} + ERROR_STATUS: ${{ needs.run_tests.outputs.ERROR_STATUS }} + SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }} + secrets: inherit diff --git a/.github/workflows/test-ios-simulator.yaml b/.github/workflows/test-ios-simulator.yaml index c25ed1fae..556affeb0 100644 --- a/.github/workflows/test-ios-simulator.yaml +++ b/.github/workflows/test-ios-simulator.yaml @@ -46,7 +46,6 @@ jobs: run: dart pub global activate --source path . && patrol - name: Start iOS simulator - id: start_simulator uses: futureware-tech/simulator-action@v2 with: model: ${{ matrix.device_model }} @@ -67,12 +66,15 @@ jobs: service_cellular_test,\ service_wifi_test,\ webview_stackoverflow_test,\ + webview_leancode_test,\ + webview_hackernews_test,\ swipe_test" target_paths="" for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do target_paths+="integration_test/${target}.dart," done + target_paths="${target_paths%,}" echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT" echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV" diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index f4103333c..b3a93a73d 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -177,7 +177,7 @@ Psst... Android is a bit easier to set up, so we recommend starting with it! end ``` - 6. Create an empty file `integration_test/example_test.dart`. From the command line, run: + 6. Create an empty file `integration_test/example_test.dart` in the root of your Flutter project. From the command line, run: ``` $ flutter build ios --config-only integration_test/example_test.dart diff --git a/packages/patrol/example/emulatorwtf.yaml b/packages/patrol/example/emulatorwtf.yaml index 3fed010d1..bae9a022c 100644 --- a/packages/patrol/example/emulatorwtf.yaml +++ b/packages/patrol/example/emulatorwtf.yaml @@ -42,6 +42,12 @@ ci_devices: # version: 27 # gpu: auto +ci_webview_devices: + device: + - model: Pixel7 + version: 31 + gpu: auto + config: use-orchestrator: true clear-package-data: true @@ -63,5 +69,8 @@ config_ci: ci: include: [apps, ci_devices, config, config_ci] +ci_webview: + include: [apps, ci_webview_devices, config, config_ci] + debug: include: [apps, all_devices, config] diff --git a/packages/patrol_cli/CHANGELOG.md b/packages/patrol_cli/CHANGELOG.md index 0a9154ab1..ad38ead48 100644 --- a/packages/patrol_cli/CHANGELOG.md +++ b/packages/patrol_cli/CHANGELOG.md @@ -10,6 +10,10 @@ This version requires version 2.3.0-dev.2 of `patrol` package. This version requires version 2.3.0-dev.1 of `patrol` package. +## 2.1.5 + +- Remove notice about migrating to Patrol 2.0 (#1738) + ## 2.1.4 - Uninstall RunnerUITests app on iOS when flavor is present (#1694) diff --git a/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart b/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart index 3b493d689..74780d63b 100644 --- a/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart +++ b/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart @@ -64,14 +64,6 @@ Disable this warning by setting the PATROL_FINDERS environment variable. '''); } - if (!platform.environment.containsKey('PATROL_MIGRATED')) { - logger.warn(''' -You're using Patrol CLI 2.0, which has breaking changes. -Read the migration guide at https://patrol.leancode.co/v2. -Disable this warning by setting the PATROL_MIGRATED environment variable. -'''); - } - ProcessSignal.sigint.watch().listen((signal) async { logger.detail('Caught SIGINT, exiting...'); await runner.dispose().onError((err, st) {