Skip to content

Commit

Permalink
Merge master into develop (#1740)
Browse files Browse the repository at this point in the history
* Use iPhone 15 Pro Max on CirrusCI (#1718)

* Move emulator/simulator webview tests to own workflows (#1716)

* Add missing label (#1719)

* Add missing label

* Use flutter 3.13 and xcode 15

* Bump to 2.8.0 (#1722)

* Trim trailing comma (#1735)

* Fix (#1736)

* remove notice about Patrol 2.0 (#1738)

* patrol: bump version to 2.2.5 (#1739)

---------

Co-authored-by: EthnosDev <[email protected]>
Co-authored-by: Filip Tuzimek <[email protected]>
  • Loading branch information
3 people authored Sep 21, 2023
1 parent cdd2df8 commit 4211399
Show file tree
Hide file tree
Showing 12 changed files with 288 additions and 16 deletions.
8 changes: 5 additions & 3 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ test_linux_task:
required_pr_labels:
- 'package: patrol_cli'
- 'package: patrol'
- 'cirrusci'
container:
image: ghcr.io/cirruslabs/flutter:stable
cpu: 6
memory: 10G
kvm: 'true'
env:
PATH: $HOME/.pub-cache/bin:$HOME/fvm/default/bin:$CIRRUS_WORKING_DIR/fvm:${PATH}
FLUTTER_VERSION: '3.10'
FLUTTER_VERSION: '3.13'
EMULATOR_API_LEVEL: '34'
EMULATOR_ABI: google_apis_playstore;x86_64
EMULATOR_IMAGE: system-images;android-${EMULATOR_API_LEVEL};${EMULATOR_ABI}
Expand Down Expand Up @@ -71,11 +72,12 @@ test_macos_task:
required_pr_labels:
- 'package: patrol_cli'
- 'package: patrol'
- 'cirrusci'
macos_instance:
image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
env:
PATH: $HOME/.pub-cache/bin:$HOME/fvm/default/bin:${PATH}
FLUTTER_VERSION: '3.10'
FLUTTER_VERSION: '3.13'
timeout_in: 30m

set_up_fvm_script: |
Expand All @@ -94,7 +96,7 @@ test_macos_task:
setup_patrol_cli_script:
- dart pub global activate --source path packages/patrol_cli && patrol
setup_simulator_script: |
xcrun simctl boot "iPhone 14 Pro Max"
xcrun simctl boot "iPhone 15 Pro Max"
patrol_test_script: |
cd dev/cli_tests
dart pub get
Expand Down
2 changes: 2 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
- packages/adb/**/*
'package: patrol_cli':
- packages/patrol_cli/**/*
'cirrusci':
- .cirrus.yml
'package: patrol':
- packages/patrol/**/*
'docs':
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/test-android-device.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
uses: google-github-actions/setup-gcloud@v1

- name: Gradle cache
uses: gradle/gradle-build-action@v2
uses: gradle/gradle-build-action@v2.8.0
with:
generate-job-summary: false

Expand Down Expand Up @@ -76,6 +76,7 @@ jobs:
for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do
target_paths+="integration_test/${target}.dart,"
done
target_paths="${target_paths%,}"
echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT"
echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
Expand Down
109 changes: 109 additions & 0 deletions .github/workflows/test-android-emulator-webview.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
name: test android emulator webview

on:
workflow_dispatch:
schedule:
- cron: '0 */24 * * *'

jobs:
run_tests:
name: Flutter ${{ matrix.flutter_version }} webview on emulator.wtf
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter_version }} webview on emulator.wtf
TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }}
URL_TO_DETAILS: ${{ steps.tests_step.outputs.URL_TO_DETAILS }}

strategy:
fail-fast: false
matrix:
flutter_version: ['3.13.x']

defaults:
run:
working-directory: packages/patrol/example

steps:
- name: Clone repository
uses: actions/checkout@v3

- name: Set up Java
uses: actions/setup-java@v3
with:
distribution: temurin
java-version: 17

- name: Gradle cache
uses: gradle/[email protected]
with:
generate-job-summary: false

- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
flutter-version: ${{ matrix.flutter_version }}

- name: Preload Flutter artifacts
run: flutter precache

- name: Set up Patrol CLI
working-directory: packages/patrol_cli
run: dart pub global activate --source path . && patrol

- name: Generate Gradle wrapper
run: flutter build apk --debug --flavor=does-not-exist || true

- name: Install ew-cli
run: |
mkdir -p "$HOME/bin"
curl "https://maven.emulator.wtf/releases/ew-cli" -o "$HOME/bin/ew-cli"
chmod a+x "$HOME/bin/ew-cli"
echo "$HOME/bin" >> $GITHUB_PATH
echo "EW_API_TOKEN=${{ secrets.EW_API_TOKEN }}" >> $GITHUB_ENV
- name: Set tests to include
run: |
TESTS_TO_INCLUDE="webview_hackernews_test,\
webview_leancode_test,\
webview_stackoverflow_test"
target_paths=""
for target in $(echo $TESTS_TO_INCLUDE | tr ',' '\n'); do
target_paths+="integration_test/${target}.dart,"
done
target_paths="${target_paths%,}"
echo "INCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
- name: patrol build android
run: patrol build android --target ${{ env.INCLUDED_TESTS }} --verbose

- name: Upload APKs to emulator.wtf and wait for tests to finish
id: tests_step
run: |
set +e
output="$(ew-cli \
emulatorwtf.yaml:ci_webview --json \
--display-name "Patrol example app (${GITHUB_SHA::7})")"
TESTS_EXIT_CODE=$?
set -e
# Extract the results URL and write it to Github Summmary
link="$(echo "$output" | jq -r ".resultsUrl")"
echo "[Test details on emulator.wtf]($link) (LeanCode members only)" >> "$GITHUB_STEP_SUMMARY"
echo "URL_TO_DETAILS=$link" >> "$GITHUB_OUTPUT"
echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT"
exit $TESTS_EXIT_CODE
call_send_slack_message:
name: Notify on Slack
uses: ./.github/workflows/send-slack-message.yaml
needs: run_tests
if: always()
with:
TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }}
SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }}
URL_TO_DETAILS: ${{ needs.run_tests.outputs.URL_TO_DETAILS }}
secrets: inherit
8 changes: 6 additions & 2 deletions .github/workflows/test-android-emulator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
java-version: 17

- name: Gradle cache
uses: gradle/gradle-build-action@v2
uses: gradle/gradle-build-action@v2.8.0
with:
generate-job-summary: false

Expand Down Expand Up @@ -68,12 +68,16 @@ jobs:
run: |
TESTS_TO_EXCLUDE="service_airplane_mode_test,\
service_bluetooth_test,\
notifications_test"
notifications_test,\
webview_hackernews_test,\
webview_leancode_test,\
webview_stackoverflow_test"
target_paths=""
for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do
target_paths+="integration_test/${target}.dart,"
done
target_paths="${target_paths%,}"
echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT"
echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/test-ios-device.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ jobs:
for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do
target_paths+="integration_test/${target}.dart,"
done
target_paths="${target_paths%,}"
echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT"
echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
Expand Down
146 changes: 146 additions & 0 deletions .github/workflows/test-ios-simulator-webview.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
name: test ios simulator webview

on:
workflow_dispatch:
schedule:
- cron: '0 */24 * * *'

jobs:
run_tests:
name: Flutter ${{ matrix.flutter_version }} webview on ${{ matrix.device_model }} (${{ matrix.os_version }}) simulator
runs-on: macos-latest
timeout-minutes: 40
outputs:
SLACK_MESSAGE_TITLE: Flutter ${{ matrix.flutter_version }} on ${{ matrix.os }} ${{ matrix.os_version }} simulator
TESTS_EXIT_CODE: ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }}
FAILURE_STATUS: ${{ steps.status_step.outputs.FAILURE_STATUS }}
ERROR_STATUS: ${{ steps.status_step.outputs.ERROR_STATUS }}

strategy:
fail-fast: false
matrix:
flutter_version: ['3.13.x']
device_model: [iPhone 14]
os: [iOS]
os_version: ['16.2']

defaults:
run:
working-directory: packages/patrol/example

steps:
- name: Clone repository
uses: actions/checkout@v3

- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
flutter-version: ${{ matrix.flutter_version }}

- name: Preload Flutter artifacts
run: flutter precache

- name: Set up Patrol CLI
working-directory: packages/patrol_cli
run: dart pub global activate --source path . && patrol

- name: Start iOS simulator
uses: futureware-tech/simulator-action@v2
with:
model: ${{ matrix.device_model }}
os: ${{ matrix.os }}
os_version: ${{ matrix.os_version }}
erase_before_boot: true
shutdown_after_job: true

- name: Set tests to include
run: |
TESTS_TO_INCLUDE="webview_hackernews_test,\
webview_leancode_test,\
webview_stackoverflow_test"
target_paths=""
for target in $(echo $TESTS_TO_INCLUDE | tr ',' '\n'); do
target_paths+="integration_test/${target}.dart,"
done
target_paths="${target_paths%,}"
echo "INCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
- name: Run tests
id: tests_step
run: |
xcrun simctl io booted recordVideo --codec=h264 "${{ matrix.device_model }}.mp4" &
recordingpid="$!"
xcrun simctl spawn booted log stream --type log --color none > all_simulator_logs.txt &
logpid="$!"
sleep 10 # See https://github.com/leancodepl/patrol/issues/1282
TESTS_EXIT_CODE=0
patrol test --target ${{ env.INCLUDED_TESTS }} --verbose || TESTS_EXIT_CODE=$?
kill -SIGINT $recordingpid
kill -SIGINT $logpid
echo "TESTS_EXIT_CODE=$TESTS_EXIT_CODE" >> "$GITHUB_OUTPUT"
exit $TESTS_EXIT_CODE
- name: Check if something went wrong
id: status_step
if: always()
run: >
if [ -z ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} ]; then
echo "ERROR_STATUS=error" >> "$GITHUB_OUTPUT";
elif [ ! ${{ steps.tests_step.outputs.TESTS_EXIT_CODE }} == 0 ]; then
echo "FAILURE_STATUS=failure" >> "$GITHUB_OUTPUT";
fi;
- name: Find xcresult path
if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
run: |
brew install coreutils # to provide realpath binary
echo "XCRESULT_PATH=$(realpath build/ios_results_*.xcresult)" >> $GITHUB_ENV
- name: Publish test report to summary
if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
uses: kishikawakatsumi/xcresulttool@v1
with:
title: Patrol tests on ${{ matrix.device_model }}
upload-bundles: never
path: |
${{ env.XCRESULT_PATH }}
- name: Upload XCRESULT test result to artifacts
if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
uses: actions/upload-artifact@v3
with:
name: Test result from ${{ matrix.device_model }}.xcresult
path: ${{ env.XCRESULT_PATH }}

- name: Upload simulator logs to artifacts
if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
uses: actions/upload-artifact@v3
with:
name: Logs from ${{ matrix.device_model }}
path: |
${{ github.workspace }}/packages/patrol/example/all_simulator_logs.txt
- name: Upload captured video to artifacts
if: ${{ (failure() || success()) && contains(fromJson('["success", "failure"]'), steps.tests_step.conclusion) }}
uses: actions/upload-artifact@v3
with:
name: Captured video from ${{ matrix.device_model }}.mp4
path: ${{ github.workspace }}/packages/patrol/example/${{ matrix.device_model }}.mp4

call_send_slack_message:
name: Notify on Slack
uses: ./.github/workflows/send-slack-message.yaml
needs: run_tests
if: always()
with:
TESTS_EXIT_CODE: ${{ needs.run_tests.outputs.TESTS_EXIT_CODE }}
FAILURE_STATUS: ${{ needs.run_tests.outputs.FAILURE_STATUS }}
ERROR_STATUS: ${{ needs.run_tests.outputs.ERROR_STATUS }}
SLACK_MESSAGE_TITLE: ${{ needs.run_tests.outputs.SLACK_MESSAGE_TITLE }}
secrets: inherit
4 changes: 3 additions & 1 deletion .github/workflows/test-ios-simulator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ jobs:
run: dart pub global activate --source path . && patrol

- name: Start iOS simulator
id: start_simulator
uses: futureware-tech/simulator-action@v2
with:
model: ${{ matrix.device_model }}
Expand All @@ -67,12 +66,15 @@ jobs:
service_cellular_test,\
service_wifi_test,\
webview_stackoverflow_test,\
webview_leancode_test,\
webview_hackernews_test,\
swipe_test"
target_paths=""
for target in $(echo $TESTS_TO_EXCLUDE | tr ',' '\n'); do
target_paths+="integration_test/${target}.dart,"
done
target_paths="${target_paths%,}"
echo "EXCLUDED_TESTS=$TESTS_TO_EXCLUDE" >> "$GITHUB_OUTPUT"
echo "EXCLUDED_TESTS=$target_paths" >> "$GITHUB_ENV"
Expand Down
2 changes: 1 addition & 1 deletion docs/getting-started.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ Psst... Android is a bit easier to set up, so we recommend starting with it!
end
```

6. Create an empty file `integration_test/example_test.dart`. From the command line, run:
6. Create an empty file `integration_test/example_test.dart` in the root of your Flutter project. From the command line, run:

```
$ flutter build ios --config-only integration_test/example_test.dart
Expand Down
Loading

0 comments on commit 4211399

Please sign in to comment.