From 28828e991dbc01e90f2c485853cecc9a2bfe4975 Mon Sep 17 00:00:00 2001 From: Filip Tuzimek <63547653+fylyppo@users.noreply.github.com> Date: Tue, 16 May 2023 22:31:29 +0200 Subject: [PATCH] Use test bundling on CI (#1248) Run `test android device`, `test android emulator` and `test ios emulator` with test bundling Co-authored-by: Bartek Pacia --- .github/workflows/test-android-device.yaml | 136 ++++++++---- .../workflows/test-android-emulator-2.yaml | 2 +- .github/workflows/test-android-emulator.yaml | 200 ++++++++++++------ .github/workflows/test-ios-simulator.yaml | 128 +++++++---- packages/patrol/example/run_android_testlab | 26 +-- 5 files changed, 318 insertions(+), 174 deletions(-) diff --git a/.github/workflows/test-android-device.yaml b/.github/workflows/test-android-device.yaml index e2549cb6b..43cc26185 100644 --- a/.github/workflows/test-android-device.yaml +++ b/.github/workflows/test-android-device.yaml @@ -7,11 +7,17 @@ on: jobs: run_tests: - name: Test on Android device on Firebase Test Lab + name: "Flutter ${{ matrix.flutter-version }} on Firebase Test Lab" runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 60 outputs: - status: ${{ steps.set_status.outputs.status }} + failure_status: ${{ steps.set_failure.outputs.failure_status }} + error_status: ${{ steps.set_error.outputs.error_status }} + + strategy: + fail-fast: false + matrix: + flutter-version: ['3.7.x'] defaults: run: @@ -43,82 +49,128 @@ jobs: - name: Set up Flutter uses: subosito/flutter-action@v2 with: - channel: ${{ matrix.flutter-channel }} + flutter-version: ${{ matrix.flutter-version }} + cache: true - name: Preload Flutter artifacts run: flutter precache - name: Set up Patrol CLI - run: dart pub global activate patrol_cli + working-directory: packages/patrol_cli + run: dart pub global activate --source path . && patrol - name: Generate Gradle wrapper run: flutter build apk --debug --flavor=does-not-exist || true - - run: ./run_android_testlab integration_test/android_app_test.dart - - - run: ./run_android_testlab integration_test/example_test.dart - if: success() || failure() - - - run: ./run_android_testlab integration_test/notifications_test.dart + - name: patrol build android + run: | + patrol build android \ + --exclude integration_test/permissions_location_test.dart \ + --exclude integration_test/service_airplane_mode_test.dart \ + --exclude integration_test/service_bluetooth_test.dart \ + --exclude integration_test/webview_hackernews_test.dart \ + --exclude integration_test/webview_leancode_test.dart \ + --exclude integration_test/webview_login_test.dart \ + --exclude integration_test/webview_stackoverflow_test.dart + + - name: Upload APKs to Firebase Test Lab and wait for tests to finish + id: tests_step + run: ./run_android_testlab + + # RESULTS_DIR_NAME is created by run_android_testlab script + - name: Get test outputs from Google Cloud Storage if: success() || failure() + run: | + mkdir "test_outputs" + gsutil -m cp -r "gs://patrol_runs/${{ env.RESULTS_DIR_NAME }}/oriole-33-en-portrait/*" "test_outputs" - - run: ./run_android_testlab integration_test/open_app_test.dart + - name: Publish test report to summary + uses: mikepenz/action-junit-report@v3 if: success() || failure() + with: + check_name: Patrol tests + report_paths: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_result_1.xml + detailed_summary: true + include_passed: true - - run: ./run_android_testlab integration_test/open_quick_settings_test.dart + - name: Generate test report as check run if: success() || failure() + uses: dorny/test-reporter@v1 + with: + name: Patrol Tests + path: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_result_1.xml + reporter: java-junit + fail-on-error: false - - run: ./run_android_testlab integration_test/permissions_location_test.dart - if: ${{ false }} # Doesn't handle the "Google location services" popup - - - run: ./run_android_testlab integration_test/permissions_many_test.dart + - name: Upload XML test report to artifacts + uses: actions/upload-artifact@v3 if: success() || failure() + with: + name: XML test report + path: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_result_1.xml - - run: ./run_android_testlab integration_test/service_airplane_mode_test.dart - if: ${{ false }} # Not implemented on Android - - - run: ./run_android_testlab integration_test/service_bluetooth_test.dart - if: ${{ false }} # Not implemented on Android - - - run: ./run_android_testlab integration_test/service_cellular_test.dart + - name: Upload captured video to artifacts + uses: actions/upload-artifact@v3 if: success() || failure() + with: + name: Captured video + path: ${{ github.workspace }}/packages/patrol/example/test_outputs/video.mp4 - - run: ./run_android_testlab integration_test/service_dark_mode_test.dart + - name: Upload test cases to artifacts + uses: actions/upload-artifact@v3 if: success() || failure() + with: + name: Test cases (logcat and captured video per test target) + path: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_cases - - run: ./run_android_testlab integration_test/service_wifi_test.dart + - name: Upload device logs to artifacts if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: Logs (logcat) + path: ${{ github.workspace }}/packages/patrol/example/test_outputs/logcat - - run: ./run_android_testlab integration_test/swipe_test.dart - if: success() || failure() + - name: Check if failed test occured + id: set_failure + if: always() + run: > + if [ "${{ steps.tests_step.conclusion }}" == "failure" ]; then + echo "failure_status=failure" >> "$GITHUB_OUTPUT"; + fi; - - name: Set job status - id: set_status - if: success() || failure() - run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT" + - name: Check if error during run occurred + id: set_error + if: always() + run: > + if [ "${{ steps.tests_step.conclusion }}" == "skipped" ] || [ "${{ steps.tests_step.conclusion }}" == "cancelled" ]; then + echo "error_status=error" >> "$GITHUB_OUTPUT"; + fi; slack_notify: name: Notify on Slack runs-on: ubuntu-latest needs: run_tests - if: always() + if: ${{ always() }} steps: - name: Set Slack message id: slack_message env: - STATUS: ${{ needs.run_tests.outputs.status }} + failure_status: ${{ needs.run_tests.outputs.failure_status }} + error_status: ${{ needs.run_tests.outputs.error_status }} run: > - status="${{ env.STATUS }}" - url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; - message=""; - if [ "$STATUS" = "failure" ]; then + if [ ! -z "$failure_status" ]; then message="There were failing tests 💥 "; - elif [ "$STATUS" = "success" ]; then - message="All tests have passed ✅ "; - else + status="failure"; + elif [ ! -z "$error_status" ]; then message="Something went wrong ⚠️"; + status="failure"; + else + message="All tests have passed on ✅ "; + status="success"; fi; + + url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; echo "message=$message" >> $GITHUB_OUTPUT; echo "url=$url" >> $GITHUB_OUTPUT; echo "status=$status" >> $GITHUB_OUTPUT; @@ -135,4 +187,4 @@ jobs: SLACK_MESSAGE: | ${{ steps.slack_message.outputs.message }} - See workflow run <${{ steps.slack_message.outputs.url }}|here> + See workflow run <${{ steps.slack_message.outputs.url }}|here> \ No newline at end of file diff --git a/.github/workflows/test-android-emulator-2.yaml b/.github/workflows/test-android-emulator-2.yaml index 6f7d058a2..48d90176b 100644 --- a/.github/workflows/test-android-emulator-2.yaml +++ b/.github/workflows/test-android-emulator-2.yaml @@ -1,4 +1,4 @@ -name: test android emulator +name: test android emulator 2 on: workflow_dispatch: diff --git a/.github/workflows/test-android-emulator.yaml b/.github/workflows/test-android-emulator.yaml index 9ba95ef53..f77ec5aad 100644 --- a/.github/workflows/test-android-emulator.yaml +++ b/.github/workflows/test-android-emulator.yaml @@ -6,18 +6,18 @@ on: - cron: '0 */12 * * *' jobs: - main: - name: Test on Android emulator (${{ matrix.api-level }}, ${{ matrix.target}}, ${{matrix.arch}}) - runs-on: macos-latest + run_tests: + name: "Flutter ${{ matrix.flutter-version }} on emulator.wtf" + runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + failure_status: ${{ steps.set_failure.outputs.failure_status }} + error_status: ${{ steps.set_error.outputs.error_status }} strategy: fail-fast: false matrix: - flutter-channel: [stable] - api-level: [28, 29, 30, 31, 33] - target: [google_apis] - arch: [x86_64] + flutter-version: ['3.7.x'] defaults: run: @@ -38,79 +38,143 @@ jobs: with: generate-job-summary: false - - name: AVD cache - uses: actions/cache@v3 - id: avd-cache - with: - path: | - ~/.android/avd/* - ~/.android/adbkey - ~/.android/adbkey.pub - key: avd-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }} - - - name: Create AVD and generate snapshot for caching - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - ram-size: 4096M - heap-size: 2048M - disk-size: 8192M - force-avd-creation: false - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - script: echo "Generated AVD snapshot for caching" - - name: Set up Flutter uses: subosito/flutter-action@v2 with: - channel: ${{ matrix.flutter-channel }} + flutter-version: ${{ matrix.flutter-version }} + cache: true - name: Preload Flutter artifacts run: flutter precache - name: Set up Patrol CLI - run: dart pub global activate patrol_cli + working-directory: packages/patrol_cli + run: dart pub global activate --source path . && patrol - name: Generate Gradle wrapper run: flutter build apk --debug --flavor=does-not-exist || true - - name: Start emulator and run integration tests - uses: reactivecircus/android-emulator-runner@v2 + - name: Install ew-cli + run: | + mkdir -p "$HOME/bin" + curl "https://maven.emulator.wtf/releases/ew-cli" -o "$HOME/bin/ew-cli" + chmod a+x "$HOME/bin/ew-cli" + echo "$HOME/bin" >> $GITHUB_PATH + echo "EW_API_TOKEN=${{ secrets.EW_API_TOKEN }}" >> $GITHUB_ENV + + - name: patrol build android + run: | + patrol build android \ + --exclude integration_test/permissions_location_test.dart \ + --exclude integration_test/service_airplane_mode_test.dart \ + --exclude integration_test/service_bluetooth_test.dart \ + --exclude integration_test/webview_hackernews_test.dart \ + --exclude integration_test/webview_leancode_test.dart \ + --exclude integration_test/webview_login_test.dart \ + --exclude integration_test/webview_stackoverflow_test.dart + + - name: Upload APKs to emulator.wtf and wait for tests to finish + id: tests_step + run: | + ew-cli \ + --app build/app/outputs/apk/debug/app-debug.apk \ + --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk \ + --num-shards 3 \ + --use-orchestrator \ + --clear-package-data \ + --record-video \ + --timeout 10m \ + --environment-variables packageName="pl.leancode.patrol.example" \ + --outputs-dir test_artifacts \ + --outputs summary,merged_results_xml,coverage,pulled_dirs,results_xml,logcat,captured_video + + - name: Publish test report to summary + uses: mikepenz/action-junit-report@v3 + if: success() || failure() with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - force-avd-creation: false - emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -wipe-data - disable-animations: false - working-directory: ${{ github.workspace }}/packages/patrol/example - script: > - adb logcat -c - - adb logcat > logcat.txt & - - patrol test - -t integration_test/android_app_test.dart - -t integration_test/example_test.dart - -t integration_test/notifications_test.dart - -t integration_test/open_app_test.dart - -t integration_test/open_quick_settings_test.dart - -t integration_test/permissions_location_test.dart - -t integration_test/permissions_many_test.dart - -t integration_test/service_cellular_test.dart - -t integration_test/service_dark_mode_test.dart - -t integration_test/service_wifi_test.dart - -t integration_test/swipe_test.dart - - - name: Upload logcat to artifacts - if: always() + check_name: Patrol tests + report_paths: ${{ github.workspace }}/packages/patrol/example/test_artifacts/results.xml + detailed_summary: true + include_passed: true + + - name: Generate test report as check run + if: success() || failure() + uses: dorny/test-reporter@v1 + with: + name: Patrol Tests + path: ${{ github.workspace }}/packages/patrol/example/test_artifacts/results.xml + reporter: java-junit + fail-on-error: false + + - name: Upload XML test report to artifacts + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: XML test report + path: ${{ github.workspace }}/packages/patrol/example/test_artifacts/results.xml + + - name: Upload test outputs to artifacts + if: success() || failure() uses: actions/upload-artifact@v3 with: - # FIXME: Reports and results of only the last test are uploadeds - name: Results for ${{ matrix.api-level }}-${{ matrix.target}}-${{matrix.arch}}.txt - path: | - ${{ github.workspace }}/packages/patrol/example/android/logcat.txt - ${{ github.workspace }}/packages/patrol/example/android/test-results - ${{ github.workspace }}/packages/patrol/example/android/test-reports + name: Test report, logs and captured videos + path: ${{ github.workspace }}/packages/patrol/example/test_artifacts + + - name: Check if failed test occured + id: set_failure + if: always() + run: > + if [ "${{ steps.tests_step.conclusion }}" == "failure" ]; then + echo "failure_status=failure" >> "$GITHUB_OUTPUT"; + fi; + + - name: Check if error during run occurred + id: set_error + if: always() + run: > + if [ "${{ steps.tests_step.conclusion }}" == "skipped" ] || [ "${{ steps.tests_step.conclusion }}" == "cancelled" ]; then + echo "error_status=error" >> "$GITHUB_OUTPUT"; + fi; + + slack_notify: + name: Notify on Slack + runs-on: ubuntu-latest + needs: run_tests + if: ${{ always() }} + + steps: + - name: Set Slack message + id: slack_message + env: + failure_status: ${{ needs.run_tests.outputs.failure_status }} + error_status: ${{ needs.run_tests.outputs.error_status }} + run: > + if [ ! -z "$failure_status" ]; then + message="There were failing tests 💥 "; + status="failure"; + elif [ ! -z "$error_status" ]; then + message="Something went wrong ⚠️"; + status="failure"; + else + message="All tests have passed ✅ "; + status="success"; + fi; + + url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; + echo "message=$message" >> $GITHUB_OUTPUT; + echo "url=$url" >> $GITHUB_OUTPUT; + echo "status=$status" >> $GITHUB_OUTPUT; + + - name: Share test results on Slack + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: ${{ vars.SLACK_STATUS_CHANNEL }} + SLACK_USERNAME: Patrol CI on GitHub Actions + SLACK_COLOR: ${{ steps.slack_message.outputs.status }} + SLACK_ICON: ${{ vars.SLACK_ICON }} + SLACK_TITLE: Test status (Android emulator on emulator.wtf) + SLACK_MESSAGE: | + ${{ steps.slack_message.outputs.message }} + + See workflow run <${{ steps.slack_message.outputs.url }}|here> \ No newline at end of file diff --git a/.github/workflows/test-ios-simulator.yaml b/.github/workflows/test-ios-simulator.yaml index 562dc8f3f..8f2de6da4 100644 --- a/.github/workflows/test-ios-simulator.yaml +++ b/.github/workflows/test-ios-simulator.yaml @@ -7,16 +7,17 @@ on: jobs: run_tests: - name: Test on ${{ matrix.device }}, ${{ matrix.os }} ${{ matrix.os_version }} + name: "Flutter ${{ matrix.flutter-version }} on ${{ matrix.device }}, ${{ matrix.os }} ${{ matrix.os_version }}" runs-on: macos-latest timeout-minutes: 60 outputs: - status: ${{ steps.set_status.outputs.status }} + failure_status: ${{ steps.set_failure.outputs.failure_status }} + error_status: ${{ steps.set_error.outputs.error_status }} strategy: fail-fast: false matrix: - flutter-channel: [stable] + flutter-version: ['3.7.x'] device: [iPhone 14, iPad (9th generation)] os: [iOS] os_version: ['16.2'] @@ -32,15 +33,18 @@ jobs: - name: Set up Flutter uses: subosito/flutter-action@v2 with: - channel: ${{ matrix.flutter-channel }} + flutter-version: ${{ matrix.flutter-version }} + cache: true - name: Preload Flutter artifacts run: flutter precache - name: Set up Patrol CLI - run: dart pub global activate patrol_cli + working-directory: packages/patrol_cli + run: dart pub global activate --source path . && patrol - name: Start iOS simulator + id: start_simulator uses: futureware-tech/simulator-action@v2 with: model: ${{ matrix.device }} @@ -52,69 +56,103 @@ jobs: - name: Set simulator location run: xcrun simctl location booted set 52.17469,21.03193 - - run: patrol test -t integration_test/example_test.dart - if: success() || failure() - - - run: patrol test -t integration_test/notifications_test.dart + - name: Run tests + id: tests_step + run: | + xcrun simctl io booted recordVideo --codec=h264 "${{ matrix.device }}.mp4" & + recordingpid="$!" + EXIT_CODE=0 + + patrol test \ + --exclude integration_test/android_app_test.dart \ + --exclude integration_test/service_airplane_mode_test.dart \ + --exclude integration_test/service_bluetooth_test.dart \ + --exclude integration_test/service_cellular_test.dart \ + --exclude integration_test/service_wifi_test.dart \ + --exclude integration_test/swipe_test.dart \ + --exclude integration_test/webview_leancode_test.dart \ + --exclude integration_test/webview_login_test.dart \ + --exclude integration_test/webview_stackoverflow_test.dart || EXIT_CODE=$? + + kill $recordingpid + exit $EXIT_CODE + + - name: Publish test report to summary if: success() || failure() + uses: kishikawakatsumi/xcresulttool@v1 + with: + title: "Patrol tests" + upload-bundles: "never" + path: | + ${{ env.XCRESULT_PATH }} - - run: patrol test -t integration_test/open_app_test.dart + - name: Find xcresult path if: success() || failure() + run: echo "XCRESULT_PATH=$(find ~/Library/Developer/Xcode/DerivedData -name "*.xcresult" -type d)" >> $GITHUB_ENV - - run: patrol test -t integration_test/open_quick_settings_test.dart - if: success() || failure() # should work, but is empty on Simulator - - - run: patrol test -t integration_test/permissions_location_test.dart + - name: Upload XCRESULT test report to artifacts if: success() || failure() - - - run: patrol test -t integration_test/permissions_many_test.dart + uses: actions/upload-artifact@v3 + with: + name: XCRESULT test report from ${{ matrix.device }} + path: | + ${{ env.XCRESULT_PATH }} + + - name: Upload simulator logs to artifacts if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: Logs from ${{ matrix.device }} + path: ~/Library/Logs/CoreSimulator/${{ steps.start_simulator.outputs.udid }}/system.log - - run: patrol test -t integration_test/service_cellular_test.dart - if: ${{ false }} # Not on Simulator - - - run: patrol test -t integration_test/service_dark_mode_test.dart + - name: Upload captured video to artifacts if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: Captured video from ${{ matrix.device }} + path: ${{ github.workspace }}/packages/patrol/example/${{ matrix.device }}.mp4 - - run: patrol test -t integration_test/service_wifi_test.dart - if: ${{ false }} # Not on Simulator - - - run: patrol test -t webview_hackernews_test.dart - if: ${{ false }} # https://github.com/leancodepl/patrol/issues/1139 - - run: patrol test -t webview_leancode_test.dart - if: ${{ false }} # https://github.com/leancodepl/patrol/issues/1139 - - run: patrol test -t webview_login_test.dart - if: ${{ false }} # https://github.com/leancodepl/patrol/issues/1139 - - run: patrol test -t webview_stackoverflow_test.dart - if: ${{ false }} # https://github.com/leancodepl/patrol/issues/1139 + - name: Check if failed test occured + id: set_failure + if: always() + run: > + if [ "${{ steps.tests_step.conclusion }}" == "failure" ]; then + echo "failure_status=failure" >> "$GITHUB_OUTPUT"; + fi; - - name: Set job status - id: set_status - if: success() || failure() - run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT" + - name: Check if error during run occurred + id: set_error + if: always() + run: > + if [ "${{ steps.tests_step.conclusion }}" == "skipped" ] || [ "${{ steps.tests_step.conclusion }}" == "cancelled" ]; then + echo "error_status=error" >> "$GITHUB_OUTPUT"; + fi; slack_notify: name: Notify on Slack runs-on: ubuntu-latest needs: run_tests - if: always() + if: ${{ always() }} steps: - name: Set Slack message id: slack_message env: - STATUS: ${{ needs.run_tests.outputs.status }} + failure_status: ${{ needs.run_tests.outputs.failure_status }} + error_status: ${{ needs.run_tests.outputs.error_status }} run: > - status="${{ env.STATUS }}" - url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; - message=""; - if [ "$STATUS" = "failure" ]; then + if [ ! -z "$failure_status" ]; then message="There were failing tests 💥 "; - elif [ "$STATUS" = "success" ]; then - message="All tests have passed ✅ "; - else + status="failure"; + elif [ ! -z "$error_status" ]; then message="Something went wrong ⚠️"; + status="failure"; + else + message="All tests have passed ✅ "; + status="success"; fi; + + url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; echo "message=$message" >> $GITHUB_OUTPUT; echo "url=$url" >> $GITHUB_OUTPUT; echo "status=$status" >> $GITHUB_OUTPUT; @@ -131,4 +169,4 @@ jobs: SLACK_MESSAGE: | ${{ steps.slack_message.outputs.message }} - See workflow run <${{ steps.slack_message.outputs.url }}|here> + See workflow run <${{ steps.slack_message.outputs.url }}|here> \ No newline at end of file diff --git a/packages/patrol/example/run_android_testlab b/packages/patrol/example/run_android_testlab index 1862daec3..e3d4b311a 100755 --- a/packages/patrol/example/run_android_testlab +++ b/packages/patrol/example/run_android_testlab @@ -2,29 +2,19 @@ set -euo pipefail cd "$(dirname "$0")" -_usage() { - printf "usage: ./run_android_testlab \n" - printf "\tfor example:\n" - printf "\t\$ ./run_android_testlab integration_test/app_test.dart\n" -} +# This script assumes that APKs are already built. -target="${1:-}" - -if [ -z "$target" ]; then - _usage - exit 1 -fi - -target_path="$(realpath "$target")" - -flutter pub get -patrol build android --target "$target_path" +RESULTS_DIR_NAME="$(date +"%Y-%m-%d_%H:%M:%S")" +echo "RESULTS_DIR_NAME=$RESULTS_DIR_NAME" >> "$GITHUB_ENV" exec gcloud firebase test android run \ --type instrumentation \ --app build/app/outputs/apk/debug/app-debug.apk \ --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk \ --device model=oriole,version=33,locale=en,orientation=portrait \ - --timeout 1m \ - --results-bucket=patrol_runs + --timeout 10m \ + --results-bucket="patrol_runs" \ + --results-dir="$RESULTS_DIR_NAME" \ + --use-orchestrator \ + --environment-variables clearPackageData=true,packageName="pl.leancode.patrol.example" # --device model=oriole,version=31,locale=en,orientation=portrait \