Use test bundling on CI (#1248)

Run `test android device`,  `test android emulator` and `test ios emulator` with test bundling

Co-authored-by: Bartek Pacia <[email protected]>

.github/workflows/test-android-device.yaml

@@ -7,11 +7,17 @@ on:
 
 jobs:
   run_tests:
-    name: Test on Android device on Firebase Test Lab
+    name: "Flutter ${{ matrix.flutter-version }} on Firebase Test Lab"
     runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 60
     outputs:
-      status: ${{ steps.set_status.outputs.status }}
+      failure_status: ${{ steps.set_failure.outputs.failure_status }}
+      error_status: ${{ steps.set_error.outputs.error_status }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        flutter-version: ['3.7.x']
 
     defaults:
       run:
@@ -43,82 +49,128 @@ jobs:
       - name: Set up Flutter
         uses: subosito/flutter-action@v2
         with:
-          channel: ${{ matrix.flutter-channel }}
+          flutter-version: ${{ matrix.flutter-version }}
+          cache: true
 
       - name: Preload Flutter artifacts
         run: flutter precache
 
       - name: Set up Patrol CLI
-        run: dart pub global activate patrol_cli
+        working-directory: packages/patrol_cli
+        run: dart pub global activate --source path . && patrol
 
       - name: Generate Gradle wrapper
         run: flutter build apk --debug --flavor=does-not-exist || true
 
-      - run: ./run_android_testlab integration_test/android_app_test.dart
-
-      - run: ./run_android_testlab integration_test/example_test.dart
-        if: success() || failure()
-
-      - run: ./run_android_testlab integration_test/notifications_test.dart
+      - name: patrol build android
+        run: |
+          patrol build android \
+            --exclude integration_test/permissions_location_test.dart \
+            --exclude integration_test/service_airplane_mode_test.dart \
+            --exclude integration_test/service_bluetooth_test.dart \
+            --exclude integration_test/webview_hackernews_test.dart \
+            --exclude integration_test/webview_leancode_test.dart \
+            --exclude integration_test/webview_login_test.dart \
+            --exclude integration_test/webview_stackoverflow_test.dart
+
+      - name: Upload APKs to Firebase Test Lab and wait for tests to finish
+        id: tests_step
+        run: ./run_android_testlab
+
+      # RESULTS_DIR_NAME is created by run_android_testlab script
+      - name: Get test outputs from Google Cloud Storage
         if: success() || failure()
+        run: |
+          mkdir "test_outputs"
+          gsutil -m cp -r "gs://patrol_runs/${{ env.RESULTS_DIR_NAME }}/oriole-33-en-portrait/*" "test_outputs"
 
-      - run: ./run_android_testlab integration_test/open_app_test.dart
+      - name: Publish test report to summary
+        uses: mikepenz/action-junit-report@v3
         if: success() || failure()
+        with:
+          check_name: Patrol tests
+          report_paths: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_result_1.xml
+          detailed_summary: true
+          include_passed: true
 
-      - run: ./run_android_testlab integration_test/open_quick_settings_test.dart
+      - name: Generate test report as check run
         if: success() || failure()
+        uses: dorny/test-reporter@v1
+        with:
+          name: Patrol Tests
+          path: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_result_1.xml
+          reporter: java-junit
+          fail-on-error: false
 
-      - run: ./run_android_testlab integration_test/permissions_location_test.dart
-        if: ${{ false }} # Doesn't handle the "Google location services" popup
-
-      - run: ./run_android_testlab integration_test/permissions_many_test.dart
+      - name: Upload XML test report to artifacts
+        uses: actions/upload-artifact@v3
         if: success() || failure()
+        with:
+          name: XML test report
+          path: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_result_1.xml
 
-      - run: ./run_android_testlab integration_test/service_airplane_mode_test.dart
-        if: ${{ false }} # Not implemented on Android
-
-      - run: ./run_android_testlab integration_test/service_bluetooth_test.dart
-        if: ${{ false }} # Not implemented on Android
-
-      - run: ./run_android_testlab integration_test/service_cellular_test.dart
+      - name:  Upload captured video to artifacts
+        uses: actions/upload-artifact@v3
         if: success() || failure()
+        with:
+          name: Captured video
+          path: ${{ github.workspace }}/packages/patrol/example/test_outputs/video.mp4
 
-      - run: ./run_android_testlab integration_test/service_dark_mode_test.dart
+      - name: Upload test cases to artifacts
+        uses: actions/upload-artifact@v3
         if: success() || failure()
+        with:
+          name: Test cases (logcat and captured video per test target)
+          path: ${{ github.workspace }}/packages/patrol/example/test_outputs/test_cases
 
-      - run: ./run_android_testlab integration_test/service_wifi_test.dart
+      - name: Upload device logs to artifacts
         if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Logs (logcat)
+          path: ${{ github.workspace }}/packages/patrol/example/test_outputs/logcat
 
-      - run: ./run_android_testlab integration_test/swipe_test.dart
-        if: success() || failure()
+      - name: Check if failed test occured
+        id: set_failure
+        if: always()
+        run: >
+          if [ "${{ steps.tests_step.conclusion }}" == "failure" ]; then
+            echo "failure_status=failure" >> "$GITHUB_OUTPUT";
+          fi;
 
-      - name: Set job status
-        id: set_status
-        if: success() || failure()
-        run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT"
+      - name: Check if error during run occurred
+        id: set_error
+        if: always()
+        run: >
+          if [ "${{ steps.tests_step.conclusion }}" == "skipped" ] || [ "${{ steps.tests_step.conclusion }}" == "cancelled" ]; then
+            echo "error_status=error" >> "$GITHUB_OUTPUT";
+          fi;
 
   slack_notify:
     name: Notify on Slack
     runs-on: ubuntu-latest
     needs: run_tests
-    if: always()
+    if: ${{ always() }}
 
     steps:
       - name: Set Slack message
         id: slack_message
         env:
-          STATUS: ${{ needs.run_tests.outputs.status }}
+          failure_status: ${{ needs.run_tests.outputs.failure_status }}
+          error_status: ${{ needs.run_tests.outputs.error_status }}
         run: >
-          status="${{ env.STATUS }}"
-          url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}";
-          message="";
-          if [ "$STATUS" = "failure" ]; then
+          if [ ! -z "$failure_status" ]; then
             message="There were failing tests 💥 ";
-          elif [ "$STATUS" = "success" ]; then
-            message="All tests have passed ✅ ";
-          else
+            status="failure";
+          elif [ ! -z "$error_status" ]; then
             message="Something went wrong ⚠️";
+            status="failure";
+          else
+            message="All tests have passed on ✅ ";
+            status="success";
           fi;
+
+          url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}";
           echo "message=$message" >> $GITHUB_OUTPUT;
           echo "url=$url" >> $GITHUB_OUTPUT;
           echo "status=$status" >> $GITHUB_OUTPUT;
@@ -135,4 +187,4 @@ jobs:
           SLACK_MESSAGE: |
             ${{ steps.slack_message.outputs.message }}
 
-            See workflow run <${{ steps.slack_message.outputs.url }}|here>
+            See workflow run <${{ steps.slack_message.outputs.url }}|here>

.github/workflows/test-android-emulator-2.yaml

@@ -1,4 +1,4 @@
-name: test android emulator
+name: test android emulator 2
 
 on:
   workflow_dispatch: