Merge pull request #1751 from leancodepl/lifecycle_callbacks_setupall

Add support for advanced test lifecycle callbacks - `setUpAll`

dev_docs/GUIDE.md

@@ -1,17 +1,53 @@
 # Working on the test bundling feature
 
+_Test bundling_, also known as _native automation_, is a core feature of Patrol.
+It bridges the native world of tests on Android and iOS with the Flutter/Dart
+world of tests.
+
+It lives in the [patrol package](../packages/patrol).
+
+To learn more about test bundling, [read this article][test_bundling_article].
+
+This document is a collection of tips and tricks to make it easier to work on
+test bundling-related code.
+
+### Tools
+
 `adb logcat` is your friend. Spice it up with `-v color`. If you need something
 more powerful, check out [`purr`](https://github.com/google/purr).
 
+### Show Dart-side logs only
+
+Search for `flutter :`.
+
 ### Find out when a test starts
 
 Search for `TestRunner: started`.
 
 ```
 09-21 12:24:09.223 23387 23406 I TestRunner: started: runDartTest[callbacks_test testA](pl.leancode.patrol.example.MainActivityTest)
-
 ```
 
 ### Find out when a test ends
 
 Search for `TestRunner: finished`.
+
+### I made some changes to test bundling code that result in a deadlock
+
+This can often happen when editing test bundling code. Because of various
+limitations of the `test` package, which Patrol has to base on, test bundling
+code is full of shared global mutable state and unobvious things happening in
+parallel.
+
+When trying to find the cause of a deadlock:
+
+- search for `await`s in custom functions provided by Patrol (e.g.
+  `patrolTest()` and `patrolSetUpAll()`) and global lifecycle callbacks
+  registered by the generated Dart test bundle or PatrolBinding (e.g.
+  `tearDown()`s)
+- Use `print`s amply to pinpint where the code is stuck.
+
+In the future, we should think about how to refactor this code to be more
+maintainable and simpler.
+
+[test_bundling_article]: https://leancode.co/blog/patrol-2-0-improved-flutter-ui-testing

packages/patrol/android/src/main/kotlin/pl/leancode/patrol/AutomatorServer.kt

@@ -1,5 +1,6 @@
 package pl.leancode.patrol
 
+import androidx.test.platform.app.InstrumentationRegistry
 import pl.leancode.patrol.contracts.Contracts.ConfigureRequest
 import pl.leancode.patrol.contracts.Contracts.DarkModeRequest
 import pl.leancode.patrol.contracts.Contracts.EnterTextRequest
@@ -9,6 +10,7 @@ import pl.leancode.patrol.contracts.Contracts.GetNotificationsRequest
 import pl.leancode.patrol.contracts.Contracts.GetNotificationsResponse
 import pl.leancode.patrol.contracts.Contracts.HandlePermissionRequest
 import pl.leancode.patrol.contracts.Contracts.HandlePermissionRequestCode
+import pl.leancode.patrol.contracts.Contracts.MarkLifecycleCallbackExecutedRequest
 import pl.leancode.patrol.contracts.Contracts.OpenAppRequest
 import pl.leancode.patrol.contracts.Contracts.OpenQuickSettingsRequest
 import pl.leancode.patrol.contracts.Contracts.PermissionDialogVisibleRequest
@@ -210,4 +212,9 @@ class AutomatorServer(private val automation: Automator) : NativeAutomatorServer
     override fun markPatrolAppServiceReady() {
         PatrolServer.appReady.open()
     }
+
+    override fun markLifecycleCallbackExecuted(request: MarkLifecycleCallbackExecutedRequest) {
+        val instrumentation = InstrumentationRegistry.getInstrumentation() as PatrolJUnitRunner
+        instrumentation.markLifecycleCallbackExecuted(request.name)
+    }
 }

packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt

@@ -33,6 +33,20 @@ class PatrolAppServiceClient {
         return result.group
     }
 
+    @Throws(PatrolAppServiceClientException::class)
+    fun listDartLifecycleCallbacks(): Contracts.ListDartLifecycleCallbacksResponse {
+        Logger.i("PatrolAppServiceClient.listDartLifecycleCallbacks()")
+        val result = client.listDartLifecycleCallbacks()
+        return result
+    }
+
+    @Throws(PatrolAppServiceClientException::class)
+    fun setLifecycleCallbacksState(data: Map<String, Boolean>): Contracts.Empty {
+        Logger.i("PatrolAppServiceClient.setLifecycleCallbacksState()")
+        val result = client.setLifecycleCallbacksState(Contracts.SetLifecycleCallbacksStateRequest(data))
+        return result
+    }
+
     @Throws(PatrolAppServiceClientException::class)
     fun runDartTest(name: String): Contracts.RunDartTestResponse {
         Logger.i("PatrolAppServiceClient.runDartTest($name)")

packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java

@@ -5,19 +5,33 @@
 
 package pl.leancode.patrol;
 
+import android.annotation.SuppressLint;
 import android.app.Instrumentation;
+import android.content.ContentResolver;
 import android.content.Intent;
+import android.net.Uri;
 import android.os.Bundle;
 import androidx.test.platform.app.InstrumentationRegistry;
 import androidx.test.runner.AndroidJUnitRunner;
-
+import androidx.test.services.storage.file.HostedFile;
+import androidx.test.services.storage.internal.TestStorageUtil;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
 import pl.leancode.patrol.contracts.Contracts;
+import pl.leancode.patrol.contracts.Contracts.ListDartLifecycleCallbacksResponse;
 import pl.leancode.patrol.contracts.PatrolAppServiceClientException;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
+import java.util.Map;
+import java.util.Scanner;
 
 import static pl.leancode.patrol.contracts.Contracts.DartGroupEntry;
 import static pl.leancode.patrol.contracts.Contracts.RunDartTestResponse;
@@ -27,9 +41,30 @@
  * A customized AndroidJUnitRunner that enables Patrol on Android.
  * </p>
  */
+@SuppressLint({"UnsafeOptInUsageError", "RestrictedApi"})
 public class PatrolJUnitRunner extends AndroidJUnitRunner {
     public PatrolAppServiceClient patrolAppServiceClient;
 
+    /**
+     * <p>
+     * Available only after onCreate() has been run.
+     * </p>
+     */
+    protected boolean isInitialRun;
+
+    private ContentResolver getContentResolver() {
+        return InstrumentationRegistry.getInstrumentation().getTargetContext().getContentResolver();
+    }
+
+    private Uri stateFileUri = HostedFile.buildUri(
+            HostedFile.FileHost.OUTPUT,
+            "patrol_callbacks.json"
+    );
+
+    public boolean isInitialRun() {
+        return isInitialRun;
+    }
+
     @Override
     protected boolean shouldWaitForActivitiesToComplete() {
         return false;
@@ -40,10 +75,10 @@ public void onCreate(Bundle arguments) {
         super.onCreate(arguments);
 
         // This is only true when the ATO requests a list of tests from the app during the initial run.
-        boolean isInitialRun = Boolean.parseBoolean(arguments.getString("listTestsForOrchestrator"));
+        this.isInitialRun = Boolean.parseBoolean(arguments.getString("listTestsForOrchestrator"));
 
         Logger.INSTANCE.i("--------------------------------");
-        Logger.INSTANCE.i("PatrolJUnitRunner.onCreate() " + (isInitialRun ? "(initial run)" : ""));
+        Logger.INSTANCE.i("PatrolJUnitRunner.onCreate() " + (this.isInitialRun ? "(initial run)" : ""));
     }
 
     /**
@@ -69,6 +104,7 @@ public void setUp(Class<?> activityClass) {
         // Currently, the only synchronization point we're interested in is when the app under test returns the list of tests.
         Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
         Intent intent = new Intent(Intent.ACTION_MAIN);
+        intent.putExtra("isInitialRun", isInitialRun);
         intent.setClassName(instrumentation.getTargetContext(), activityClass.getCanonicalName());
         intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
         instrumentation.getTargetContext().startActivity(intent);
@@ -94,7 +130,7 @@ public PatrolAppServiceClient createAppServiceClient() {
      * </p>
      */
     public void waitForPatrolAppService() {
-        final String TAG = "PatrolJUnitRunner.setUp(): ";
+        final String TAG = "PatrolJUnitRunner.waitForPatrolAppService(): ";
 
         Logger.INSTANCE.i(TAG + "Waiting for PatrolAppService to report its readiness...");
         PatrolServer.Companion.getAppReady().block();
@@ -105,6 +141,11 @@ public void waitForPatrolAppService() {
     public Object[] listDartTests() {
         final String TAG = "PatrolJUnitRunner.listDartTests(): ";
 
+        // This call should be in MainActivityTest.java, but that would require
+        // users to change that file in their projects, thus breaking backward
+        // compatibility.
+        handleLifecycleCallbacks();
+
         try {
             final DartGroupEntry dartTestGroup = patrolAppServiceClient.listDartTests();
             List<DartGroupEntry> dartTestCases = ContractsExtensionsKt.listTestsFlat(dartTestGroup, "");
@@ -121,12 +162,99 @@ public Object[] listDartTests() {
         }
     }
 
+    private void handleLifecycleCallbacks() {
+        if (isInitialRun()) {
+            Object[] lifecycleCallbacks = listLifecycleCallbacks();
+            saveLifecycleCallbacks(lifecycleCallbacks);
+        } else {
+            setLifecycleCallbacksState();
+        }
+    }
+
+    public Object[] listLifecycleCallbacks() {
+        final String TAG = "PatrolJUnitRunner.listLifecycleCallbacks(): ";
+
+        try {
+            final ListDartLifecycleCallbacksResponse response = patrolAppServiceClient.listDartLifecycleCallbacks();
+            final List<String> setUpAlls = response.getSetUpAlls();
+            Logger.INSTANCE.i(TAG + "Got Dart lifecycle callbacks: " + setUpAlls);
+
+            return setUpAlls.toArray();
+        } catch (PatrolAppServiceClientException e) {
+            Logger.INSTANCE.e(TAG + "Failed to list Dart lifecycle callbacks: ", e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void saveLifecycleCallbacks(Object[] callbacks) {
+        Map<String, Boolean> callbackMap = new HashMap<>();
+        for (Object callback : callbacks) {
+            callbackMap.put((String) callback, false);
+        }
+
+        writeStateFile(callbackMap);
+    }
+
+    public void markLifecycleCallbackExecuted(String name) {
+        Logger.INSTANCE.i("PatrolJUnitRunnerMarking.markLifecycleCallbackExecuted(" + name + ")");
+        Map<String, Boolean> state = readStateFile();
+        state.put(name, true);
+        writeStateFile(state);
+    }
+
+    private Map<String, Boolean> readStateFile() {
+        try {
+            InputStream inputStream = TestStorageUtil.getInputStream(stateFileUri, getContentResolver());
+            String content = convertStreamToString(inputStream);
+            Gson gson = new Gson();
+            Type typeOfHashMap = new TypeToken<Map<String, Boolean>>() {}.getType();
+            Map<String, Boolean> data = gson.fromJson(content, typeOfHashMap);
+            return data;
+        } catch (FileNotFoundException e) {
+            throw new RuntimeException("Failed to read state file", e);
+        }
+    }
+
+    private void writeStateFile(Map<String, Boolean> data) {
+        try {
+            OutputStream outputStream = TestStorageUtil.getOutputStream(stateFileUri, getContentResolver());
+            Gson gson = new Gson();
+            Type typeOfHashMap = new TypeToken<Map<String, Boolean>>() {}.getType();
+            String json = gson.toJson(data, typeOfHashMap);
+            outputStream.write(json.getBytes());
+            outputStream.write("\n".getBytes());
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to write state file", e);
+        }
+    }
+
+    static String convertStreamToString(InputStream inputStream) {
+        Scanner s = new Scanner(inputStream).useDelimiter("\\A");
+        return s.hasNext() ? s.next() : "";
+    }
+
+    /**
+     * Sets the state of lifecycle callbacks in the app.
+     * <p>
+     * This is required because the app is launched in a new process for each test.
+     */
+    public void setLifecycleCallbacksState() {
+        final String TAG = "PatrolJUnitRunner.setLifecycleCallbacksStateInApp(): ";
+
+        try {
+            patrolAppServiceClient.setLifecycleCallbacksState(readStateFile());
+        } catch (PatrolAppServiceClientException e) {
+            Logger.INSTANCE.e(TAG + "Failed to set lifecycle callbacks state in app: ", e);
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Requests execution of a Dart test and waits for it to finish.
      * Throws AssertionError if the test fails.
      */
     public RunDartTestResponse runDartTest(String name) {
-        final String TAG = "PatrolJUnitRunner.runDartTest(" + name + "): ";
+        final String TAG = "PatrolJUnitRunner.runDartTest(\"" + name + "\"): ";
 
         try {
             Logger.INSTANCE.i(TAG + "Requested execution");

packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolPlugin.kt

@@ -1,24 +1,52 @@
 package pl.leancode.patrol
 
 import io.flutter.embedding.engine.plugins.FlutterPlugin
+import io.flutter.embedding.engine.plugins.activity.ActivityAware
+import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding
 import io.flutter.plugin.common.MethodCall
 import io.flutter.plugin.common.MethodChannel
 import io.flutter.plugin.common.MethodChannel.MethodCallHandler
 import io.flutter.plugin.common.MethodChannel.Result
 
-class PatrolPlugin : FlutterPlugin, MethodCallHandler {
+class PatrolPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
     private lateinit var channel: MethodChannel
 
+    private var isInitialRun: Boolean? = null
+
     override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
         channel = MethodChannel(flutterPluginBinding.binaryMessenger, "pl.leancode.patrol/main")
         channel.setMethodCallHandler(this)
     }
 
     override fun onMethodCall(call: MethodCall, result: Result) {
-        result.notImplemented()
+        when (call.method) {
+            "isInitialRun" -> result.success(isInitialRun)
+            else -> result.notImplemented()
+        }
     }
 
     override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) {
         channel.setMethodCallHandler(null)
     }
+
+    override fun onAttachedToActivity(binding: ActivityPluginBinding) {
+        val intent = binding.activity.intent
+        if (!intent.hasExtra("isInitialRun")) {
+            throw IllegalStateException("PatrolPlugin must be initialized with intent having isInitialRun boolean")
+        }
+
+        isInitialRun = intent.getBooleanExtra("isInitialRun", false)
+    }
+
+    override fun onDetachedFromActivityForConfigChanges() {
+        // Do nothing
+    }
+
+    override fun onReattachedToActivityForConfigChanges(binding: ActivityPluginBinding) {
+        // Do nothing
+    }
+
+    override fun onDetachedFromActivity() {
+        // Do nothing
+    }
 }