From bdf12eef6acff9c1d0b053eab2dc6e5bf7fe2522 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 28 Sep 2023 12:45:24 +0200 Subject: [PATCH 01/77] create CustomPatrolJUnitRunner --- .../patrol/CustomPatrolJUnitRunner.kt | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt new file mode 100644 index 000000000..94fa44674 --- /dev/null +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt @@ -0,0 +1,52 @@ +package pl.leancode.patrol + +import android.os.Build +import android.os.Bundle +import androidx.annotation.OptIn +import androidx.test.annotation.ExperimentalTestApi +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.services.storage.TestStorage +import org.junit.runner.* +import org.junit.runner.notification.* +import java.io.FileNotFoundException + +@OptIn(ExperimentalTestApi::class) +class CustomPatrolJUnitRunner : PatrolJUnitRunner() { + private var testStorage: TestStorage? = null + override fun onCreate(arguments: Bundle) { + super.onCreate(arguments) + testStorage = TestStorage() + try { + testStorage!!.openOutputFile("patrol_state/teardowns.txt") + } catch (e: FileNotFoundException) { + throw RuntimeException(e) + } + + val listeners = listOf( + arguments.getCharSequence("listener"), + PatrolStoragePermissionListener::class.java.name, + ).joinToString(separator = ",") + + arguments.putCharSequence("listener", listeners) + super.onCreate(arguments) + } +} + +class PatrolStoragePermissionListener : RunListener() { + @Throws(Exception::class) + override fun testRunStarted(description: Description) { + InstrumentationRegistry.getInstrumentation().uiAutomation.apply { + val testServicesPackage = "androidx.test.services" + when { + Build.VERSION.SDK_INT >= Build.VERSION_CODES.R -> { + executeShellCommand("appops set $testServicesPackage MANAGE_EXTERNAL_STORAGE allow") + } + + Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP -> { + executeShellCommand("pm grant $testServicesPackage android.permission.READ_EXTERNAL_STORAGE") + executeShellCommand("pm grant $testServicesPackage android.permission.WRITE_EXTERNAL_STORAGE") + } + } + } + } +} From a84464f9c74d0dae344183cd81b7ce70000fde6a Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 29 Sep 2023 10:59:54 +0200 Subject: [PATCH 02/77] write persistent file to `/sdcard/googletest/test_outputfiles` --- .../patrol/CustomPatrolJUnitRunner.kt | 42 +++++++++++++------ .../patrol/example/android/app/build.gradle | 2 +- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt index 94fa44674..35e84ff49 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt @@ -1,28 +1,28 @@ package pl.leancode.patrol +import android.annotation.SuppressLint import android.os.Build import android.os.Bundle import androidx.annotation.OptIn import androidx.test.annotation.ExperimentalTestApi import androidx.test.platform.app.InstrumentationRegistry import androidx.test.services.storage.TestStorage -import org.junit.runner.* -import org.junit.runner.notification.* -import java.io.FileNotFoundException +import org.junit.runner.Description +import org.junit.runner.notification.RunListener @OptIn(ExperimentalTestApi::class) class CustomPatrolJUnitRunner : PatrolJUnitRunner() { private var testStorage: TestStorage? = null + override fun onCreate(arguments: Bundle) { - super.onCreate(arguments) - testStorage = TestStorage() - try { - testStorage!!.openOutputFile("patrol_state/teardowns.txt") - } catch (e: FileNotFoundException) { - throw RuntimeException(e) - } - val listeners = listOf( +// try { +// val outputStream = testStorage!!.openOutputFile("patrol_state/teardowns.txt") +// } catch (e: FileNotFoundException) { +// throw RuntimeException(e) +// } + + val listeners = listOfNotNull( arguments.getCharSequence("listener"), PatrolStoragePermissionListener::class.java.name, ).joinToString(separator = ",") @@ -30,11 +30,27 @@ class CustomPatrolJUnitRunner : PatrolJUnitRunner() { arguments.putCharSequence("listener", listeners) super.onCreate(arguments) } + + override fun onStart() { + super.onStart() + + testStorage = TestStorage() + } } +@SuppressLint("UnsafeOptInUsageError") class PatrolStoragePermissionListener : RunListener() { - @Throws(Exception::class) - override fun testRunStarted(description: Description) { + + val testStorage by lazy { TestStorage() } + + override fun testStarted(description: Description) { + Logger.d("testStarted with description: $description") + + + val os = testStorage.openOutputFile("patrol.txt", true) + os.write("testStarted with description: $description\n".toByteArray()) + + InstrumentationRegistry.getInstrumentation().uiAutomation.apply { val testServicesPackage = "androidx.test.services" when { diff --git a/packages/patrol/example/android/app/build.gradle b/packages/patrol/example/android/app/build.gradle index cb9dd3f5c..4b082c7dc 100644 --- a/packages/patrol/example/android/app/build.gradle +++ b/packages/patrol/example/android/app/build.gradle @@ -43,7 +43,7 @@ android { targetSdk 33 versionCode flutterVersionCode.toInteger() versionName flutterVersionName - testInstrumentationRunner "pl.leancode.patrol.PatrolJUnitRunner" + testInstrumentationRunner "pl.leancode.patrol.CustomPatrolJUnitRunner" testInstrumentationRunnerArguments clearPackageData: "true" } From 2142bc6b1f9bb8f1aca9c798f307a6208da03cec Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 2 Oct 2023 21:06:42 +0200 Subject: [PATCH 03/77] satisfy ktlint --- .../kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt index 35e84ff49..d4855fa2f 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt @@ -15,7 +15,6 @@ class CustomPatrolJUnitRunner : PatrolJUnitRunner() { private var testStorage: TestStorage? = null override fun onCreate(arguments: Bundle) { - // try { // val outputStream = testStorage!!.openOutputFile("patrol_state/teardowns.txt") // } catch (e: FileNotFoundException) { @@ -24,7 +23,7 @@ class CustomPatrolJUnitRunner : PatrolJUnitRunner() { val listeners = listOfNotNull( arguments.getCharSequence("listener"), - PatrolStoragePermissionListener::class.java.name, + PatrolStoragePermissionListener::class.java.name ).joinToString(separator = ",") arguments.putCharSequence("listener", listeners) @@ -45,12 +44,10 @@ class PatrolStoragePermissionListener : RunListener() { override fun testStarted(description: Description) { Logger.d("testStarted with description: $description") - - + val os = testStorage.openOutputFile("patrol.txt", true) os.write("testStarted with description: $description\n".toByteArray()) - InstrumentationRegistry.getInstrumentation().uiAutomation.apply { val testServicesPackage = "androidx.test.services" when { From 0e9b69d5c848167e5d46d051c9d24d50f3dcecc3 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 2 Oct 2023 21:06:59 +0200 Subject: [PATCH 04/77] move callbacks_test to internal/ --- .../example/integration_test/{ => internal}/callbacks_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename packages/patrol/example/integration_test/{ => internal}/callbacks_test.dart (98%) diff --git a/packages/patrol/example/integration_test/callbacks_test.dart b/packages/patrol/example/integration_test/internal/callbacks_test.dart similarity index 98% rename from packages/patrol/example/integration_test/callbacks_test.dart rename to packages/patrol/example/integration_test/internal/callbacks_test.dart index 95d69ef4d..43959dbfe 100644 --- a/packages/patrol/example/integration_test/callbacks_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_test.dart @@ -3,7 +3,7 @@ import 'package:patrol/src/extensions.dart'; // ignore: depend_on_referenced_packages import 'package:test_api/src/backend/invoker.dart'; -import 'common.dart'; +import '../common.dart'; String get currentTest => Invoker.current!.fullCurrentTestName(); From 40bd9a46ef4390ab0d7eba3744b65f5ff9aa37cb Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 3 Oct 2023 00:02:05 +0200 Subject: [PATCH 05/77] WIP (try out `currentGroupFullName` approach) --- .../internal/callbacks_all_test.dart | 35 +++++++++++++++ .../patrol/example/test/callbacks_test.dart | 45 +++++++++++++++++++ packages/patrol/lib/src/common.dart | 19 ++++++++ packages/patrol/lib/src/global_state.dart | 4 ++ .../lib/src/native/patrol_app_service.dart | 11 +++++ 5 files changed, 114 insertions(+) create mode 100644 packages/patrol/example/integration_test/internal/callbacks_all_test.dart create mode 100644 packages/patrol/example/test/callbacks_test.dart diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart new file mode 100644 index 000000000..0995a81fe --- /dev/null +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -0,0 +1,35 @@ +import 'package:flutter/material.dart'; +import 'package:patrol/src/extensions.dart'; +// ignore: depend_on_referenced_packages +import 'package:test_api/src/backend/invoker.dart'; + +import '../common.dart'; + +String get currentTest => Invoker.current!.fullCurrentTestName(); + +void _print(String text) => print('PATROL_DEBUG: $text'); + +void main() { + patrolSetUpAll(() async { + await Future.delayed(Duration(seconds: 1)); + _print('setting up all before $currentTest'); + }); + + patrolTest('testA', nativeAutomation: true, _body); + patrolTest('testB', nativeAutomation: true, _body); + patrolTest('testC', nativeAutomation: true, _body); +} + +Future _body(PatrolTester $) async { + final testName = Invoker.current!.fullCurrentTestName(); + _print('test body: name=$testName'); + + await createApp($); + + await $(FloatingActionButton).tap(); + expect($(#counterText).text, '1'); + + await $(#textField).enterText(testName); + + await $.pumpAndSettle(duration: Duration(seconds: 2)); +} diff --git a/packages/patrol/example/test/callbacks_test.dart b/packages/patrol/example/test/callbacks_test.dart new file mode 100644 index 000000000..fb7f325d8 --- /dev/null +++ b/packages/patrol/example/test/callbacks_test.dart @@ -0,0 +1,45 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:patrol/src/extensions.dart'; +// ignore: depend_on_referenced_packages +import 'package:test_api/src/backend/invoker.dart'; + +const String requestedTest = 'testA'; + +String get currentTest => Invoker.current!.fullCurrentTestName(); + +void main() { + group('alpha', () { + setUpAll(() { + final groupName = Invoker.current!.liveTest.groups.last.name; + final individualName = Invoker.current!.liveTest.individualName; + print( + 'setUpAll: parentGroupName=$groupName, individualName=$individualName', + ); + }); + + setUp(() { + final groupName = Invoker.current!.liveTest.groups.last.name; + final individualName = Invoker.current!.liveTest.individualName; + print( + 'setUpAll: parentGroupName=$groupName, individualName=$individualName', + ); + }); + + patrolTest('testA', _body); + patrolTest('testB', _body); + patrolTest('testC', _body); + }); +} + +Future _body() async => print(Invoker.current!.fullCurrentTestName()); + +void patrolTest(String name, Future Function() body) { + test(name, () async { + final currentTest = Invoker.current!.fullCurrentTestName(); + + if (currentTest == requestedTest) { + print('Requested test $currentTest, will execute it'); + await body(); + } + }); +} diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index b6b479e00..7c81a0f2c 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -48,6 +48,25 @@ void patrolTearDown(Future Function() body) { }); } +/// A modification of [setUpAll] that works with Patrol's native automation. +void patrolSetUpAll(Future Function() body) { + setUpAll(() async { + final currentTest = global_state.currentTestFullName; + + final parentGroups = global_state.currentGroupFullName; + + final patrolAppService = PatrolBinding.instance.patrolAppService; + final currentSetUpAllIndex = patrolAppService.setUpAllCount += 1; + + final requestedToExecute = await PatrolBinding.instance.patrolAppService + .waitForExecutionRequest(currentTest); + + if (requestedToExecute) { + await body(); + } + }); +} + /// Like [testWidgets], but with support for Patrol custom finders. /// /// To customize the Patrol-specific configuration, set [config]. diff --git a/packages/patrol/lib/src/global_state.dart b/packages/patrol/lib/src/global_state.dart index 318574d3e..4abf8d264 100644 --- a/packages/patrol/lib/src/global_state.dart +++ b/packages/patrol/lib/src/global_state.dart @@ -24,6 +24,10 @@ String get currentTestFullName { return nameCandidate; } +String get currentGroupFullName { + return Invoker.current!.liveTest.groups.last.name; +} + /// Returns the individual name of the current test. Omits all ancestor groups. String get currentTestIndividualName { return Invoker.current!.liveTest.individualName; diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index f67806ff7..d0a3544c8 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -10,6 +10,8 @@ import 'package:patrol/src/native/contracts/contracts.dart'; import 'package:patrol/src/native/contracts/patrol_app_service_server.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:shelf/shelf_io.dart' as shelf_io; +// ignore: implementation_imports +import 'package:test_api/src/backend/live_test.dart'; const _port = 8082; const _idleTimeout = Duration(hours: 2); @@ -55,6 +57,13 @@ class PatrolAppService extends PatrolAppServiceServer { /// bundled Dart test file. final DartGroupEntry topLevelDartTestGroup; + /// The number of all setUpAll callbacks. + /// + /// setUpAlls, unlike setUps, aren't executed in the [LiveTest] context. + /// Because of this, we can't depend on the [LiveTest]'s name, so we identify + /// them by indexes instead. + int setUpAllCount = 0; + /// A completer that completes with the name of the Dart test file that was /// requested to execute by the native side. final _testExecutionRequested = Completer(); @@ -136,6 +145,8 @@ class PatrolAppService extends PatrolAppServiceServer { return ListDartTestsResponse(group: topLevelDartTestGroup); } + + @override Future runDartTest(RunDartTestRequest request) async { assert(_testExecutionCompleted.isCompleted == false); From f6a570aa1a5304f290bc678c3e0d6aad4e6c5ec4 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 5 Oct 2023 13:29:14 +0200 Subject: [PATCH 06/77] schema.dart: add `ListDartLifecycleCallbacksResponse` --- schema.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schema.dart b/schema.dart index d117a8707..1b1aa198e 100644 --- a/schema.dart +++ b/schema.dart @@ -19,6 +19,11 @@ class ListDartTestsResponse { late DartGroupEntry group; } +class ListDartLifecycleCallbacksResponse { + late List setUpAlls; + late List tearDownAlls; +} + enum RunDartTestResponseResult { success, skipped, @@ -36,6 +41,7 @@ class RunDartTestResponse { abstract class PatrolAppService { ListDartTestsResponse listDartTests(); + ListDartLifecycleCallbacksResponse listDartLifecycleCallbacks(); RunDartTestResponse runDartTest(RunDartTestRequest request); } From 8c5e8d584e81a225e55a82089d1097e7b0fe7388 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 5 Oct 2023 14:08:57 +0200 Subject: [PATCH 07/77] patrol_gen: bump dependency versions --- packages/patrol_gen/pubspec.lock | 2 +- packages/patrol_gen/pubspec.yaml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/patrol_gen/pubspec.lock b/packages/patrol_gen/pubspec.lock index a7f24b26e..b7dda4027 100644 --- a/packages/patrol_gen/pubspec.lock +++ b/packages/patrol_gen/pubspec.lock @@ -170,7 +170,7 @@ packages: source: hosted version: "1.3.2" watcher: - dependency: transitive + dependency: "direct overridden" description: name: watcher sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8" diff --git a/packages/patrol_gen/pubspec.yaml b/packages/patrol_gen/pubspec.yaml index 363972ed6..ee58d0387 100644 --- a/packages/patrol_gen/pubspec.yaml +++ b/packages/patrol_gen/pubspec.yaml @@ -11,10 +11,13 @@ environment: sdk: '>=3.0.0 <4.0.0' dependencies: - analyzer: ^6.1.0 + analyzer: ^6.2.0 dart_style: ^2.3.2 meta: ^1.7.0 path: ^1.8.2 - + dev_dependencies: leancode_lint: ^3.0.0 + +dependency_overrides: + watcher: ^1.1.0 From e77de99705dfd1a256dee0549527f4d6680ee3f1 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 5 Oct 2023 14:11:14 +0200 Subject: [PATCH 08/77] regenerate contracts --- .../pl/leancode/patrol/contracts/Contracts.kt | 6 +++++ .../contracts/PatrolAppServiceClient.kt | 5 ++++ .../Classes/AutomatorServer/Contracts.swift | 5 ++++ .../PatrolAppServiceClient.swift | 4 ++++ .../lib/src/native/contracts/contracts.dart | 24 +++++++++++++++++++ .../lib/src/native/contracts/contracts.g.dart | 17 +++++++++++++ .../contracts/patrol_app_service_server.dart | 6 +++++ 7 files changed, 67 insertions(+) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt index cf0ac5b2d..a6131c20a 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt @@ -52,6 +52,12 @@ class Contracts { val group: DartGroupEntry ) + @Serializable + data class ListDartLifecycleCallbacksResponse ( + val setUpAlls: List, + val tearDownAlls: List + ) + @Serializable data class RunDartTestRequest ( val name: String diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt index 5f39bd12e..60441ab67 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt @@ -20,6 +20,11 @@ class PatrolAppServiceClient(address: String, port: Int, private val timeout: Lo return json.decodeFromString(response) } + fun listDartLifecycleCallbacks(): Contracts.ListDartLifecycleCallbacksResponse { + val response = performRequest("listDartLifecycleCallbacks") + return json.decodeFromString(response) + } + fun runDartTest(request: Contracts.RunDartTestRequest): Contracts.RunDartTestResponse { val response = performRequest("runDartTest", json.encodeToString(request)) return json.decodeFromString(response) diff --git a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift index 7a7e4ac5c..7aaad7418 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift @@ -42,6 +42,11 @@ struct ListDartTestsResponse: Codable { var group: DartGroupEntry } +struct ListDartLifecycleCallbacksResponse: Codable { + var setUpAlls: [String] + var tearDownAlls: [String] +} + struct RunDartTestRequest: Codable { var name: String } diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift index f6da06cbe..6a3247923 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift @@ -20,6 +20,10 @@ class PatrolAppServiceClient { performRequest(requestName: "listDartTests", completion: completion) } + func listDartLifecycleCallbacks(completion: @escaping (Result) -> Void) { + performRequest(requestName: "listDartLifecycleCallbacks", completion: completion) + } + func runDartTest(request: RunDartTestRequest, completion: @escaping (Result) -> Void) { do { let body = try JSONEncoder().encode(request) diff --git a/packages/patrol/lib/src/native/contracts/contracts.dart b/packages/patrol/lib/src/native/contracts/contracts.dart index 27fb69c01..57dea2d71 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.dart @@ -92,6 +92,30 @@ class ListDartTestsResponse with EquatableMixin { ]; } +@JsonSerializable() +class ListDartLifecycleCallbacksResponse with EquatableMixin { + ListDartLifecycleCallbacksResponse({ + required this.setUpAlls, + required this.tearDownAlls, + }); + + factory ListDartLifecycleCallbacksResponse.fromJson( + Map json) => + _$ListDartLifecycleCallbacksResponseFromJson(json); + + final List setUpAlls; + final List tearDownAlls; + + Map toJson() => + _$ListDartLifecycleCallbacksResponseToJson(this); + + @override + List get props => [ + setUpAlls, + tearDownAlls, + ]; +} + @JsonSerializable() class RunDartTestRequest with EquatableMixin { RunDartTestRequest({ diff --git a/packages/patrol/lib/src/native/contracts/contracts.g.dart b/packages/patrol/lib/src/native/contracts/contracts.g.dart index 02a6c767f..29f1ca088 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.g.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.g.dart @@ -39,6 +39,23 @@ Map _$ListDartTestsResponseToJson( 'group': instance.group, }; +ListDartLifecycleCallbacksResponse _$ListDartLifecycleCallbacksResponseFromJson( + Map json) => + ListDartLifecycleCallbacksResponse( + setUpAlls: + (json['setUpAlls'] as List).map((e) => e as String).toList(), + tearDownAlls: (json['tearDownAlls'] as List) + .map((e) => e as String) + .toList(), + ); + +Map _$ListDartLifecycleCallbacksResponseToJson( + ListDartLifecycleCallbacksResponse instance) => + { + 'setUpAlls': instance.setUpAlls, + 'tearDownAlls': instance.tearDownAlls, + }; + RunDartTestRequest _$RunDartTestRequestFromJson(Map json) => RunDartTestRequest( name: json['name'] as String, diff --git a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart index dca57118d..7ddbf2a75 100644 --- a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart +++ b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart @@ -16,6 +16,11 @@ abstract class PatrolAppServiceServer { if ('listDartTests' == request.url.path) { final result = await listDartTests(); + final body = jsonEncode(result.toJson()); + return Response.ok(body); + } else if ('listDartLifecycleCallbacks' == request.url.path) { + final result = await listDartLifecycleCallbacks(); + final body = jsonEncode(result.toJson()); return Response.ok(body); } else if ('runDartTest' == request.url.path) { @@ -34,5 +39,6 @@ abstract class PatrolAppServiceServer { } Future listDartTests(); + Future listDartLifecycleCallbacks(); Future runDartTest(RunDartTestRequest request); } From 38fcdf3e18447a8d6bb0ea42e9c47e235e9bfbca Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 5 Oct 2023 22:53:05 +0200 Subject: [PATCH 09/77] PatrolAppService: start implementing tracking of setUpAlls --- .../internal/callbacks_all_test.dart | 16 +++++++------- packages/patrol/lib/src/common.dart | 4 +++- .../lib/src/native/patrol_app_service.dart | 21 ++++++++++++++----- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart index 0995a81fe..6140d5b89 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -10,14 +10,16 @@ String get currentTest => Invoker.current!.fullCurrentTestName(); void _print(String text) => print('PATROL_DEBUG: $text'); void main() { - patrolSetUpAll(() async { - await Future.delayed(Duration(seconds: 1)); - _print('setting up all before $currentTest'); + group('parent', () { + patrolSetUpAll(() async { + await Future.delayed(Duration(seconds: 1)); + _print('setting up all before $currentTest'); + }); + + patrolTest('testA', nativeAutomation: true, _body); + patrolTest('testB', nativeAutomation: true, _body); + patrolTest('testC', nativeAutomation: true, _body); }); - - patrolTest('testA', nativeAutomation: true, _body); - patrolTest('testB', nativeAutomation: true, _body); - patrolTest('testC', nativeAutomation: true, _body); } Future _body(PatrolTester $) async { diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 7c81a0f2c..c47158306 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -53,7 +53,9 @@ void patrolSetUpAll(Future Function() body) { setUpAll(() async { final currentTest = global_state.currentTestFullName; - final parentGroups = global_state.currentGroupFullName; + final parentGroup = global_state.currentGroupFullName; + + final setUpAllName = 'setUpAll $parentGroup'; final patrolAppService = PatrolBinding.instance.patrolAppService; final currentSetUpAllIndex = patrolAppService.setUpAllCount += 1; diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index d0a3544c8..1dd766b03 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -45,7 +45,7 @@ Future runAppService(PatrolAppService service) async { ); } -/// Implements a stateful gRPC service for querying and executing Dart tests. +/// Implements a stateful HTTP service for querying and executing Dart tests. /// /// This is an internal class and you don't want to use it. It's public so that /// the generated code can access it. @@ -61,8 +61,8 @@ class PatrolAppService extends PatrolAppServiceServer { /// /// setUpAlls, unlike setUps, aren't executed in the [LiveTest] context. /// Because of this, we can't depend on the [LiveTest]'s name, so we identify - /// them by indexes instead. - int setUpAllCount = 0; + /// them by the parent group ID instead. + final List setUpAlls = []; /// A completer that completes with the name of the Dart test file that was /// requested to execute by the native side. @@ -82,6 +82,13 @@ class PatrolAppService extends PatrolAppServiceServer { return _testExecutionCompleted.future; } + /// Adds a setUpAll callback to the list of all setUpAll callbacks. + void addSetUpAll(String group) { + // add an index at the end of setUpAlls + // parent testA 1 + // parent testA 2 + } + /// Marks [dartFileName] as completed with the given [passed] status. /// /// If an exception was thrown during the test, [details] should contain the @@ -145,8 +152,6 @@ class PatrolAppService extends PatrolAppServiceServer { return ListDartTestsResponse(group: topLevelDartTestGroup); } - - @override Future runDartTest(RunDartTestRequest request) async { assert(_testExecutionCompleted.isCompleted == false); @@ -163,4 +168,10 @@ class PatrolAppService extends PatrolAppServiceServer { details: testExecutionResult.details, ); } + + @override + Future listDartLifecycleCallbacks() { + // TODO: implement listDartLifecycleCallbacks + throw UnimplementedError(); + } } From cd0dac61c328b26a5b819c70e7cd963ddbdae8a0 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 9 Oct 2023 17:18:31 +0200 Subject: [PATCH 10/77] wip --- .../internal/callbacks_all_test.dart | 7 +++++- packages/patrol/lib/src/common.dart | 20 +++++++---------- .../lib/src/native/patrol_app_service.dart | 22 ++++++++++++++++++- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart index 6140d5b89..ac5e6f6b7 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -13,7 +13,12 @@ void main() { group('parent', () { patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - _print('setting up all before $currentTest'); + _print('setting up all (1) before $currentTest'); + }); + + patrolSetUpAll(() async { + await Future.delayed(Duration(seconds: 1)); + _print('setting up all (2) before $currentTest'); }); patrolTest('testA', nativeAutomation: true, _body); diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index c47158306..e6d02ff46 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -51,21 +51,17 @@ void patrolTearDown(Future Function() body) { /// A modification of [setUpAll] that works with Patrol's native automation. void patrolSetUpAll(Future Function() body) { setUpAll(() async { - final currentTest = global_state.currentTestFullName; - - final parentGroup = global_state.currentGroupFullName; - - final setUpAllName = 'setUpAll $parentGroup'; - final patrolAppService = PatrolBinding.instance.patrolAppService; - final currentSetUpAllIndex = patrolAppService.setUpAllCount += 1; - final requestedToExecute = await PatrolBinding.instance.patrolAppService - .waitForExecutionRequest(currentTest); + final parentGroupsName = global_state.currentGroupFullName; + patrolAppService.addSetUpAll(parentGroupsName); - if (requestedToExecute) { - await body(); - } + // final requestedToExecute = await PatrolBinding.instance.patrolAppService + // .waitForExecutionRequest(currentSetUpAllIndex); + + // if (requestedToExecute) { + // await body(); + // } }); } diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index 1dd766b03..68a62f2f4 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -57,7 +57,7 @@ class PatrolAppService extends PatrolAppServiceServer { /// bundled Dart test file. final DartGroupEntry topLevelDartTestGroup; - /// The number of all setUpAll callbacks. + /// The names of all setUpAll callbacks. /// /// setUpAlls, unlike setUps, aren't executed in the [LiveTest] context. /// Because of this, we can't depend on the [LiveTest]'s name, so we identify @@ -84,6 +84,24 @@ class PatrolAppService extends PatrolAppServiceServer { /// Adds a setUpAll callback to the list of all setUpAll callbacks. void addSetUpAll(String group) { + // Not very optimal, but good enough for now. + + setUpAlls.add('$group + ${group.hashCode}'); + + // for (final setUpAll in setUpAlls) { + // final parts = setUpAll.split(' '); + // final groupName = parts.sublist(0, parts.length - 1).join(' '); + + // if (groupName == group) { + + // } + + // final index = parts.last; + // if (setUpAll == group) { + // return; + // } + // } + // add an index at the end of setUpAlls // parent testA 1 // parent testA 2 @@ -149,6 +167,8 @@ class PatrolAppService extends PatrolAppServiceServer { @override Future listDartTests() async { print('PatrolAppService.listDartTests() called'); + print('PATROL_DEBUG: setUpAlls: $setUpAlls'); + return ListDartTestsResponse(group: topLevelDartTestGroup); } From 6045fa7a211772b2b58b29e4bd5f7700ff53a73c Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 9 Oct 2023 20:02:40 +0200 Subject: [PATCH 11/77] remember setUpAll callbacks --- .../patrol/example/test/callbacks_test.dart | 103 +++++++++++++++--- 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/packages/patrol/example/test/callbacks_test.dart b/packages/patrol/example/test/callbacks_test.dart index fb7f325d8..ffbbcbf33 100644 --- a/packages/patrol/example/test/callbacks_test.dart +++ b/packages/patrol/example/test/callbacks_test.dart @@ -3,32 +3,77 @@ import 'package:patrol/src/extensions.dart'; // ignore: depend_on_referenced_packages import 'package:test_api/src/backend/invoker.dart'; +// Input: const String requestedTest = 'testA'; +// Global state: + String get currentTest => Invoker.current!.fullCurrentTestName(); +/// A list holding IDs of all setUpAll callbacks. +/// +/// This is basically the list of groupNames + an index appended. +List setUpAlls = []; + +/// Adds a setUpAll callback to the list of all setUpAll callbacks. +/// +/// Returns the name under which this setUpAll callback was added. +String addSetUpAll(String group) { + // Not optimal, but good enough for now. + + // Go over all groups, checking if the group is already in the list. + var groupIndex = 0; + for (final setUpAll in setUpAlls) { + final parts = setUpAll.split(' '); + final groupName = parts.sublist(0, parts.length - 1).join(' '); + + print('Checking if $groupName is in the list'); + + if (groupName == group) { + groupIndex++; + } + } + + final name = '$group $groupIndex'; + + setUpAlls.add(name); + return name; +} + +void printSetUpAlls() { + print('setUpAlls:'); + for (final setUpAll in setUpAlls) { + print(' $setUpAll'); + } +} + void main() { - group('alpha', () { - setUpAll(() { - final groupName = Invoker.current!.liveTest.groups.last.name; - final individualName = Invoker.current!.liveTest.individualName; - print( - 'setUpAll: parentGroupName=$groupName, individualName=$individualName', - ); - }); + group('A', () { + customSetUpAll(() async {}); - setUp(() { - final groupName = Invoker.current!.liveTest.groups.last.name; - final individualName = Invoker.current!.liveTest.individualName; - print( - 'setUpAll: parentGroupName=$groupName, individualName=$individualName', - ); + customSetUpAll(() async {}); + + group('B', () { + customSetUpAll(() async {}); + customSetUpAll(() async {}); + customSetUpAll(() async {}); + customSetUpAll(() async {}); + + group('C', () { + customSetUpAll(() async {}); + customSetUpAll(() async {}); + }); }); patrolTest('testA', _body); patrolTest('testB', _body); patrolTest('testC', _body); }); + + tearDownAll(() { + print('Reporting status!'); + printSetUpAlls(); + }); } Future _body() async => print(Invoker.current!.fullCurrentTestName()); @@ -43,3 +88,33 @@ void patrolTest(String name, Future Function() body) { } }); } + +/// A modification of [setUpAll] that works with Patrol's native automation. +/// +/// Its main purpose is to keep track of calls made to setUpAll. +/// +/// groupA +/// - setUpAll +/// - setUpAll +/// - groupB +/// - setUpAll +void customSetUpAll(Future Function() body) { + setUpAll(() async { + final parentGroupsName = Invoker.current!.liveTest.groups.last.name; + + final name = addSetUpAll(parentGroupsName); + + // if (!isInTestDiscoveryPhase) { + // return; + // } + + // if (patrolAppService.wasSetUpAllExecuted()) {} + + // final requestedToExecute = await PatrolBinding.instance.patrolAppService + // .waitForExecutionRequest(currentSetUpAllIndex); + + // if (requestedToExecute) { + // await body(); + // } + }); +} From 768396099be704defa74fdc4f93791ac22a0a38f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 9 Oct 2023 23:41:00 +0200 Subject: [PATCH 12/77] add missing comments --- packages/patrol/lib/src/global_state.dart | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/patrol/lib/src/global_state.dart b/packages/patrol/lib/src/global_state.dart index 4abf8d264..5a8c01df2 100644 --- a/packages/patrol/lib/src/global_state.dart +++ b/packages/patrol/lib/src/global_state.dart @@ -24,11 +24,16 @@ String get currentTestFullName { return nameCandidate; } +/// Returns the name of the current group. +/// +/// Includes all ancestor groups. String get currentGroupFullName { return Invoker.current!.liveTest.groups.last.name; } -/// Returns the individual name of the current test. Omits all ancestor groups. +/// Returns the individual name of the current test. +/// +/// Omits all ancestor groups. String get currentTestIndividualName { return Invoker.current!.liveTest.individualName; } From 9edac164d622ba1690c3e26c4bec80a40ca1a505 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 10 Oct 2023 23:15:07 +0200 Subject: [PATCH 13/77] WIP: forward isInitialRun from instrumentation app to app under test --- .../pl/leancode/patrol/PatrolJUnitRunner.java | 14 +++++--- .../kotlin/pl/leancode/patrol/PatrolPlugin.kt | 32 +++++++++++++++++-- packages/patrol/lib/src/common.dart | 8 +++++ packages/patrol/lib/src/global_state.dart | 7 ++++ packages/patrol_cli/lib/src/test_bundler.dart | 5 +++ 5 files changed, 60 insertions(+), 6 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 03c969a7e..6485ff84c 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -10,14 +10,12 @@ import android.os.Bundle; import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.runner.AndroidJUnitRunner; - import pl.leancode.patrol.contracts.Contracts; import pl.leancode.patrol.contracts.PatrolAppServiceClientException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.ExecutionException; import static pl.leancode.patrol.contracts.Contracts.DartGroupEntry; import static pl.leancode.patrol.contracts.Contracts.RunDartTestResponse; @@ -30,6 +28,13 @@ public class PatrolJUnitRunner extends AndroidJUnitRunner { public PatrolAppServiceClient patrolAppServiceClient; + /** + *

+ * Available only after onCreate() has been run. + *

+ */ + protected boolean isInitialRun; + @Override protected boolean shouldWaitForActivitiesToComplete() { return false; @@ -40,10 +45,10 @@ public void onCreate(Bundle arguments) { super.onCreate(arguments); // This is only true when the ATO requests a list of tests from the app during the initial run. - boolean isInitialRun = Boolean.parseBoolean(arguments.getString("listTestsForOrchestrator")); + this.isInitialRun = Boolean.parseBoolean(arguments.getString("listTestsForOrchestrator")); Logger.INSTANCE.i("--------------------------------"); - Logger.INSTANCE.i("PatrolJUnitRunner.onCreate() " + (isInitialRun ? "(initial run)" : "")); + Logger.INSTANCE.i("PatrolJUnitRunner.onCreate() " + (this.isInitialRun ? "(initial run)" : "")); } /** @@ -69,6 +74,7 @@ public void setUp(Class activityClass) { // Currently, the only synchronization point we're interested in is when the app under test returns the list of tests. Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); Intent intent = new Intent(Intent.ACTION_MAIN); + intent.putExtra("isInitialRun", isInitialRun); intent.setClassName(instrumentation.getTargetContext(), activityClass.getCanonicalName()); intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); instrumentation.getTargetContext().startActivity(intent); diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolPlugin.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolPlugin.kt index 998420419..c930fd0ec 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolPlugin.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolPlugin.kt @@ -1,24 +1,52 @@ package pl.leancode.patrol import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.embedding.engine.plugins.activity.ActivityAware +import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel.MethodCallHandler import io.flutter.plugin.common.MethodChannel.Result -class PatrolPlugin : FlutterPlugin, MethodCallHandler { +class PatrolPlugin : FlutterPlugin, MethodCallHandler, ActivityAware { private lateinit var channel: MethodChannel + private var isInitialRun: Boolean? = null + override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { channel = MethodChannel(flutterPluginBinding.binaryMessenger, "pl.leancode.patrol/main") channel.setMethodCallHandler(this) } override fun onMethodCall(call: MethodCall, result: Result) { - result.notImplemented() + when (call.method) { + "isInitialRun" -> result.success(isInitialRun) + else -> result.notImplemented() + } } override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) { channel.setMethodCallHandler(null) } + + override fun onAttachedToActivity(binding: ActivityPluginBinding) { + val intent = binding.activity.intent + if (!intent.hasExtra("isInitialRun")) { + throw IllegalStateException("PatrolPlugin must be initialized with intent having isInitialRun boolean") + } + + isInitialRun = intent.getBooleanExtra("isInitialRun", false) + } + + override fun onDetachedFromActivityForConfigChanges() { + // Do nothing + } + + override fun onReattachedToActivityForConfigChanges(binding: ActivityPluginBinding) { + // Do nothing + } + + override fun onDetachedFromActivity() { + // Do nothing + } } diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index e6d02ff46..21d100ce0 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -141,6 +141,13 @@ void patrolTest( tags: tags, (widgetTester) async { if (!constants.hotRestartEnabled) { + if (await global_state.isInitialRun) { + // Fall through tests during the initial run that discovers tests. + // + // This is required to be able to find all setUpAll callbacks. + return; + } + // If Patrol's native automation feature is enabled, then this test will // be executed only if the native side requested it to be executed. // Otherwise, it returns early. @@ -153,6 +160,7 @@ void patrolTest( return; } } + if (!kIsWeb && io.Platform.isIOS) { widgetTester.binding.platformDispatcher.onSemanticsEnabledChanged = () { // This callback is empty on purpose. It's a workaround for tests diff --git a/packages/patrol/lib/src/global_state.dart b/packages/patrol/lib/src/global_state.dart index 5a8c01df2..81b922c53 100644 --- a/packages/patrol/lib/src/global_state.dart +++ b/packages/patrol/lib/src/global_state.dart @@ -1,4 +1,5 @@ // ignore: implementation_imports +import 'package:flutter/services.dart'; import 'package:test_api/src/backend/invoker.dart'; /// Maximum test case length for ATO, after transformations. @@ -42,3 +43,9 @@ String get currentTestIndividualName { bool get isCurrentTestPassing { return Invoker.current!.liveTest.state.result.isPassing; } + +const _channel = MethodChannel('pl.leancode.patrol/main'); + +Future get isInitialRun async { + return await _channel.invokeMethod('isInitialRun') as bool; +} diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 4b2c489d9..36906cb0d 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -92,6 +92,11 @@ Future main() async { ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // END: GENERATED TEST GROUPS + // An additional callback to discover setUpAlls. + tearDownAll(() { + print('PATROL_DEBUG: calling tearDownAll to print setUpAlls'); + }); + final dartTestGroup = await testExplorationCompleter.future; final appService = PatrolAppService(topLevelDartTestGroup: dartTestGroup); binding.patrolAppService = appService; From 7ed92352c1c9ea88c400893f44cafae1f1ac3610 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 10 Oct 2023 23:50:24 +0200 Subject: [PATCH 14/77] delete `example/test/callbacks_test.dart` (moved to bartekpacia/test_api_callbacks) --- .../patrol/example/test/callbacks_test.dart | 120 ------------------ 1 file changed, 120 deletions(-) delete mode 100644 packages/patrol/example/test/callbacks_test.dart diff --git a/packages/patrol/example/test/callbacks_test.dart b/packages/patrol/example/test/callbacks_test.dart deleted file mode 100644 index ffbbcbf33..000000000 --- a/packages/patrol/example/test/callbacks_test.dart +++ /dev/null @@ -1,120 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:patrol/src/extensions.dart'; -// ignore: depend_on_referenced_packages -import 'package:test_api/src/backend/invoker.dart'; - -// Input: -const String requestedTest = 'testA'; - -// Global state: - -String get currentTest => Invoker.current!.fullCurrentTestName(); - -/// A list holding IDs of all setUpAll callbacks. -/// -/// This is basically the list of groupNames + an index appended. -List setUpAlls = []; - -/// Adds a setUpAll callback to the list of all setUpAll callbacks. -/// -/// Returns the name under which this setUpAll callback was added. -String addSetUpAll(String group) { - // Not optimal, but good enough for now. - - // Go over all groups, checking if the group is already in the list. - var groupIndex = 0; - for (final setUpAll in setUpAlls) { - final parts = setUpAll.split(' '); - final groupName = parts.sublist(0, parts.length - 1).join(' '); - - print('Checking if $groupName is in the list'); - - if (groupName == group) { - groupIndex++; - } - } - - final name = '$group $groupIndex'; - - setUpAlls.add(name); - return name; -} - -void printSetUpAlls() { - print('setUpAlls:'); - for (final setUpAll in setUpAlls) { - print(' $setUpAll'); - } -} - -void main() { - group('A', () { - customSetUpAll(() async {}); - - customSetUpAll(() async {}); - - group('B', () { - customSetUpAll(() async {}); - customSetUpAll(() async {}); - customSetUpAll(() async {}); - customSetUpAll(() async {}); - - group('C', () { - customSetUpAll(() async {}); - customSetUpAll(() async {}); - }); - }); - - patrolTest('testA', _body); - patrolTest('testB', _body); - patrolTest('testC', _body); - }); - - tearDownAll(() { - print('Reporting status!'); - printSetUpAlls(); - }); -} - -Future _body() async => print(Invoker.current!.fullCurrentTestName()); - -void patrolTest(String name, Future Function() body) { - test(name, () async { - final currentTest = Invoker.current!.fullCurrentTestName(); - - if (currentTest == requestedTest) { - print('Requested test $currentTest, will execute it'); - await body(); - } - }); -} - -/// A modification of [setUpAll] that works with Patrol's native automation. -/// -/// Its main purpose is to keep track of calls made to setUpAll. -/// -/// groupA -/// - setUpAll -/// - setUpAll -/// - groupB -/// - setUpAll -void customSetUpAll(Future Function() body) { - setUpAll(() async { - final parentGroupsName = Invoker.current!.liveTest.groups.last.name; - - final name = addSetUpAll(parentGroupsName); - - // if (!isInTestDiscoveryPhase) { - // return; - // } - - // if (patrolAppService.wasSetUpAllExecuted()) {} - - // final requestedToExecute = await PatrolBinding.instance.patrolAppService - // .waitForExecutionRequest(currentSetUpAllIndex); - - // if (requestedToExecute) { - // await body(); - // } - }); -} From 56dca3b592496961bf925e2a945ebadff0ed41da Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 10 Oct 2023 23:55:57 +0200 Subject: [PATCH 15/77] implement PatrolAppService.addSetUpAll and remembering setUpAlls Problem: not all tests are "falling through" during the initial run --- .../pl/leancode/patrol/PatrolJUnitRunner.java | 2 +- packages/patrol/lib/src/common.dart | 33 +++++++++--- packages/patrol/lib/src/global_state.dart | 4 +- .../lib/src/native/patrol_app_service.dart | 54 ++++++++++--------- packages/patrol_cli/lib/src/test_bundler.dart | 5 +- 5 files changed, 62 insertions(+), 36 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 6485ff84c..313435362 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -132,7 +132,7 @@ public Object[] listDartTests() { * Throws AssertionError if the test fails. */ public RunDartTestResponse runDartTest(String name) { - final String TAG = "PatrolJUnitRunner.runDartTest(" + name + "): "; + final String TAG = "PatrolJUnitRunner.runDartTest(\"" + name + "\"): "; try { Logger.INSTANCE.i(TAG + "Requested execution"); diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 21d100ce0..65c62e0f8 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -49,19 +49,35 @@ void patrolTearDown(Future Function() body) { } /// A modification of [setUpAll] that works with Patrol's native automation. +/// +/// It keeps track of calls made to setUpAll. void patrolSetUpAll(Future Function() body) { setUpAll(() async { final patrolAppService = PatrolBinding.instance.patrolAppService; - final parentGroupsName = global_state.currentGroupFullName; - patrolAppService.addSetUpAll(parentGroupsName); + final setUpAllName = patrolAppService.addSetUpAll(parentGroupsName); + + if (await global_state.isInitialRun) { + // Skip calling body if we're in test discovery phase + print( + "PATROL_DEBUG: Skipping setUpAll '$setUpAllName' because it's test discovery phase", + ); + return; + } + + // TODO: Skip calling body if it this setUpAll was already executed + + final requestedTest = await patrolAppService.testExecutionRequested; + + // Skip calling if parentGroupName is not a substring of requestedTestName + if (!requestedTest.startsWith(parentGroupsName)) { + // This is not exhaustive. + return; + } - // final requestedToExecute = await PatrolBinding.instance.patrolAppService - // .waitForExecutionRequest(currentSetUpAllIndex); + await body(); - // if (requestedToExecute) { - // await body(); - // } + // TODO: Mark this setUpAll as executed }); } @@ -142,6 +158,9 @@ void patrolTest( (widgetTester) async { if (!constants.hotRestartEnabled) { if (await global_state.isInitialRun) { + print( + 'PATROL_DEBUG: falling through test "${global_state.currentTestFullName}"', + ); // Fall through tests during the initial run that discovers tests. // // This is required to be able to find all setUpAll callbacks. diff --git a/packages/patrol/lib/src/global_state.dart b/packages/patrol/lib/src/global_state.dart index 81b922c53..ca4e33987 100644 --- a/packages/patrol/lib/src/global_state.dart +++ b/packages/patrol/lib/src/global_state.dart @@ -1,5 +1,5 @@ -// ignore: implementation_imports import 'package:flutter/services.dart'; +// ignore: implementation_imports import 'package:test_api/src/backend/invoker.dart'; /// Maximum test case length for ATO, after transformations. @@ -46,6 +46,8 @@ bool get isCurrentTestPassing { const _channel = MethodChannel('pl.leancode.patrol/main'); +/// Returns whether this is the first run of the app under test during which +/// test discovery happens. Future get isInitialRun async { return await _channel.invokeMethod('isInitialRun') as bool; } diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index 68a62f2f4..4fbaac2ff 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -62,6 +62,9 @@ class PatrolAppService extends PatrolAppServiceServer { /// setUpAlls, unlike setUps, aren't executed in the [LiveTest] context. /// Because of this, we can't depend on the [LiveTest]'s name, so we identify /// them by the parent group ID instead. + /// + /// This is a list of groups containing setUpAllCallbacks with an index + /// appended. final List setUpAlls = []; /// A completer that completes with the name of the Dart test file that was @@ -83,28 +86,26 @@ class PatrolAppService extends PatrolAppServiceServer { } /// Adds a setUpAll callback to the list of all setUpAll callbacks. - void addSetUpAll(String group) { - // Not very optimal, but good enough for now. - - setUpAlls.add('$group + ${group.hashCode}'); - - // for (final setUpAll in setUpAlls) { - // final parts = setUpAll.split(' '); - // final groupName = parts.sublist(0, parts.length - 1).join(' '); - - // if (groupName == group) { - - // } + /// + /// Returns the name under which this setUpAll callback was added. + String addSetUpAll(String group) { + // Not optimal, but good enough for now. + + // Go over all groups, checking if the group is already in the list. + var groupIndex = 0; + for (final setUpAll in setUpAlls) { + final parts = setUpAll.split(' '); + final groupName = parts.sublist(0, parts.length - 1).join(' '); + + if (groupName == group) { + groupIndex++; + } + } - // final index = parts.last; - // if (setUpAll == group) { - // return; - // } - // } + final name = '$group $groupIndex'; - // add an index at the end of setUpAlls - // parent testA 1 - // parent testA 2 + setUpAlls.add(name); + return name; } /// Marks [dartFileName] as completed with the given [passed] status. @@ -167,7 +168,6 @@ class PatrolAppService extends PatrolAppServiceServer { @override Future listDartTests() async { print('PatrolAppService.listDartTests() called'); - print('PATROL_DEBUG: setUpAlls: $setUpAlls'); return ListDartTestsResponse(group: topLevelDartTestGroup); } @@ -175,7 +175,6 @@ class PatrolAppService extends PatrolAppServiceServer { @override Future runDartTest(RunDartTestRequest request) async { assert(_testExecutionCompleted.isCompleted == false); - // patrolTest() always calls this method. print('PatrolAppService.runDartTest(${request.name}) called'); _testExecutionRequested.complete(request.name); @@ -190,8 +189,13 @@ class PatrolAppService extends PatrolAppServiceServer { } @override - Future listDartLifecycleCallbacks() { - // TODO: implement listDartLifecycleCallbacks - throw UnimplementedError(); + Future + listDartLifecycleCallbacks() async { + print('PatrolAppService.listDartLifecycleCallbacks() called'); + + return ListDartLifecycleCallbacksResponse( + setUpAlls: setUpAlls, + tearDownAlls: [], + ); } } diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 36906cb0d..d14716c45 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -72,6 +72,7 @@ Future main() async { await nativeAutomator.initialize(); final binding = PatrolBinding.ensureInitialized(); final testExplorationCompleter = Completer(); + late final PatrolAppService appService; // A special test to expore the hierarchy of groups and tests. This is a hack // around https://github.com/dart-lang/test/issues/1998. @@ -94,11 +95,11 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // An additional callback to discover setUpAlls. tearDownAll(() { - print('PATROL_DEBUG: calling tearDownAll to print setUpAlls'); + print('PATROL_DEBUG: tearDownAll(): setUpAlls: \${appService.setUpAlls}'); }); final dartTestGroup = await testExplorationCompleter.future; - final appService = PatrolAppService(topLevelDartTestGroup: dartTestGroup); + appService = PatrolAppService(topLevelDartTestGroup: dartTestGroup); binding.patrolAppService = appService; await runAppService(appService); From 2a7ae16abc74db5ee4ac92a9782b46fa5b2d7a33 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 11 Oct 2023 23:21:30 +0200 Subject: [PATCH 16/77] PatrolJUnitRunner: fix log tag in waitForPatrolAppService() --- .../src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 313435362..90c07838e 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -100,7 +100,7 @@ public PatrolAppServiceClient createAppServiceClient() { *

*/ public void waitForPatrolAppService() { - final String TAG = "PatrolJUnitRunner.setUp(): "; + final String TAG = "PatrolJUnitRunner.waitForPatrolAppService(): "; Logger.INSTANCE.i(TAG + "Waiting for PatrolAppService to report its readiness..."); PatrolServer.Companion.getAppReady().block(); From 1f2eaa4d572b52eac839be9062c4f6e08b85ebea Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 11 Oct 2023 23:34:16 +0200 Subject: [PATCH 17/77] PatrolBinding.setUp(): fix patrol_test_explorer not being ignored --- packages/patrol/lib/src/binding.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 4e2a54601..a64eb18d1 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -53,12 +53,12 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { return; } - _currentDartTest = global_state.currentTestFullName; - if (_currentDartTest == 'patrol_test_explorer') { + if (global_state.currentTestIndividualName == 'patrol_test_explorer') { // Ignore the fake test. return; } + _currentDartTest = global_state.currentTestFullName; logger('setUp(): called with current Dart test = "$_currentDartTest"'); }); From 69a89f3f2b47ff145fc9900d1a0d89ac20b5f1ac Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 11 Oct 2023 23:35:03 +0200 Subject: [PATCH 18/77] PatrolBinding.tearDown(): don't block on testExecutionRequested during the initial run --- packages/patrol/lib/src/binding.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index a64eb18d1..79afc7e7d 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -68,6 +68,12 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { return; } + if (await global_state.isInitialRun) { + // If this is the initial run, then no test has been requested to + // execute. Return to avoid blocking on testExecutionRequested below. + return; + } + final testName = global_state.currentTestIndividualName; final isTestExplorer = testName == 'patrol_test_explorer'; if (isTestExplorer) { From cf0e8d4194e35aa30c0d05cf7a2d56f8065d43b1 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 12 Oct 2023 01:37:22 +0200 Subject: [PATCH 19/77] move hotRestartEnabled from constants.dart to global_state.dart, delete constants.dart --- packages/patrol/lib/src/binding.dart | 11 +++++------ packages/patrol/lib/src/common.dart | 3 +-- packages/patrol/lib/src/constants.dart | 5 ----- packages/patrol/lib/src/global_state.dart | 5 +++++ packages/patrol_cli/lib/src/test_bundler.dart | 7 ++++++- 5 files changed, 17 insertions(+), 14 deletions(-) delete mode 100644 packages/patrol/lib/src/constants.dart diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 79afc7e7d..8710b8e9f 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -6,7 +6,6 @@ import 'package:integration_test/integration_test.dart'; import 'package:patrol/patrol.dart'; import 'package:patrol/src/global_state.dart' as global_state; -import 'constants.dart' as constants; const _success = 'success'; @@ -41,14 +40,14 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { reportTestException = (details, testDescription) { final currentDartTest = _currentDartTest; if (currentDartTest != null) { - assert(!constants.hotRestartEnabled); + assert(!global_state.hotRestartEnabled); _testResults[currentDartTest] = Failure(testDescription, '$details'); } oldTestExceptionReporter(details, testDescription); }; setUp(() { - if (constants.hotRestartEnabled) { + if (global_state.hotRestartEnabled) { // Sending results ends the test, which we don't want for Hot Restart return; } @@ -63,7 +62,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { }); tearDown(() async { - if (constants.hotRestartEnabled) { + if (global_state.hotRestartEnabled) { // Sending results ends the test, which we don't want for Hot Restart return; } @@ -176,12 +175,12 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { @override void attachRootWidget(Widget rootWidget) { assert( - (_currentDartTest != null) != (constants.hotRestartEnabled), + (_currentDartTest != null) != (global_state.hotRestartEnabled), '_currentDartTest can be null if and only if Hot Restart is enabled', ); const testLabelEnabled = bool.fromEnvironment('PATROL_TEST_LABEL_ENABLED'); - if (!testLabelEnabled || constants.hotRestartEnabled) { + if (!testLabelEnabled || global_state.hotRestartEnabled) { super.attachRootWidget(RepaintBoundary(child: rootWidget)); } else { super.attachRootWidget( diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 65c62e0f8..a86a23e80 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -14,7 +14,6 @@ import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/test.dart'; -import 'constants.dart' as constants; import 'custom_finders/patrol_integration_tester.dart'; /// Signature for callback to [patrolTest]. @@ -156,7 +155,7 @@ void patrolTest( variant: variant, tags: tags, (widgetTester) async { - if (!constants.hotRestartEnabled) { + if (!global_state.hotRestartEnabled) { if (await global_state.isInitialRun) { print( 'PATROL_DEBUG: falling through test "${global_state.currentTestFullName}"', diff --git a/packages/patrol/lib/src/constants.dart b/packages/patrol/lib/src/constants.dart deleted file mode 100644 index 01440e6ce..000000000 --- a/packages/patrol/lib/src/constants.dart +++ /dev/null @@ -1,5 +0,0 @@ -import 'package:meta/meta.dart'; - -/// Whether Hot Restart is enabled. -@internal -const bool hotRestartEnabled = bool.fromEnvironment('PATROL_HOT_RESTART'); diff --git a/packages/patrol/lib/src/global_state.dart b/packages/patrol/lib/src/global_state.dart index ca4e33987..a4479ce1a 100644 --- a/packages/patrol/lib/src/global_state.dart +++ b/packages/patrol/lib/src/global_state.dart @@ -1,4 +1,5 @@ import 'package:flutter/services.dart'; +import 'package:meta/meta.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/invoker.dart'; @@ -7,6 +8,10 @@ import 'package:test_api/src/backend/invoker.dart'; /// See https://github.com/leancodepl/patrol/issues/1725 const maxTestLength = 190; +/// Whether Hot Restart is enabled. +@internal +const bool hotRestartEnabled = bool.fromEnvironment('PATROL_HOT_RESTART'); + /// This file wraps the [Invoker] API, which is internal to package:test. We /// want to minimize the usage of internal APIs to a minimum. diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index d14716c45..67a7f6461 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -72,6 +72,7 @@ Future main() async { await nativeAutomator.initialize(); final binding = PatrolBinding.ensureInitialized(); final testExplorationCompleter = Completer(); + final callbacksExplorationCompleter = Completer(); late final PatrolAppService appService; // A special test to expore the hierarchy of groups and tests. This is a hack @@ -94,8 +95,10 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // END: GENERATED TEST GROUPS // An additional callback to discover setUpAlls. - tearDownAll(() { + tearDownAll(() async { print('PATROL_DEBUG: tearDownAll(): setUpAlls: \${appService.setUpAlls}'); + + callbacksExplorationCompleter.complete(); }); final dartTestGroup = await testExplorationCompleter.future; @@ -108,6 +111,8 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // about Dart tests. await nativeAutomator.markPatrolAppServiceReady(); + await callbacksExplorationCompleter.future; + await appService.testExecutionCompleted; } '''; From 3ba9ae16862dcbe66412d321e49351564fe072c9 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 12 Oct 2023 10:49:20 +0200 Subject: [PATCH 20/77] add patrolDebug() log helper to differentiat between app process runs --- .../integration_test/internal/callbacks_all_test.dart | 10 +++++----- packages/patrol/lib/src/binding.dart | 4 +++- packages/patrol/lib/src/common.dart | 11 ++++++----- packages/patrol/lib/src/logs.dart | 8 ++++++++ packages/patrol_cli/lib/src/test_bundler.dart | 3 ++- 5 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 packages/patrol/lib/src/logs.dart diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart index ac5e6f6b7..832eab325 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -1,5 +1,7 @@ import 'package:flutter/material.dart'; import 'package:patrol/src/extensions.dart'; +// ignore: implementation_imports +import 'package:patrol/src/logs.dart'; // ignore: depend_on_referenced_packages import 'package:test_api/src/backend/invoker.dart'; @@ -7,18 +9,16 @@ import '../common.dart'; String get currentTest => Invoker.current!.fullCurrentTestName(); -void _print(String text) => print('PATROL_DEBUG: $text'); - void main() { group('parent', () { patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - _print('setting up all (1) before $currentTest'); + patrolDebug('setting up all (1) before $currentTest'); }); patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - _print('setting up all (2) before $currentTest'); + patrolDebug('setting up all (2) before $currentTest'); }); patrolTest('testA', nativeAutomation: true, _body); @@ -29,7 +29,7 @@ void main() { Future _body(PatrolTester $) async { final testName = Invoker.current!.fullCurrentTestName(); - _print('test body: name=$testName'); + patrolDebug('test body: name=$testName'); await createApp($); diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 8710b8e9f..85a56f7ca 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -5,7 +5,7 @@ import 'package:integration_test/common.dart'; import 'package:integration_test/integration_test.dart'; import 'package:patrol/patrol.dart'; import 'package:patrol/src/global_state.dart' as global_state; - +import 'package:patrol/src/logs.dart'; const _success = 'success'; @@ -86,7 +86,9 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { logger('tearDown(): test "$dartTestName": "$result"'); }); + patrolDebug('WE ARE HERE'); final requestedDartTest = await patrolAppService.testExecutionRequested; + patrolDebug('BUT ARE WE HERE?'); if (requestedDartTest == _currentDartTest) { logger( 'tearDown(): finished test "$_currentDartTest". Will report its status back to the native side', diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index a86a23e80..392edcaea 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -6,6 +6,7 @@ import 'package:integration_test/integration_test.dart'; import 'package:meta/meta.dart'; import 'package:patrol/src/binding.dart'; import 'package:patrol/src/global_state.dart' as global_state; +import 'package:patrol/src/logs.dart'; import 'package:patrol/src/native/contracts/contracts.dart'; import 'package:patrol/src/native/native.dart'; import 'package:patrol_finders/patrol_finders.dart' as finders; @@ -58,14 +59,15 @@ void patrolSetUpAll(Future Function() body) { if (await global_state.isInitialRun) { // Skip calling body if we're in test discovery phase - print( - "PATROL_DEBUG: Skipping setUpAll '$setUpAllName' because it's test discovery phase", + patrolDebug( + "skipping setUpAll '$setUpAllName' because it's test discovery phase", ); return; } // TODO: Skip calling body if it this setUpAll was already executed + patrolDebug('OH SO ARE WE BLOCKED HERE???'); final requestedTest = await patrolAppService.testExecutionRequested; // Skip calling if parentGroupName is not a substring of requestedTestName @@ -157,9 +159,8 @@ void patrolTest( (widgetTester) async { if (!global_state.hotRestartEnabled) { if (await global_state.isInitialRun) { - print( - 'PATROL_DEBUG: falling through test "${global_state.currentTestFullName}"', - ); + await Future.delayed(const Duration(seconds: 2)); + patrolDebug('fallthrough test "${global_state.currentTestFullName}"'); // Fall through tests during the initial run that discovers tests. // // This is required to be able to find all setUpAll callbacks. diff --git a/packages/patrol/lib/src/logs.dart b/packages/patrol/lib/src/logs.dart new file mode 100644 index 000000000..7cdbbfad2 --- /dev/null +++ b/packages/patrol/lib/src/logs.dart @@ -0,0 +1,8 @@ +import 'package:meta/meta.dart'; + +final runKey = Object().hashCode; + +@internal +void patrolDebug(String message) { + print('PATROL_DEBUG($runKey): $message'); +} diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 67a7f6461..a7ce6a622 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -28,6 +28,7 @@ import 'dart:async'; import 'package:flutter_test/flutter_test.dart'; import 'package:patrol/patrol.dart'; +import 'package:patrol/src/logs.dart'; import 'package:patrol/src/native/contracts/contracts.dart'; import 'package:test_api/src/backend/invoker.dart'; @@ -96,7 +97,7 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // An additional callback to discover setUpAlls. tearDownAll(() async { - print('PATROL_DEBUG: tearDownAll(): setUpAlls: \${appService.setUpAlls}'); + patrolDebug('tearDownAll(): setUpAlls: \${appService.setUpAlls}'); callbacksExplorationCompleter.complete(); }); From 6ce85db98d775de0459dd11f4b0de93de765718f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 12 Oct 2023 11:25:01 +0200 Subject: [PATCH 21/77] update dev_docs/GUIDE.md --- dev_docs/GUIDE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev_docs/GUIDE.md b/dev_docs/GUIDE.md index b86f0d99b..f021c39e6 100644 --- a/dev_docs/GUIDE.md +++ b/dev_docs/GUIDE.md @@ -15,3 +15,13 @@ Search for `TestRunner: started`. ### Find out when a test ends Search for `TestRunner: finished`. + +### I made some changes to test bundling code that result in a deadlock + +This is normal when editing this code. It's a mine field of shared global +mutable state and things happening in parallel. + +Look for `await`s in custom functions provided by Patrol (e.g. `patrolTest()` +and `patrolSetUpAll()`) and global lifecycle callbacks registered by the +generated Dart test bundle or PatrolBinding (e.g. `tearDown()`s). Use `print`s +amply to pinpint where the code is stuck. From 6f0125be7c8be7dbe3c4402a0349c2222d375283 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 12 Oct 2023 12:48:26 +0200 Subject: [PATCH 22/77] PatrolBinding: remove unneeded debug logs in tearDown() --- packages/patrol/lib/src/binding.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 85a56f7ca..586ef30f1 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -5,7 +5,6 @@ import 'package:integration_test/common.dart'; import 'package:integration_test/integration_test.dart'; import 'package:patrol/patrol.dart'; import 'package:patrol/src/global_state.dart' as global_state; -import 'package:patrol/src/logs.dart'; const _success = 'success'; @@ -86,9 +85,8 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { logger('tearDown(): test "$dartTestName": "$result"'); }); - patrolDebug('WE ARE HERE'); final requestedDartTest = await patrolAppService.testExecutionRequested; - patrolDebug('BUT ARE WE HERE?'); + if (requestedDartTest == _currentDartTest) { logger( 'tearDown(): finished test "$_currentDartTest". Will report its status back to the native side', From 0f61514484d26e84c7ac8d4c05c5c522b1dca2b1 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 12 Oct 2023 13:07:24 +0200 Subject: [PATCH 23/77] WIP: start adding PatrolJUnitRunner.listLifecycleCallbacks() --- .../leancode/patrol/PatrolAppServiceClient.kt | 6 ++++++ .../pl/leancode/patrol/PatrolJUnitRunner.java | 19 +++++++++++++++++++ .../patrol/example/MainActivityTest.java | 5 ++++- .../internal/callbacks_all_test.dart | 6 +++--- packages/patrol/lib/src/common.dart | 2 +- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt index 7bdfa7bad..d95159d79 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt @@ -38,4 +38,10 @@ class PatrolAppServiceClient { Logger.i("PatrolAppServiceClient.runDartTest($name)") return client.runDartTest(Contracts.RunDartTestRequest(name)) } + + @Throws(PatrolAppServiceClientException::class) + fun runDartTest(name: String): Contracts.RunDartTestResponse { + Logger.i("PatrolAppServiceClient.runDartTest($name)") + return client.runDartTest(Contracts.RunDartTestRequest(name)) + } } diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 90c07838e..427954867 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -127,6 +127,25 @@ public Object[] listDartTests() { } } + public Object[] listLifecycleCallbacks() { + final String TAG = "PatrolJUnitRunner.listLifecycleCallbacks(): "; + + try { + final DartGroupEntry dartTestGroup = patrolAppServiceClient.listDartTests(); + List dartTestCases = ContractsExtensionsKt.listTestsFlat(dartTestGroup, ""); + List dartTestCaseNamesList = new ArrayList<>(); + for (DartGroupEntry dartTestCase : dartTestCases) { + dartTestCaseNamesList.add(dartTestCase.getName()); + } + Object[] dartTestCaseNames = dartTestCaseNamesList.toArray(); + Logger.INSTANCE.i(TAG + "Got Dart tests: " + Arrays.toString(dartTestCaseNames)); + return dartTestCaseNames; + } catch (PatrolAppServiceClientException e) { + Logger.INSTANCE.e(TAG + "Failed to list Dart tests: ", e); + throw new RuntimeException(e); + } + } + /** * Requests execution of a Dart test and waits for it to finish. * Throws AssertionError if the test fails. diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index 675e46464..143c4a775 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -14,7 +14,10 @@ public static Object[] testCases() { PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); instrumentation.setUp(MainActivity.class); instrumentation.waitForPatrolAppService(); - return instrumentation.listDartTests(); + Object[] dartTests = instrumentation.listDartTests(); + Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); + + return dartTests; } public MainActivityTest(String dartTestName) { diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart index 832eab325..8d3d0b7b5 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -13,12 +13,12 @@ void main() { group('parent', () { patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - patrolDebug('setting up all (1) before $currentTest'); + patrolDebug('ran setUpAll (1) before "$currentTest"'); }); patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - patrolDebug('setting up all (2) before $currentTest'); + patrolDebug('ran setUpAll (2) before "$currentTest"'); }); patrolTest('testA', nativeAutomation: true, _body); @@ -29,7 +29,7 @@ void main() { Future _body(PatrolTester $) async { final testName = Invoker.current!.fullCurrentTestName(); - patrolDebug('test body: name=$testName'); + patrolDebug('ran body of test "$testName"'); await createApp($); diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 392edcaea..4acb2fc15 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -67,7 +67,7 @@ void patrolSetUpAll(Future Function() body) { // TODO: Skip calling body if it this setUpAll was already executed - patrolDebug('OH SO ARE WE BLOCKED HERE???'); + // patrolDebug('OH SO ARE WE BLOCKED HERE???'); final requestedTest = await patrolAppService.testExecutionRequested; // Skip calling if parentGroupName is not a substring of requestedTestName From f1ccf538c18b314acdbc7fd0c37be4ed227069db Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 13 Oct 2023 22:41:42 +0200 Subject: [PATCH 24/77] PatrolAppServiceClient: fix duplicated override of runDartTest() --- .../kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt index d95159d79..311761f20 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt @@ -40,8 +40,9 @@ class PatrolAppServiceClient { } @Throws(PatrolAppServiceClientException::class) - fun runDartTest(name: String): Contracts.RunDartTestResponse { - Logger.i("PatrolAppServiceClient.runDartTest($name)") - return client.runDartTest(Contracts.RunDartTestRequest(name)) + fun listDartLifecycleCallbacks(name: String): Contracts.ListDartLifecycleCallbacksResponse { + Logger.i("PatrolAppServiceClient.listDartLifecycleCallbacks($name)") + val result = client.listDartLifecycleCallbacks() + return result } } From f9bd800acfdc3f971a3f6226bca6cb6385e8e250 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 13 Oct 2023 23:26:38 +0200 Subject: [PATCH 25/77] PatrolBinding: accept PatrolAppService in constructor --- packages/patrol/lib/src/binding.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 586ef30f1..df9bf1811 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -33,7 +33,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// Creates a new [PatrolBinding]. /// /// You most likely don't want to call it yourself. - PatrolBinding() { + PatrolBinding(this.patrolAppService) { logger('created'); final oldTestExceptionReporter = reportTestException; reportTestException = (details, testDescription) { @@ -113,9 +113,9 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// if necessary. /// /// This method is idempotent. - factory PatrolBinding.ensureInitialized() { + factory PatrolBinding.ensureInitialized(PatrolAppService patrolAppService) { if (_instance == null) { - PatrolBinding(); + PatrolBinding(patrolAppService); } return _instance!; } @@ -128,7 +128,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// /// It's only for test reporting purposes and should not be used for anything /// else. - late PatrolAppService patrolAppService; + final PatrolAppService patrolAppService; /// The singleton instance of this object. /// From 1b19ddd90b18752749af9cf565e09474f5f76b9c Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 13 Oct 2023 23:32:12 +0200 Subject: [PATCH 26/77] PatrolJUnitRunner: implement basic listDartLifecycleCallbacks() --- .../leancode/patrol/PatrolAppServiceClient.kt | 4 ++-- .../pl/leancode/patrol/PatrolJUnitRunner.java | 17 +++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt index 311761f20..6be99a815 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt @@ -40,8 +40,8 @@ class PatrolAppServiceClient { } @Throws(PatrolAppServiceClientException::class) - fun listDartLifecycleCallbacks(name: String): Contracts.ListDartLifecycleCallbacksResponse { - Logger.i("PatrolAppServiceClient.listDartLifecycleCallbacks($name)") + fun listDartLifecycleCallbacks(): Contracts.ListDartLifecycleCallbacksResponse { + Logger.i("PatrolAppServiceClient.listDartLifecycleCallbacks()") val result = client.listDartLifecycleCallbacks() return result } diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 427954867..c33d25e25 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -11,6 +11,7 @@ import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.runner.AndroidJUnitRunner; import pl.leancode.patrol.contracts.Contracts; +import pl.leancode.patrol.contracts.Contracts.ListDartLifecycleCallbacksResponse; import pl.leancode.patrol.contracts.PatrolAppServiceClientException; import java.util.ArrayList; @@ -131,17 +132,13 @@ public Object[] listLifecycleCallbacks() { final String TAG = "PatrolJUnitRunner.listLifecycleCallbacks(): "; try { - final DartGroupEntry dartTestGroup = patrolAppServiceClient.listDartTests(); - List dartTestCases = ContractsExtensionsKt.listTestsFlat(dartTestGroup, ""); - List dartTestCaseNamesList = new ArrayList<>(); - for (DartGroupEntry dartTestCase : dartTestCases) { - dartTestCaseNamesList.add(dartTestCase.getName()); - } - Object[] dartTestCaseNames = dartTestCaseNamesList.toArray(); - Logger.INSTANCE.i(TAG + "Got Dart tests: " + Arrays.toString(dartTestCaseNames)); - return dartTestCaseNames; + final ListDartLifecycleCallbacksResponse response = patrolAppServiceClient.listDartLifecycleCallbacks(); + final List setUpAlls = response.getSetUpAlls(); + Logger.INSTANCE.i(TAG + "Got Dart lifecycle callbacks: " + setUpAlls); + + return setUpAlls.toArray(); } catch (PatrolAppServiceClientException e) { - Logger.INSTANCE.e(TAG + "Failed to list Dart tests: ", e); + Logger.INSTANCE.e(TAG + "Failed to list Dart lifecycle callbacks: ", e); throw new RuntimeException(e); } } From 712258714f28ebee276b1bde25ad7a69a9018db6 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 13 Oct 2023 23:32:41 +0200 Subject: [PATCH 27/77] PatrolAppService: remove topLevelGroup from constructor --- .../lib/src/native/patrol_app_service.dart | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index 4fbaac2ff..4c9452b63 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -49,13 +49,25 @@ Future runAppService(PatrolAppService service) async { /// /// This is an internal class and you don't want to use it. It's public so that /// the generated code can access it. +/// +/// PatrolAppService lifecycle during initial run: +/// +/// 1. Initial +/// 2. Has Dart tests +/// 3. Has Dart lifecycle callbacks +/// +/// PatrolAppService lifecycle during test run: +/// +/// 1. Initial +/// 2. Has Dart tests +/// 3. Has Dart test to execute class PatrolAppService extends PatrolAppServiceServer { /// Creates a new [PatrolAppService]. - PatrolAppService({required this.topLevelDartTestGroup}); + PatrolAppService(); /// The ambient test group that wraps all the other groups and tests in the /// bundled Dart test file. - final DartGroupEntry topLevelDartTestGroup; + DartGroupEntry? topLevelDartTestGroup; /// The names of all setUpAll callbacks. /// @@ -65,7 +77,7 @@ class PatrolAppService extends PatrolAppServiceServer { /// /// This is a list of groups containing setUpAllCallbacks with an index /// appended. - final List setUpAlls = []; + List setUpAlls = []; /// A completer that completes with the name of the Dart test file that was /// requested to execute by the native side. @@ -169,6 +181,13 @@ class PatrolAppService extends PatrolAppServiceServer { Future listDartTests() async { print('PatrolAppService.listDartTests() called'); + final topLevelDartTestGroup = this.topLevelDartTestGroup; + if (topLevelDartTestGroup == null) { + throw StateError( + 'PatrolAppService.listDartTests(): tests not discovered yet', + ); + } + return ListDartTestsResponse(group: topLevelDartTestGroup); } From c89b10da0a1c8d223f957494c8f9e441cd625ea2 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 13 Oct 2023 23:33:27 +0200 Subject: [PATCH 28/77] test_bundler: adjust generated bundle to PatrolBinding and PatrolAppService changes Also wait for callbacksExplorationCompleter during the initial run --- packages/patrol_cli/lib/src/test_bundler.dart | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 001e03494..cf4393dfc 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -28,6 +28,7 @@ import 'dart:async'; import 'package:flutter_test/flutter_test.dart'; import 'package:patrol/patrol.dart'; +import 'package:patrol/src/global_state.dart' as global_state; import 'package:patrol/src/logs.dart'; import 'package:patrol/src/native/contracts/contracts.dart'; import 'package:test_api/src/backend/invoker.dart'; @@ -71,10 +72,14 @@ Future main() async { final nativeAutomator = NativeAutomator(config: NativeAutomatorConfig()); await nativeAutomator.initialize(); - final binding = PatrolBinding.ensureInitialized(); - final testExplorationCompleter = Completer(); + + final appService = PatrolAppService(); + await runAppService(appService); + + PatrolBinding.ensureInitialized(appService); + + final testsExplorationCompleter = Completer(); final callbacksExplorationCompleter = Completer(); - late final PatrolAppService appService; // A special test to expore the hierarchy of groups and tests. This is a hack // around https://github.com/dart-lang/test/issues/1998. @@ -86,7 +91,7 @@ Future main() async { // to group() below. final topLevelGroup = Invoker.current!.liveTest.groups.first; final dartTestGroup = createDartTestGroup(topLevelGroup); - testExplorationCompleter.complete(dartTestGroup); + testsExplorationCompleter.complete(dartTestGroup); print('patrol_test_explorer: obtained Dart-side test hierarchy:'); printGroupStructure(dartTestGroup); }); @@ -97,23 +102,23 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // An additional callback to discover setUpAlls. tearDownAll(() async { - patrolDebug('tearDownAll(): setUpAlls: \${appService.setUpAlls}'); - - callbacksExplorationCompleter.complete(); + if (await global_state.isInitialRun) { + patrolDebug('tearDownAll(): setUpAlls: \${appService.setUpAlls}'); + callbacksExplorationCompleter.complete(); + } }); - final dartTestGroup = await testExplorationCompleter.future; - appService = PatrolAppService(topLevelDartTestGroup: dartTestGroup); - binding.patrolAppService = appService; - await runAppService(appService); + appService.topLevelDartTestGroup = await testsExplorationCompleter.future; + + if (await global_state.isInitialRun) { + await callbacksExplorationCompleter.future; + } // Until now, the native test runner was waiting for us, the Dart side, to // come alive. Now that we did, let's tell it that we're ready to be asked // about Dart tests. await nativeAutomator.markPatrolAppServiceReady(); - await callbacksExplorationCompleter.future; - await appService.testExecutionCompleted; } '''; From 67c37b7d135399c2b19da0b83a98de9a96623ba4 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Sat, 14 Oct 2023 00:00:03 +0200 Subject: [PATCH 29/77] native Android side: only call listDartLifecycleCallbacks() during initial run --- .../main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java | 4 ++++ .../java/pl/leancode/patrol/example/MainActivityTest.java | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index c33d25e25..b91bb2c57 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -36,6 +36,10 @@ public class PatrolJUnitRunner extends AndroidJUnitRunner { */ protected boolean isInitialRun; + public boolean isInitialRun() { + return isInitialRun; + } + @Override protected boolean shouldWaitForActivitiesToComplete() { return false; diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index 143c4a775..7f6aa56a0 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -15,7 +15,10 @@ public static Object[] testCases() { instrumentation.setUp(MainActivity.class); instrumentation.waitForPatrolAppService(); Object[] dartTests = instrumentation.listDartTests(); - Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); + + if (instrumentation.isInitialRun()) { + Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); + } return dartTests; } From f558713c61e437860e4df261f13ffc8ce33bcd7f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 14:00:11 +0200 Subject: [PATCH 30/77] PatrolJUnitRunner: add comment about where patrol.txt is created --- .../main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt index d4855fa2f..a8f6276c3 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt @@ -44,7 +44,8 @@ class PatrolStoragePermissionListener : RunListener() { override fun testStarted(description: Description) { Logger.d("testStarted with description: $description") - + + // This file is created at /sdcard/googletest/test_outputfiles val os = testStorage.openOutputFile("patrol.txt", true) os.write("testStarted with description: $description\n".toByteArray()) From 6114b30814a7e5fcd82327a44365547e092d75ad Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 14:39:29 +0200 Subject: [PATCH 31/77] PatrolJUnitRunner: save lifecycle callbacks to JSON during initial run --- .../pl/leancode/patrol/PatrolJUnitRunner.java | 29 +++++++++++++++++++ .../patrol/example/MainActivityTest.java | 1 + 2 files changed, 30 insertions(+) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index b91bb2c57..224095229 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -5,17 +5,23 @@ package pl.leancode.patrol; +import android.annotation.SuppressLint; import android.app.Instrumentation; import android.content.Intent; import android.os.Bundle; import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.runner.AndroidJUnitRunner; +import androidx.test.services.storage.TestStorage; +import com.google.gson.Gson; import pl.leancode.patrol.contracts.Contracts; import pl.leancode.patrol.contracts.Contracts.ListDartLifecycleCallbacksResponse; import pl.leancode.patrol.contracts.PatrolAppServiceClientException; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import static pl.leancode.patrol.contracts.Contracts.DartGroupEntry; @@ -26,6 +32,7 @@ * A customized AndroidJUnitRunner that enables Patrol on Android. *

*/ +@SuppressLint("UnsafeOptInUsageError") public class PatrolJUnitRunner extends AndroidJUnitRunner { public PatrolAppServiceClient patrolAppServiceClient; @@ -36,6 +43,8 @@ public class PatrolJUnitRunner extends AndroidJUnitRunner { */ protected boolean isInitialRun; + private TestStorage testStorage; + public boolean isInitialRun() { return isInitialRun; } @@ -49,6 +58,8 @@ protected boolean shouldWaitForActivitiesToComplete() { public void onCreate(Bundle arguments) { super.onCreate(arguments); + this.testStorage = new TestStorage(); + // This is only true when the ATO requests a list of tests from the app during the initial run. this.isInitialRun = Boolean.parseBoolean(arguments.getString("listTestsForOrchestrator")); @@ -147,6 +158,24 @@ public Object[] listLifecycleCallbacks() { } } + @SuppressLint("UnsafeOptInUsageError") + public void saveLifecycleCallbacks(Object[] callbacks) { + HashMap callbackMap = new HashMap<>(); + for (Object callback : callbacks) { + callbackMap.put((String) callback, false); + } + + Gson gson = new Gson(); + try { + String json = gson.toJson(callbackMap); + OutputStream outputStream = testStorage.openOutputFile("patrol_callbacks.json"); + outputStream.write(json.getBytes()); + outputStream.write("\n".getBytes()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Requests execution of a Dart test and waits for it to finish. * Throws AssertionError if the test fails. diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index 7f6aa56a0..60beeeb35 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -18,6 +18,7 @@ public static Object[] testCases() { if (instrumentation.isInitialRun()) { Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); + instrumentation.saveLifecycleCallbacks(lifecycleCallbacks); } return dartTests; From d877bca69952e851463e8a2d0d5a076ba015e95d Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 15:06:17 +0200 Subject: [PATCH 32/77] PatrolBinding: add dependency on NativeAutomator --- packages/patrol/lib/src/binding.dart | 13 ++++++++++--- packages/patrol/lib/src/common.dart | 15 ++++++--------- packages/patrol_cli/lib/src/test_bundler.dart | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index df9bf1811..863641a65 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -33,7 +33,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// Creates a new [PatrolBinding]. /// /// You most likely don't want to call it yourself. - PatrolBinding(this.patrolAppService) { + PatrolBinding(this.patrolAppService, this.nativeAutomator) { logger('created'); final oldTestExceptionReporter = reportTestException; reportTestException = (details, testDescription) { @@ -113,9 +113,12 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// if necessary. /// /// This method is idempotent. - factory PatrolBinding.ensureInitialized(PatrolAppService patrolAppService) { + factory PatrolBinding.ensureInitialized( + PatrolAppService patrolAppService, + NativeAutomator nativeAutomator, + ) { if (_instance == null) { - PatrolBinding(patrolAppService); + PatrolBinding(patrolAppService, nativeAutomator); } return _instance!; } @@ -130,6 +133,10 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// else. final PatrolAppService patrolAppService; + /// The [NativeAutomator] used by this binding to interact with the native + /// side. + final NativeAutomator? nativeAutomator; + /// The singleton instance of this object. /// /// Provides access to the features exposed by this class. The binding must be diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 4acb2fc15..126a52777 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -76,13 +76,14 @@ void patrolSetUpAll(Future Function() body) { return; } - await body(); - // TODO: Mark this setUpAll as executed + final nativeAutomator = PatrolBinding.instance.nativeAutomator; + + await body(); }); } -/// Like [testWidgets], but with support for Patrol custom finders. +/// Like [testWidgets], but with support for Patrol custom fiders. /// /// To customize the Patrol-specific configuration, set [config]. /// @@ -118,8 +119,6 @@ void patrolTest( LiveTestWidgetsFlutterBindingFramePolicy framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers, }) { - NativeAutomator? automator; - if (!nativeAutomation) { debugPrint(''' ╔════════════════════════════════════════════════════════════════════════════════════╗ @@ -134,8 +133,6 @@ void patrolTest( if (nativeAutomation) { switch (bindingType) { case BindingType.patrol: - automator = NativeAutomator(config: nativeAutomatorConfig); - // PatrolBinding is initialized in the generated test bundle file. PatrolBinding.instance.framePolicy = framePolicy; break; @@ -188,11 +185,11 @@ void patrolTest( // See https://github.com/leancodepl/patrol/issues/1474 }; } - await automator?.configure(); + await PatrolBinding.instance.nativeAutomator?.configure(); final patrolTester = PatrolIntegrationTester( tester: widgetTester, - nativeAutomator: automator, + nativeAutomator: PatrolBinding.instance.nativeAutomator, config: config, ); await callback(patrolTester); diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index cf4393dfc..5b39978af 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -76,7 +76,7 @@ Future main() async { final appService = PatrolAppService(); await runAppService(appService); - PatrolBinding.ensureInitialized(appService); + PatrolBinding.ensureInitialized(appService, nativeAutomator); final testsExplorationCompleter = Completer(); final callbacksExplorationCompleter = Completer(); From 716c13fcda7a55e73de82ba086c6bbf75b3ed5df Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 15:16:34 +0200 Subject: [PATCH 33/77] schema: add NativeAutomator.markLifecycleCallbackExecuted() --- .../pl/leancode/patrol/contracts/Contracts.kt | 4 ++++ .../patrol/contracts/NativeAutomatorServer.kt | 6 ++++++ .../Classes/AutomatorServer/Contracts.swift | 4 ++++ .../NativeAutomatorServer.swift | 12 +++++++++++ .../lib/src/native/contracts/contracts.dart | 21 +++++++++++++++++++ .../lib/src/native/contracts/contracts.g.dart | 12 +++++++++++ .../contracts/native_automator_client.dart | 9 ++++++++ schema.dart | 6 ++++++ 8 files changed, 74 insertions(+) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt index a89914b70..47d991497 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt @@ -252,4 +252,8 @@ class Contracts { val locationAccuracy: SetLocationAccuracyRequestLocationAccuracy ) + data class MarkLifecycleCallbackExecutedRequest ( + val name: String + ) + } diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/NativeAutomatorServer.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/NativeAutomatorServer.kt index 617aee7cc..defb029aa 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/NativeAutomatorServer.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/NativeAutomatorServer.kt @@ -47,6 +47,7 @@ abstract class NativeAutomatorServer { abstract fun setLocationAccuracy(request: Contracts.SetLocationAccuracyRequest) abstract fun debug() abstract fun markPatrolAppServiceReady() + abstract fun markLifecycleCallbackExecuted(request: Contracts.MarkLifecycleCallbackExecutedRequest) val router = routes( "initialize" bind POST to { @@ -200,6 +201,11 @@ abstract class NativeAutomatorServer { "markPatrolAppServiceReady" bind POST to { markPatrolAppServiceReady() Response(OK) + }, + "markLifecycleCallbackExecuted" bind POST to { + val body = json.fromJson(it.bodyString(), Contracts.MarkLifecycleCallbackExecutedRequest::class.java) + markLifecycleCallbackExecuted(body) + Response(OK) } ) diff --git a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift index 7aaad7418..0a59fe2d6 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift @@ -170,3 +170,7 @@ struct SetLocationAccuracyRequest: Codable { var locationAccuracy: SetLocationAccuracyRequestLocationAccuracy } +struct MarkLifecycleCallbackExecutedRequest: Codable { + var name: String +} + diff --git a/packages/patrol/ios/Classes/AutomatorServer/NativeAutomatorServer.swift b/packages/patrol/ios/Classes/AutomatorServer/NativeAutomatorServer.swift index 6343bc8a8..4611b7385 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/NativeAutomatorServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/NativeAutomatorServer.swift @@ -42,6 +42,7 @@ protocol NativeAutomatorServer { func setLocationAccuracy(request: SetLocationAccuracyRequest) throws func debug() throws func markPatrolAppServiceReady() throws + func markLifecycleCallbackExecuted(request: MarkLifecycleCallbackExecutedRequest) throws } extension NativeAutomatorServer { @@ -233,6 +234,12 @@ extension NativeAutomatorServer { try markPatrolAppServiceReady() return HTTPResponse(.ok) } + + private func markLifecycleCallbackExecutedHandler(request: HTTPRequest) throws -> HTTPResponse { + let requestArg = try JSONDecoder().decode(MarkLifecycleCallbackExecutedRequest.self, from: request.body) + try markLifecycleCallbackExecuted(request: requestArg) + return HTTPResponse(.ok) + } } extension NativeAutomatorServer { @@ -407,6 +414,11 @@ extension NativeAutomatorServer { request: request, handler: markPatrolAppServiceReadyHandler) } + server.route(.POST, "markLifecycleCallbackExecuted") { + request in handleRequest( + request: request, + handler: markLifecycleCallbackExecutedHandler) + } } } diff --git a/packages/patrol/lib/src/native/contracts/contracts.dart b/packages/patrol/lib/src/native/contracts/contracts.dart index 57dea2d71..a8294f816 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.dart @@ -627,3 +627,24 @@ class SetLocationAccuracyRequest with EquatableMixin { locationAccuracy, ]; } + +@JsonSerializable() +class MarkLifecycleCallbackExecutedRequest with EquatableMixin { + MarkLifecycleCallbackExecutedRequest({ + required this.name, + }); + + factory MarkLifecycleCallbackExecutedRequest.fromJson( + Map json) => + _$MarkLifecycleCallbackExecutedRequestFromJson(json); + + final String name; + + Map toJson() => + _$MarkLifecycleCallbackExecutedRequestToJson(this); + + @override + List get props => [ + name, + ]; +} diff --git a/packages/patrol/lib/src/native/contracts/contracts.g.dart b/packages/patrol/lib/src/native/contracts/contracts.g.dart index 29f1ca088..09c5ca740 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.g.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.g.dart @@ -391,3 +391,15 @@ const _$SetLocationAccuracyRequestLocationAccuracyEnumMap = { SetLocationAccuracyRequestLocationAccuracy.coarse: 'coarse', SetLocationAccuracyRequestLocationAccuracy.fine: 'fine', }; + +MarkLifecycleCallbackExecutedRequest + _$MarkLifecycleCallbackExecutedRequestFromJson(Map json) => + MarkLifecycleCallbackExecutedRequest( + name: json['name'] as String, + ); + +Map _$MarkLifecycleCallbackExecutedRequestToJson( + MarkLifecycleCallbackExecutedRequest instance) => + { + 'name': instance.name, + }; diff --git a/packages/patrol/lib/src/native/contracts/native_automator_client.dart b/packages/patrol/lib/src/native/contracts/native_automator_client.dart index ae1c10d6a..70a245d87 100644 --- a/packages/patrol/lib/src/native/contracts/native_automator_client.dart +++ b/packages/patrol/lib/src/native/contracts/native_automator_client.dart @@ -294,6 +294,15 @@ class NativeAutomatorClient { ); } + Future markLifecycleCallbackExecuted( + MarkLifecycleCallbackExecutedRequest request, + ) { + return _sendRequest( + 'markLifecycleCallbackExecuted', + request.toJson(), + ); + } + Future> _sendRequest( String requestName, [ Map? request, diff --git a/schema.dart b/schema.dart index 1b1aa198e..8aea3fcbd 100644 --- a/schema.dart +++ b/schema.dart @@ -171,6 +171,10 @@ class SetLocationAccuracyRequest { late SetLocationAccuracyRequestLocationAccuracy locationAccuracy; } +class MarkLifecycleCallbackExecutedRequest { + late String name; +} + abstract class NativeAutomator { void initialize(); void configure(ConfigureRequest request); @@ -221,4 +225,6 @@ abstract class NativeAutomator { // TODO(bartekpacia): Move this RPC into a new PatrolNativeTestService service because it doesn't fit here void markPatrolAppServiceReady(); + void markLifecycleCallbackExecuted( + MarkLifecycleCallbackExecutedRequest request); } From 37d012d4f8d287a135e30aa407ce4952d4367a50 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 17:43:56 +0200 Subject: [PATCH 34/77] implement NativeAutomator.markLifecycleCallbackExecuted --- .../pl/leancode/patrol/AutomatorServer.kt | 7 ++ .../pl/leancode/patrol/PatrolJUnitRunner.java | 65 ++++++++++++++++--- packages/patrol/lib/src/binding.dart | 2 +- packages/patrol/lib/src/common.dart | 3 +- .../lib/src/native/native_automator.dart | 15 ++++- 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/AutomatorServer.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/AutomatorServer.kt index 6ea616c19..49031b6d3 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/AutomatorServer.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/AutomatorServer.kt @@ -1,5 +1,6 @@ package pl.leancode.patrol +import androidx.test.platform.app.InstrumentationRegistry import pl.leancode.patrol.contracts.Contracts.ConfigureRequest import pl.leancode.patrol.contracts.Contracts.DarkModeRequest import pl.leancode.patrol.contracts.Contracts.EnterTextRequest @@ -9,6 +10,7 @@ import pl.leancode.patrol.contracts.Contracts.GetNotificationsRequest import pl.leancode.patrol.contracts.Contracts.GetNotificationsResponse import pl.leancode.patrol.contracts.Contracts.HandlePermissionRequest import pl.leancode.patrol.contracts.Contracts.HandlePermissionRequestCode +import pl.leancode.patrol.contracts.Contracts.MarkLifecycleCallbackExecutedRequest import pl.leancode.patrol.contracts.Contracts.OpenAppRequest import pl.leancode.patrol.contracts.Contracts.OpenQuickSettingsRequest import pl.leancode.patrol.contracts.Contracts.PermissionDialogVisibleRequest @@ -210,4 +212,9 @@ class AutomatorServer(private val automation: Automator) : NativeAutomatorServer override fun markPatrolAppServiceReady() { PatrolServer.appReady.open() } + + override fun markLifecycleCallbackExecuted(request: MarkLifecycleCallbackExecutedRequest) { + val instrumentation = InstrumentationRegistry.getInstrumentation() as PatrolJUnitRunner + instrumentation.markLifecycleCallbackExecuted(request.name) + } } diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 224095229..28039daca 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -7,22 +7,32 @@ import android.annotation.SuppressLint; import android.app.Instrumentation; +import android.content.ContentResolver; import android.content.Intent; +import android.net.Uri; import android.os.Bundle; import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.runner.AndroidJUnitRunner; import androidx.test.services.storage.TestStorage; +import androidx.test.services.storage.file.HostedFile; +import androidx.test.services.storage.internal.TestStorageUtil; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import pl.leancode.patrol.contracts.Contracts; import pl.leancode.patrol.contracts.Contracts.ListDartLifecycleCallbacksResponse; import pl.leancode.patrol.contracts.PatrolAppServiceClientException; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Scanner; import static pl.leancode.patrol.contracts.Contracts.DartGroupEntry; import static pl.leancode.patrol.contracts.Contracts.RunDartTestResponse; @@ -32,7 +42,7 @@ * A customized AndroidJUnitRunner that enables Patrol on Android. *

*/ -@SuppressLint("UnsafeOptInUsageError") +@SuppressLint({"UnsafeOptInUsageError", "RestrictedApi"}) public class PatrolJUnitRunner extends AndroidJUnitRunner { public PatrolAppServiceClient patrolAppServiceClient; @@ -43,7 +53,14 @@ public class PatrolJUnitRunner extends AndroidJUnitRunner { */ protected boolean isInitialRun; - private TestStorage testStorage; + private ContentResolver getContentResolver() { + return InstrumentationRegistry.getInstrumentation().getTargetContext().getContentResolver(); + } + + private Uri stateFileUri = HostedFile.buildUri( + HostedFile.FileHost.OUTPUT, + "patrol_callbacks.json" + ); public boolean isInitialRun() { return isInitialRun; @@ -58,8 +75,6 @@ protected boolean shouldWaitForActivitiesToComplete() { public void onCreate(Bundle arguments) { super.onCreate(arguments); - this.testStorage = new TestStorage(); - // This is only true when the ATO requests a list of tests from the app during the initial run. this.isInitialRun = Boolean.parseBoolean(arguments.getString("listTestsForOrchestrator")); @@ -158,24 +173,54 @@ public Object[] listLifecycleCallbacks() { } } - @SuppressLint("UnsafeOptInUsageError") public void saveLifecycleCallbacks(Object[] callbacks) { - HashMap callbackMap = new HashMap<>(); + Map callbackMap = new HashMap<>(); for (Object callback : callbacks) { callbackMap.put((String) callback, false); } - Gson gson = new Gson(); + writeStateFile(callbackMap); + } + + public void markLifecycleCallbackExecuted(String name) { + Logger.INSTANCE.i("PatrolJUnitRunnerMarking.markLifecycleCallbackExecuted(" + name + ")"); + Map state = readStateFile(); + state.put(name, true); + writeStateFile(state); + } + + private Map readStateFile() { + try { + InputStream inputStream = TestStorageUtil.getInputStream(stateFileUri, getContentResolver()); + String content = convertStreamToString(inputStream); + Gson gson = new Gson(); + Type typeOfHashMap = new TypeToken>() {}.getType(); + Map data = gson.fromJson(content, typeOfHashMap); + return data; + } catch (FileNotFoundException e) { + throw new RuntimeException("Failed to read state file", e); + } + } + + private void writeStateFile(Map data) { try { - String json = gson.toJson(callbackMap); - OutputStream outputStream = testStorage.openOutputFile("patrol_callbacks.json"); + OutputStream outputStream = TestStorageUtil.getOutputStream(stateFileUri, getContentResolver()); + Gson gson = new Gson(); + Type typeOfHashMap = new TypeToken>() { + }.getType(); + String json = gson.toJson(data, typeOfHashMap); outputStream.write(json.getBytes()); outputStream.write("\n".getBytes()); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to write state file", e); } } + static String convertStreamToString(InputStream inputStream) { + Scanner s = new Scanner(inputStream).useDelimiter("\\A"); + return s.hasNext() ? s.next() : ""; + } + /** * Requests execution of a Dart test and waits for it to finish. * Throws AssertionError if the test fails. diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 863641a65..63c22407d 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -135,7 +135,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { /// The [NativeAutomator] used by this binding to interact with the native /// side. - final NativeAutomator? nativeAutomator; + final NativeAutomator nativeAutomator; /// The singleton instance of this object. /// diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 126a52777..f734ebd56 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -78,6 +78,7 @@ void patrolSetUpAll(Future Function() body) { // TODO: Mark this setUpAll as executed final nativeAutomator = PatrolBinding.instance.nativeAutomator; + await nativeAutomator.markLifecycleCallbackExecuted(setUpAllName); await body(); }); @@ -185,7 +186,7 @@ void patrolTest( // See https://github.com/leancodepl/patrol/issues/1474 }; } - await PatrolBinding.instance.nativeAutomator?.configure(); + await PatrolBinding.instance.nativeAutomator.configure(); final patrolTester = PatrolIntegrationTester( tester: widgetTester, diff --git a/packages/patrol/lib/src/native/native_automator.dart b/packages/patrol/lib/src/native/native_automator.dart index cde6bfaf5..493890a85 100644 --- a/packages/patrol/lib/src/native/native_automator.dart +++ b/packages/patrol/lib/src/native/native_automator.dart @@ -769,7 +769,7 @@ class NativeAutomator { ); } - /// Tells the AndroidJUnitRunner that PatrolAppService is ready to answer + /// Tells the native test runner that PatrolAppService is ready to answer /// requests about the structure of Dart tests. @internal Future markPatrolAppServiceReady() async { @@ -778,4 +778,17 @@ class NativeAutomator { _client.markPatrolAppServiceReady, ); } + + /// Tells the native test runner that callback [name] has been executed. + /// + /// The native test runner persists that information. + @internal + Future markLifecycleCallbackExecuted(String name) async { + await _wrapRequest( + 'markLifecycleCallbackExecuted', + () => _client.markLifecycleCallbackExecuted( + MarkLifecycleCallbackExecutedRequest(name: name), + ), + ); + } } From 2cc9def248fee70691001a79e37213a3d38df450 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 18:12:23 +0200 Subject: [PATCH 35/77] implement clearing state on initial run (might be unneeded?) --- .../main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java | 5 +++++ .../java/pl/leancode/patrol/example/MainActivityTest.java | 3 +++ 2 files changed, 8 insertions(+) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 28039daca..e59ff4c2f 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -189,6 +189,11 @@ public void markLifecycleCallbackExecuted(String name) { writeStateFile(state); } + public void clearStateFile() { + Logger.INSTANCE.i("PatrolJUnitRunner.clearStateFile()"); + writeStateFile(new HashMap<>()); + } + private Map readStateFile() { try { InputStream inputStream = TestStorageUtil.getInputStream(stateFileUri, getContentResolver()); diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index 60beeeb35..5e952d939 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -17,6 +17,9 @@ public static Object[] testCases() { Object[] dartTests = instrumentation.listDartTests(); if (instrumentation.isInitialRun()) { + // Clear the list of lifecycle callbacks from the previous run. + instrumentation.clearStateFile(); + Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); instrumentation.saveLifecycleCallbacks(lifecycleCallbacks); } From e62d0a05c6f2921f99d699a707ff8669f3670242 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 18:17:54 +0200 Subject: [PATCH 36/77] schema: add PatrolAppService.setLifecycleCallbacksState --- .../pl/leancode/patrol/PatrolJUnitRunner.java | 14 +++++++++++-- .../pl/leancode/patrol/contracts/Contracts.kt | 4 ++++ .../contracts/PatrolAppServiceClient.kt | 4 ++++ .../patrol/example/MainActivityTest.java | 1 + .../Classes/AutomatorServer/Contracts.swift | 4 ++++ .../PatrolAppServiceClient.swift | 9 ++++++++ .../lib/src/native/contracts/contracts.dart | 21 +++++++++++++++++++ .../lib/src/native/contracts/contracts.g.dart | 12 +++++++++++ .../contracts/patrol_app_service_server.dart | 9 ++++++++ schema.dart | 5 +++++ 10 files changed, 81 insertions(+), 2 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index e59ff4c2f..61df8734f 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -211,8 +211,7 @@ private void writeStateFile(Map data) { try { OutputStream outputStream = TestStorageUtil.getOutputStream(stateFileUri, getContentResolver()); Gson gson = new Gson(); - Type typeOfHashMap = new TypeToken>() { - }.getType(); + Type typeOfHashMap = new TypeToken>() {}.getType(); String json = gson.toJson(data, typeOfHashMap); outputStream.write(json.getBytes()); outputStream.write("\n".getBytes()); @@ -226,6 +225,17 @@ static String convertStreamToString(InputStream inputStream) { return s.hasNext() ? s.next() : ""; } + /** + * Sets the state of lifecycle callbacks in the app. + * + * This is required because the app is launched in a new process for each test. + */ + public void setLifecycleCallbacksStateInApp() { + final String TAG = "PatrolJUnitRunner.setLifecycleCallbacksStateInApp(): "; + + + } + /** * Requests execution of a Dart test and waits for it to finish. * Throws AssertionError if the test fails. diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt index 47d991497..062c2d50f 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt @@ -61,6 +61,10 @@ class Contracts { } } + data class SetLifecycleCallbacksStateRequest ( + val state: Map + ) + data class ConfigureRequest ( val findTimeoutMillis: Long ) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt index ef6e3ef36..48af70593 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt @@ -24,6 +24,10 @@ class PatrolAppServiceClient(address: String, port: Int, private val timeout: Lo return json.fromJson(response, Contracts.ListDartLifecycleCallbacksResponse::class.java) } + fun setLifecycleCallbacksState(request: Contracts.SetLifecycleCallbacksStateRequest) { + return performRequest("setLifecycleCallbacksState", json.toJson(request)) + } + fun runDartTest(request: Contracts.RunDartTestRequest): Contracts.RunDartTestResponse { val response = performRequest("runDartTest", json.toJson(request)) return json.fromJson(response, Contracts.RunDartTestResponse::class.java) diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index 5e952d939..ac4f5bd27 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -36,6 +36,7 @@ public MainActivityTest(String dartTestName) { @Test public void runDartTest() { PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); + instrumentation.setLifecycleCallbacksState(); instrumentation.runDartTest(dartTestName); } } diff --git a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift index 0a59fe2d6..1045c8d2f 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift @@ -56,6 +56,10 @@ struct RunDartTestResponse: Codable { var details: String? } +struct SetLifecycleCallbacksStateRequest: Codable { + var state: Map +} + struct ConfigureRequest: Codable { var findTimeoutMillis: Int } diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift index 6a3247923..a5ef9c379 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift @@ -24,6 +24,15 @@ class PatrolAppServiceClient { performRequest(requestName: "listDartLifecycleCallbacks", completion: completion) } + func setLifecycleCallbacksState(request: SetLifecycleCallbacksStateRequest, completion: @escaping (Result) -> Void) { + do { + let body = try JSONEncoder().encode(request) + performRequest(requestName: "setLifecycleCallbacksState", body: body, completion: completion) + } catch let err { + completion(.failure(err)) + } + } + func runDartTest(request: RunDartTestRequest, completion: @escaping (Result) -> Void) { do { let body = try JSONEncoder().encode(request) diff --git a/packages/patrol/lib/src/native/contracts/contracts.dart b/packages/patrol/lib/src/native/contracts/contracts.dart index a8294f816..7e8879e09 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.dart @@ -157,6 +157,27 @@ class RunDartTestResponse with EquatableMixin { ]; } +@JsonSerializable() +class SetLifecycleCallbacksStateRequest with EquatableMixin { + SetLifecycleCallbacksStateRequest({ + required this.state, + }); + + factory SetLifecycleCallbacksStateRequest.fromJson( + Map json) => + _$SetLifecycleCallbacksStateRequestFromJson(json); + + final Map state; + + Map toJson() => + _$SetLifecycleCallbacksStateRequestToJson(this); + + @override + List get props => [ + state, + ]; +} + @JsonSerializable() class ConfigureRequest with EquatableMixin { ConfigureRequest({ diff --git a/packages/patrol/lib/src/native/contracts/contracts.g.dart b/packages/patrol/lib/src/native/contracts/contracts.g.dart index 09c5ca740..d82e1289e 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.g.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.g.dart @@ -85,6 +85,18 @@ const _$RunDartTestResponseResultEnumMap = { RunDartTestResponseResult.failure: 'failure', }; +SetLifecycleCallbacksStateRequest _$SetLifecycleCallbacksStateRequestFromJson( + Map json) => + SetLifecycleCallbacksStateRequest( + state: json['state'] as Map, + ); + +Map _$SetLifecycleCallbacksStateRequestToJson( + SetLifecycleCallbacksStateRequest instance) => + { + 'state': instance.state, + }; + ConfigureRequest _$ConfigureRequestFromJson(Map json) => ConfigureRequest( findTimeoutMillis: json['findTimeoutMillis'] as int, diff --git a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart index 7ddbf2a75..5133e3e76 100644 --- a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart +++ b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart @@ -23,6 +23,13 @@ abstract class PatrolAppServiceServer { final body = jsonEncode(result.toJson()); return Response.ok(body); + } else if ('setLifecycleCallbacksState' == request.url.path) { + final stringContent = await request.readAsString(utf8); + final json = jsonDecode(stringContent); + final requestObj = SetLifecycleCallbacksStateRequest.fromJson( + json as Map); + + final result = await setLifecycleCallbacksState(requestObj); } else if ('runDartTest' == request.url.path) { final stringContent = await request.readAsString(utf8); final json = jsonDecode(stringContent); @@ -40,5 +47,7 @@ abstract class PatrolAppServiceServer { Future listDartTests(); Future listDartLifecycleCallbacks(); + Future setLifecycleCallbacksState( + SetLifecycleCallbacksStateRequest request); Future runDartTest(RunDartTestRequest request); } diff --git a/schema.dart b/schema.dart index 8aea3fcbd..4da813b42 100644 --- a/schema.dart +++ b/schema.dart @@ -39,9 +39,14 @@ class RunDartTestResponse { String? details; } +class SetLifecycleCallbacksStateRequest { + late Map state; +} + abstract class PatrolAppService { ListDartTestsResponse listDartTests(); ListDartLifecycleCallbacksResponse listDartLifecycleCallbacks(); + void setLifecycleCallbacksState(SetLifecycleCallbacksStateRequest request); RunDartTestResponse runDartTest(RunDartTestRequest request); } From ae9c3da3741a9686c25c5aeb24fb7ed33272138b Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 16 Oct 2023 18:53:24 +0200 Subject: [PATCH 37/77] progress with PatrolAppService.setLifecycleCallbacksState --- .../leancode/patrol/PatrolAppServiceClient.kt | 19 +++++++++++++------ .../pl/leancode/patrol/PatrolJUnitRunner.java | 9 +++++++-- .../pl/leancode/patrol/contracts/Contracts.kt | 6 +++++- .../contracts/PatrolAppServiceClient.kt | 5 +++-- .../Classes/AutomatorServer/Contracts.swift | 4 ++++ .../PatrolAppServiceClient.swift | 2 +- packages/patrol/lib/src/common.dart | 13 +++++++++++-- .../lib/src/native/contracts/contracts.dart | 12 ++++++++++++ .../lib/src/native/contracts/contracts.g.dart | 4 ++++ .../contracts/patrol_app_service_server.dart | 5 ++++- .../lib/src/native/patrol_app_service.dart | 14 ++++++++++++++ schema.dart | 5 ++++- 12 files changed, 82 insertions(+), 16 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt index 6be99a815..f072b97e0 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolAppServiceClient.kt @@ -33,16 +33,23 @@ class PatrolAppServiceClient { return result.group } - @Throws(PatrolAppServiceClientException::class) - fun runDartTest(name: String): Contracts.RunDartTestResponse { - Logger.i("PatrolAppServiceClient.runDartTest($name)") - return client.runDartTest(Contracts.RunDartTestRequest(name)) - } - @Throws(PatrolAppServiceClientException::class) fun listDartLifecycleCallbacks(): Contracts.ListDartLifecycleCallbacksResponse { Logger.i("PatrolAppServiceClient.listDartLifecycleCallbacks()") val result = client.listDartLifecycleCallbacks() return result } + + @Throws(PatrolAppServiceClientException::class) + fun setLifecycleCallbacksState(data: Map): Contracts.Empty { + Logger.i("PatrolAppServiceClient.setLifecycleCallbacksState()") + val result = client.setLifecycleCallbacksState(Contracts.SetLifecycleCallbacksStateRequest(data)) + return result + } + + @Throws(PatrolAppServiceClientException::class) + fun runDartTest(name: String): Contracts.RunDartTestResponse { + Logger.i("PatrolAppServiceClient.runDartTest($name)") + return client.runDartTest(Contracts.RunDartTestRequest(name)) + } } diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 61df8734f..6800ff8b7 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -230,10 +230,15 @@ static String convertStreamToString(InputStream inputStream) { * * This is required because the app is launched in a new process for each test. */ - public void setLifecycleCallbacksStateInApp() { + public void setLifecycleCallbacksState() { final String TAG = "PatrolJUnitRunner.setLifecycleCallbacksStateInApp(): "; - + try { + patrolAppServiceClient.setLifecycleCallbacksState(readStateFile()); + } catch (PatrolAppServiceClientException e) { + Logger.INSTANCE.e(TAG + "Failed to set lifecycle callbacks state in app: ", e); + throw new RuntimeException(e); + } } /** diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt index 062c2d50f..e1545c5f7 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/Contracts.kt @@ -33,6 +33,10 @@ class Contracts { fine, } + class Empty ( + + ) + data class DartGroupEntry ( val name: String, val type: GroupEntryType, @@ -62,7 +66,7 @@ class Contracts { } data class SetLifecycleCallbacksStateRequest ( - val state: Map + val state: Map ) data class ConfigureRequest ( diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt index 48af70593..12bdc9bf8 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/contracts/PatrolAppServiceClient.kt @@ -24,8 +24,9 @@ class PatrolAppServiceClient(address: String, port: Int, private val timeout: Lo return json.fromJson(response, Contracts.ListDartLifecycleCallbacksResponse::class.java) } - fun setLifecycleCallbacksState(request: Contracts.SetLifecycleCallbacksStateRequest) { - return performRequest("setLifecycleCallbacksState", json.toJson(request)) + fun setLifecycleCallbacksState(request: Contracts.SetLifecycleCallbacksStateRequest): Contracts.Empty { + val response = performRequest("setLifecycleCallbacksState", json.toJson(request)) + return json.fromJson(response, Contracts.Empty::class.java) } fun runDartTest(request: Contracts.RunDartTestRequest): Contracts.RunDartTestResponse { diff --git a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift index 1045c8d2f..a413af830 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift @@ -32,6 +32,10 @@ enum SetLocationAccuracyRequestLocationAccuracy: String, Codable { case fine } +struct Empty: Codable { + +} + struct DartGroupEntry: Codable { var name: String var type: GroupEntryType diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift index a5ef9c379..67dcd3d48 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift @@ -24,7 +24,7 @@ class PatrolAppServiceClient { performRequest(requestName: "listDartLifecycleCallbacks", completion: completion) } - func setLifecycleCallbacksState(request: SetLifecycleCallbacksStateRequest, completion: @escaping (Result) -> Void) { + func setLifecycleCallbacksState(request: SetLifecycleCallbacksStateRequest, completion: @escaping (Result) -> Void) { do { let body = try JSONEncoder().encode(request) performRequest(requestName: "setLifecycleCallbacksState", body: body, completion: completion) diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index f734ebd56..63784b373 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -65,7 +65,16 @@ void patrolSetUpAll(Future Function() body) { return; } - // TODO: Skip calling body if it this setUpAll was already executed + // Skip calling body if it this setUpAll was already executed + assert( + patrolAppService.lifecycleCallbacksState[setUpAllName] != null, + 'setUpAll "$setUpAllName" was not registered in PatrolAppService. This looks very nasty.', + ); + + if (patrolAppService.lifecycleCallbacksState[setUpAllName] ?? false) { + patrolDebug('skipping setUpAll "$setUpAllName" because it already ran'); + return; + } // patrolDebug('OH SO ARE WE BLOCKED HERE???'); final requestedTest = await patrolAppService.testExecutionRequested; @@ -76,7 +85,7 @@ void patrolSetUpAll(Future Function() body) { return; } - // TODO: Mark this setUpAll as executed + // Mark this setUpAll as executed final nativeAutomator = PatrolBinding.instance.nativeAutomator; await nativeAutomator.markLifecycleCallbackExecuted(setUpAllName); diff --git a/packages/patrol/lib/src/native/contracts/contracts.dart b/packages/patrol/lib/src/native/contracts/contracts.dart index 7e8879e09..9c4e0101f 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.dart @@ -48,6 +48,18 @@ enum SetLocationAccuracyRequestLocationAccuracy { fine } +@JsonSerializable() +class Empty with EquatableMixin { + Empty(); + + factory Empty.fromJson(Map json) => _$EmptyFromJson(json); + + Map toJson() => _$EmptyToJson(this); + + @override + List get props => const []; +} + @JsonSerializable() class DartGroupEntry with EquatableMixin { DartGroupEntry({ diff --git a/packages/patrol/lib/src/native/contracts/contracts.g.dart b/packages/patrol/lib/src/native/contracts/contracts.g.dart index d82e1289e..4a5bad73e 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.g.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.g.dart @@ -6,6 +6,10 @@ part of 'contracts.dart'; // JsonSerializableGenerator // ************************************************************************** +Empty _$EmptyFromJson(Map json) => Empty(); + +Map _$EmptyToJson(Empty instance) => {}; + DartGroupEntry _$DartGroupEntryFromJson(Map json) => DartGroupEntry( name: json['name'] as String, diff --git a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart index 5133e3e76..95e5b08d4 100644 --- a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart +++ b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart @@ -30,6 +30,9 @@ abstract class PatrolAppServiceServer { json as Map); final result = await setLifecycleCallbacksState(requestObj); + + final body = jsonEncode(result.toJson()); + return Response.ok(body); } else if ('runDartTest' == request.url.path) { final stringContent = await request.readAsString(utf8); final json = jsonDecode(stringContent); @@ -47,7 +50,7 @@ abstract class PatrolAppServiceServer { Future listDartTests(); Future listDartLifecycleCallbacks(); - Future setLifecycleCallbacksState( + Future setLifecycleCallbacksState( SetLifecycleCallbacksStateRequest request); Future runDartTest(RunDartTestRequest request); } diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index 4c9452b63..14bed8b24 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -79,6 +79,11 @@ class PatrolAppService extends PatrolAppServiceServer { /// appended. List setUpAlls = []; + /// Map that knows which lifecycle callbacks have been already executed. + /// + /// It's populated near the beginning of the app startup during each test. + Map lifecycleCallbacksState = {}; + /// A completer that completes with the name of the Dart test file that was /// requested to execute by the native side. final _testExecutionRequested = Completer(); @@ -217,4 +222,13 @@ class PatrolAppService extends PatrolAppServiceServer { tearDownAlls: [], ); } + + @override + Future setLifecycleCallbacksState( + SetLifecycleCallbacksStateRequest request, + ) async { + print('PatrolAppService.setLifecycleCallbacksState() called'); + lifecycleCallbacksState = request.state.cast(); + return Empty(); + } } diff --git a/schema.dart b/schema.dart index 4da813b42..e469b63b1 100644 --- a/schema.dart +++ b/schema.dart @@ -7,6 +7,9 @@ // - Generic types (IOSServer, IOSClient, AndroidServer, AndroidClient, DartServer, DartClient) // control where we need clients and servers +// void doesn't work +class Empty {} + class DartGroupEntry { late String name; late GroupEntryType type; @@ -46,7 +49,7 @@ class SetLifecycleCallbacksStateRequest { abstract class PatrolAppService { ListDartTestsResponse listDartTests(); ListDartLifecycleCallbacksResponse listDartLifecycleCallbacks(); - void setLifecycleCallbacksState(SetLifecycleCallbacksStateRequest request); + Empty setLifecycleCallbacksState(SetLifecycleCallbacksStateRequest request); RunDartTestResponse runDartTest(RunDartTestRequest request); } From 1aadc02d03f51b735f42abc9a2b6557b73f32bc5 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 11:20:13 +0200 Subject: [PATCH 38/77] fix bug with white screen on normal test run --- .../patrol/example/MainActivityTest.java | 3 +- packages/patrol/lib/src/common.dart | 10 +++++-- .../lib/src/native/patrol_app_service.dart | 29 +++++++++++++++---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index ac4f5bd27..ea91ea78c 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -22,6 +22,8 @@ public static Object[] testCases() { Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); instrumentation.saveLifecycleCallbacks(lifecycleCallbacks); + } else { + instrumentation.setLifecycleCallbacksState(); } return dartTests; @@ -36,7 +38,6 @@ public MainActivityTest(String dartTestName) { @Test public void runDartTest() { PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); - instrumentation.setLifecycleCallbacksState(); instrumentation.runDartTest(dartTestName); } } diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 63784b373..a1e34bed3 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -60,18 +60,22 @@ void patrolSetUpAll(Future Function() body) { if (await global_state.isInitialRun) { // Skip calling body if we're in test discovery phase patrolDebug( - "skipping setUpAll '$setUpAllName' because it's test discovery phase", + 'skipping setUpAll "$setUpAllName" because we are in the initial run', ); return; } + final callbacksState = await patrolAppService.callbacksStateSet; + // Skip calling body if it this setUpAll was already executed + print('lifecycleCallbacksState: $callbacksState'); + assert( - patrolAppService.lifecycleCallbacksState[setUpAllName] != null, + callbacksState[setUpAllName] != null, 'setUpAll "$setUpAllName" was not registered in PatrolAppService. This looks very nasty.', ); - if (patrolAppService.lifecycleCallbacksState[setUpAllName] ?? false) { + if (callbacksState[setUpAllName] ?? false) { patrolDebug('skipping setUpAll "$setUpAllName" because it already ran'); return; } diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index 14bed8b24..a8029a924 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -79,10 +79,14 @@ class PatrolAppService extends PatrolAppServiceServer { /// appended. List setUpAlls = []; - /// Map that knows which lifecycle callbacks have been already executed. + final _callbacksStateSet = Completer>(); + + /// A completer that completes when the native side sets the state of + /// lifecycle callbacks. /// - /// It's populated near the beginning of the app startup during each test. - Map lifecycleCallbacksState = {}; + /// Completes with a map that knows which lifecycle callbacks have been + /// already executed. + Future> get callbacksStateSet => _callbacksStateSet.future; /// A completer that completes with the name of the Dart test file that was /// requested to execute by the native side. @@ -135,6 +139,7 @@ class PatrolAppService extends PatrolAppServiceServer { required String? details, }) async { print('PatrolAppService.markDartTestAsCompleted(): $dartFileName'); + assert( _testExecutionRequested.isCompleted, 'Tried to mark a test as completed, but no tests were requested to run', @@ -198,9 +203,10 @@ class PatrolAppService extends PatrolAppServiceServer { @override Future runDartTest(RunDartTestRequest request) async { + print('PatrolAppService.runDartTest(${request.name}) called'); + assert(_testExecutionCompleted.isCompleted == false); - print('PatrolAppService.runDartTest(${request.name}) called'); _testExecutionRequested.complete(request.name); final testExecutionResult = await testExecutionCompleted; @@ -217,6 +223,10 @@ class PatrolAppService extends PatrolAppServiceServer { listDartLifecycleCallbacks() async { print('PatrolAppService.listDartLifecycleCallbacks() called'); + assert(_testExecutionRequested.isCompleted == false); + assert(_testExecutionCompleted.isCompleted == false); + assert(_callbacksStateSet.isCompleted == false); + return ListDartLifecycleCallbacksResponse( setUpAlls: setUpAlls, tearDownAlls: [], @@ -228,7 +238,16 @@ class PatrolAppService extends PatrolAppServiceServer { SetLifecycleCallbacksStateRequest request, ) async { print('PatrolAppService.setLifecycleCallbacksState() called'); - lifecycleCallbacksState = request.state.cast(); + assert(_testExecutionRequested.isCompleted == false); + assert(_testExecutionCompleted.isCompleted == false); + assert(_callbacksStateSet.isCompleted == false); + + final state = {}; + for (final e in request.state.entries) { + state[e.key as String] = (e.value as String).toLowerCase() == 'true'; + } + + _callbacksStateSet.complete(state); return Empty(); } } From 636a476b3af64a5163be8580080c7cc0cf8876cd Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 11:22:52 +0200 Subject: [PATCH 39/77] logs.dart: fix analyzer warnings --- packages/patrol/lib/src/logs.dart | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/patrol/lib/src/logs.dart b/packages/patrol/lib/src/logs.dart index 7cdbbfad2..cec31afa6 100644 --- a/packages/patrol/lib/src/logs.dart +++ b/packages/patrol/lib/src/logs.dart @@ -1,8 +1,13 @@ import 'package:meta/meta.dart'; -final runKey = Object().hashCode; +final _runKey = Object().hashCode; +/// Logs a message with a tag that's constant in the single app process +/// instance. +/// +/// Helps to differentiate same logs that happen in different app process runs. @internal void patrolDebug(String message) { - print('PATROL_DEBUG($runKey): $message'); + // ignore: avoid_print + print('PATROL_DEBUG($_runKey): $message'); } From f629d7a78b4417f5715e20cd9834e75f788d9e06 Mon Sep 17 00:00:00 2001 From: Damian Cudzik Date: Tue, 17 Oct 2023 12:52:07 +0200 Subject: [PATCH 40/77] Add map support --- .../android/android_contracts_generator.dart | 11 +++++--- .../dart/dart_contracts_generator.dart | 11 +++++--- .../ios/ios_contracts_generator.dart | 11 +++++--- .../patrol_gen/lib/src/resolve_schema.dart | 18 +++++++++---- packages/patrol_gen/lib/src/schema.dart | 25 ++++++++++++++++--- 5 files changed, 59 insertions(+), 17 deletions(-) diff --git a/packages/patrol_gen/lib/src/generators/android/android_contracts_generator.dart b/packages/patrol_gen/lib/src/generators/android/android_contracts_generator.dart index c2a1ca671..efbfabd1c 100644 --- a/packages/patrol_gen/lib/src/generators/android/android_contracts_generator.dart +++ b/packages/patrol_gen/lib/src/generators/android/android_contracts_generator.dart @@ -39,9 +39,14 @@ package ${config.package}; String _createMessage(Message message) { final fields = message.fields.map((e) { final optional = e.isOptional ? '? = null' : ''; - return e.isList - ? ' val ${e.name}: List<${_transformType(e.type)}>$optional' - : ' val ${e.name}: ${_transformType(e.type)}$optional'; + return switch (e.type) { + MapFieldType(keyType: final keyType, valueType: final valueType) => + ' val ${e.name}: Map<${_transformType(keyType)}, ${_transformType(valueType)}>$optional', + ListFieldType(type: final type) => + ' val ${e.name}: List<${_transformType(type)}>$optional', + OrdinaryFieldType(type: final type) => + ' val ${e.name}: ${_transformType(type)}$optional', + }; }).join(',\n'); final dataKeyword = fields.isNotEmpty ? 'data ' : ''; diff --git a/packages/patrol_gen/lib/src/generators/dart/dart_contracts_generator.dart b/packages/patrol_gen/lib/src/generators/dart/dart_contracts_generator.dart index 7a467ea6b..5898293f9 100644 --- a/packages/patrol_gen/lib/src/generators/dart/dart_contracts_generator.dart +++ b/packages/patrol_gen/lib/src/generators/dart/dart_contracts_generator.dart @@ -53,9 +53,14 @@ enum ${enumDefinition.name} { String? _createMessage(Message message) { final fieldsContent = message.fields .map( - (f) => f.isList - ? 'final List<${f.type}>${f.isOptional ? '?' : ''} ${f.name};' - : 'final ${f.type}${f.isOptional ? '?' : ''} ${f.name};', + (f) => switch (f.type) { + ListFieldType(type: final type) => + 'final List<$type>${f.isOptional ? '?' : ''} ${f.name};', + MapFieldType(keyType: final keyType, valueType: final valueType) => + 'final Map<$keyType,$valueType>${f.isOptional ? '?' : ''} ${f.name};', + OrdinaryFieldType(type: final type) => + 'final $type${f.isOptional ? '?' : ''} ${f.name};' + }, ) .join('\n'); diff --git a/packages/patrol_gen/lib/src/generators/ios/ios_contracts_generator.dart b/packages/patrol_gen/lib/src/generators/ios/ios_contracts_generator.dart index d6288532d..1f1fd9392 100644 --- a/packages/patrol_gen/lib/src/generators/ios/ios_contracts_generator.dart +++ b/packages/patrol_gen/lib/src/generators/ios/ios_contracts_generator.dart @@ -34,9 +34,14 @@ class IOSContractsGenerator { String _createMessage(Message message) { final fields = message.fields.map((e) { final optional = e.isOptional ? '?' : ''; - return e.isList - ? ' var ${e.name}: [${_transformType(e.type)}]$optional' - : ' var ${e.name}: ${_transformType(e.type)}$optional'; + return switch (e.type) { + MapFieldType(keyType: final keyType, valueType: final valueType) => + ' var ${e.name}: [${_transformType(keyType)}: ${_transformType(valueType)}]$optional', + ListFieldType(type: final type) => + ' var ${e.name}: [${_transformType(type)}]$optional', + OrdinaryFieldType(type: final type) => + ' var ${e.name}: ${_transformType(type)}$optional', + }; }).join('\n'); return ''' diff --git a/packages/patrol_gen/lib/src/resolve_schema.dart b/packages/patrol_gen/lib/src/resolve_schema.dart index ad1b515c0..0be68dd0d 100644 --- a/packages/patrol_gen/lib/src/resolve_schema.dart +++ b/packages/patrol_gen/lib/src/resolve_schema.dart @@ -106,20 +106,28 @@ Message _createMessage(ClassDeclaration declaration) { final isOptional = type.question != null; final fieldName = e.variables.first.name.lexeme; - if (type.type?.isDartCoreList ?? false) { + if (type.type?.isDartCoreMap ?? false) { + final arguments = type.typeArguments!.arguments; + return MessageField( + isOptional: isOptional, + name: fieldName, + type: MapFieldType( + keyType: (arguments[0] as NamedType).name2.lexeme, + valueType: (arguments[1] as NamedType).name2.lexeme, + ), + ); + } else if (type.type?.isDartCoreList ?? false) { final genericType = type.typeArguments!.arguments.first as NamedType; return MessageField( isOptional: isOptional, name: fieldName, - type: genericType.name2.lexeme, - isList: true, + type: ListFieldType(type: genericType.name2.lexeme), ); } else { return MessageField( isOptional: isOptional, name: fieldName, - type: type.name2.lexeme, - isList: false, + type: OrdinaryFieldType(type: type.name2.lexeme), ); } } else { diff --git a/packages/patrol_gen/lib/src/schema.dart b/packages/patrol_gen/lib/src/schema.dart index 7d40d3179..8aab84578 100644 --- a/packages/patrol_gen/lib/src/schema.dart +++ b/packages/patrol_gen/lib/src/schema.dart @@ -5,18 +5,37 @@ class Enum { final List fields; } +sealed class MessageFieldType {} + +class OrdinaryFieldType implements MessageFieldType { + const OrdinaryFieldType({required this.type}); + + final String type; +} + +class ListFieldType implements MessageFieldType { + const ListFieldType({required this.type}); + + final String type; +} + +class MapFieldType implements MessageFieldType { + const MapFieldType({required this.keyType, required this.valueType}); + + final String keyType; + final String valueType; +} + class MessageField { const MessageField({ required this.name, required this.type, required this.isOptional, - required this.isList, }); final bool isOptional; - final bool isList; final String name; - final String type; + final MessageFieldType type; } class Message { From 1b718a1c0f1d59f5f3acd638789f6f317b30fe61 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 13:10:05 +0200 Subject: [PATCH 41/77] remove PatrolJUnitRunner --- .../patrol/CustomPatrolJUnitRunner.kt | 66 ------------------- .../patrol/example/android/app/build.gradle | 2 +- 2 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt deleted file mode 100644 index a8f6276c3..000000000 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/CustomPatrolJUnitRunner.kt +++ /dev/null @@ -1,66 +0,0 @@ -package pl.leancode.patrol - -import android.annotation.SuppressLint -import android.os.Build -import android.os.Bundle -import androidx.annotation.OptIn -import androidx.test.annotation.ExperimentalTestApi -import androidx.test.platform.app.InstrumentationRegistry -import androidx.test.services.storage.TestStorage -import org.junit.runner.Description -import org.junit.runner.notification.RunListener - -@OptIn(ExperimentalTestApi::class) -class CustomPatrolJUnitRunner : PatrolJUnitRunner() { - private var testStorage: TestStorage? = null - - override fun onCreate(arguments: Bundle) { -// try { -// val outputStream = testStorage!!.openOutputFile("patrol_state/teardowns.txt") -// } catch (e: FileNotFoundException) { -// throw RuntimeException(e) -// } - - val listeners = listOfNotNull( - arguments.getCharSequence("listener"), - PatrolStoragePermissionListener::class.java.name - ).joinToString(separator = ",") - - arguments.putCharSequence("listener", listeners) - super.onCreate(arguments) - } - - override fun onStart() { - super.onStart() - - testStorage = TestStorage() - } -} - -@SuppressLint("UnsafeOptInUsageError") -class PatrolStoragePermissionListener : RunListener() { - - val testStorage by lazy { TestStorage() } - - override fun testStarted(description: Description) { - Logger.d("testStarted with description: $description") - - // This file is created at /sdcard/googletest/test_outputfiles - val os = testStorage.openOutputFile("patrol.txt", true) - os.write("testStarted with description: $description\n".toByteArray()) - - InstrumentationRegistry.getInstrumentation().uiAutomation.apply { - val testServicesPackage = "androidx.test.services" - when { - Build.VERSION.SDK_INT >= Build.VERSION_CODES.R -> { - executeShellCommand("appops set $testServicesPackage MANAGE_EXTERNAL_STORAGE allow") - } - - Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP -> { - executeShellCommand("pm grant $testServicesPackage android.permission.READ_EXTERNAL_STORAGE") - executeShellCommand("pm grant $testServicesPackage android.permission.WRITE_EXTERNAL_STORAGE") - } - } - } - } -} diff --git a/packages/patrol/example/android/app/build.gradle b/packages/patrol/example/android/app/build.gradle index 4b082c7dc..cb9dd3f5c 100644 --- a/packages/patrol/example/android/app/build.gradle +++ b/packages/patrol/example/android/app/build.gradle @@ -43,7 +43,7 @@ android { targetSdk 33 versionCode flutterVersionCode.toInteger() versionName flutterVersionName - testInstrumentationRunner "pl.leancode.patrol.CustomPatrolJUnitRunner" + testInstrumentationRunner "pl.leancode.patrol.PatrolJUnitRunner" testInstrumentationRunnerArguments clearPackageData: "true" } From 005ae57099dcbdd0ca71a9a6f456eb19c20ed1e9 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 14:17:04 +0200 Subject: [PATCH 42/77] delete cleaning state file on initial run (not needed) --- .../main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java | 5 ----- .../java/pl/leancode/patrol/example/MainActivityTest.java | 3 --- 2 files changed, 8 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 6800ff8b7..22e8caef6 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -189,11 +189,6 @@ public void markLifecycleCallbackExecuted(String name) { writeStateFile(state); } - public void clearStateFile() { - Logger.INSTANCE.i("PatrolJUnitRunner.clearStateFile()"); - writeStateFile(new HashMap<>()); - } - private Map readStateFile() { try { InputStream inputStream = TestStorageUtil.getInputStream(stateFileUri, getContentResolver()); diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index ea91ea78c..a419edebc 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -17,9 +17,6 @@ public static Object[] testCases() { Object[] dartTests = instrumentation.listDartTests(); if (instrumentation.isInitialRun()) { - // Clear the list of lifecycle callbacks from the previous run. - instrumentation.clearStateFile(); - Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); instrumentation.saveLifecycleCallbacks(lifecycleCallbacks); } else { From d4151de6fca00c459113990703f5a942cc9e2016 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 14:19:53 +0200 Subject: [PATCH 43/77] remove redundant debugLog from generated test_bundle --- packages/patrol_cli/lib/src/test_bundler.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 5b39978af..3c44aeed2 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -103,7 +103,6 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // An additional callback to discover setUpAlls. tearDownAll(() async { if (await global_state.isInitialRun) { - patrolDebug('tearDownAll(): setUpAlls: \${appService.setUpAlls}'); callbacksExplorationCompleter.complete(); } }); From cef44a71151a527ea5be05c9c300c43f9911c433 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 14:31:09 +0200 Subject: [PATCH 44/77] remove some debugging aids --- packages/patrol/lib/src/common.dart | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index a1e34bed3..fb40539c9 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -67,20 +67,17 @@ void patrolSetUpAll(Future Function() body) { final callbacksState = await patrolAppService.callbacksStateSet; - // Skip calling body if it this setUpAll was already executed - print('lifecycleCallbacksState: $callbacksState'); - assert( callbacksState[setUpAllName] != null, 'setUpAll "$setUpAllName" was not registered in PatrolAppService. This looks very nasty.', ); if (callbacksState[setUpAllName] ?? false) { + // Skip calling body if this setUpAll was already executed patrolDebug('skipping setUpAll "$setUpAllName" because it already ran'); return; } - // patrolDebug('OH SO ARE WE BLOCKED HERE???'); final requestedTest = await patrolAppService.testExecutionRequested; // Skip calling if parentGroupName is not a substring of requestedTestName From 9c748e751d0e3ef293bb9a40fc630bc42ac06b93 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 14:38:37 +0200 Subject: [PATCH 45/77] update completer names to be `didX` - more readable --- packages/patrol/lib/src/binding.dart | 4 +- packages/patrol/lib/src/common.dart | 5 ++- .../lib/src/native/patrol_app_service.dart | 44 ++++++++++--------- packages/patrol_cli/lib/src/test_bundler.dart | 12 ++--- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 63c22407d..e366dd8a9 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -68,7 +68,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { if (await global_state.isInitialRun) { // If this is the initial run, then no test has been requested to - // execute. Return to avoid blocking on testExecutionRequested below. + // execute. Return to avoid blocking on didRequestTestExecution below. return; } @@ -85,7 +85,7 @@ class PatrolBinding extends IntegrationTestWidgetsFlutterBinding { logger('tearDown(): test "$dartTestName": "$result"'); }); - final requestedDartTest = await patrolAppService.testExecutionRequested; + final requestedDartTest = await patrolAppService.didRequestTestExecution; if (requestedDartTest == _currentDartTest) { logger( diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index fb40539c9..775a4fba9 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -65,7 +65,8 @@ void patrolSetUpAll(Future Function() body) { return; } - final callbacksState = await patrolAppService.callbacksStateSet; + final callbacksState = + await patrolAppService.didReceiveLifecycleCallbacksState; assert( callbacksState[setUpAllName] != null, @@ -78,7 +79,7 @@ void patrolSetUpAll(Future Function() body) { return; } - final requestedTest = await patrolAppService.testExecutionRequested; + final requestedTest = await patrolAppService.didRequestTestExecution; // Skip calling if parentGroupName is not a substring of requestedTestName if (!requestedTest.startsWith(parentGroupsName)) { diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index a8029a924..b32a56725 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -79,31 +79,33 @@ class PatrolAppService extends PatrolAppServiceServer { /// appended. List setUpAlls = []; - final _callbacksStateSet = Completer>(); + final _didReceiveLifecycleCallbacksState = Completer>(); /// A completer that completes when the native side sets the state of /// lifecycle callbacks. /// /// Completes with a map that knows which lifecycle callbacks have been /// already executed. - Future> get callbacksStateSet => _callbacksStateSet.future; + Future> get didReceiveLifecycleCallbacksState { + return _didReceiveLifecycleCallbacksState.future; + } /// A completer that completes with the name of the Dart test file that was /// requested to execute by the native side. - final _testExecutionRequested = Completer(); + final _didRequestTestExecution = Completer(); /// A future that completes with the name of the Dart test file that was /// requested to execute by the native side. - Future get testExecutionRequested => _testExecutionRequested.future; + Future get didRequestTestExecution => _didRequestTestExecution.future; - final _testExecutionCompleted = Completer<_TestExecutionResult>(); + final _didCompleteTestExecution = Completer<_TestExecutionResult>(); /// A future that completes when the Dart test file (whose execution was /// requested by the native side) completes. /// /// Returns true if the test passed, false otherwise. - Future<_TestExecutionResult> get testExecutionCompleted { - return _testExecutionCompleted.future; + Future<_TestExecutionResult> get didCompleteTestExecution { + return _didCompleteTestExecution.future; } /// Adds a setUpAll callback to the list of all setUpAll callbacks. @@ -141,18 +143,18 @@ class PatrolAppService extends PatrolAppServiceServer { print('PatrolAppService.markDartTestAsCompleted(): $dartFileName'); assert( - _testExecutionRequested.isCompleted, + _didRequestTestExecution.isCompleted, 'Tried to mark a test as completed, but no tests were requested to run', ); - final requestedDartTestName = await testExecutionRequested; + final requestedDartTestName = await didRequestTestExecution; assert( requestedDartTestName == dartFileName, 'Tried to mark test $dartFileName as completed, but the test ' 'that was most recently requested to run was $requestedDartTestName', ); - _testExecutionCompleted.complete( + _didCompleteTestExecution.complete( _TestExecutionResult(passed: passed, details: details), ); } @@ -169,7 +171,7 @@ class PatrolAppService extends PatrolAppServiceServer { Future waitForExecutionRequest(String dartTest) async { print('PatrolAppService: registered "$dartTest"'); - final requestedDartTest = await testExecutionRequested; + final requestedDartTest = await didRequestTestExecution; if (requestedDartTest != dartTest) { // If the requested Dart test is not the one we're waiting for now, it // means that dartTest was already executed. Return false so that callers @@ -205,11 +207,11 @@ class PatrolAppService extends PatrolAppServiceServer { Future runDartTest(RunDartTestRequest request) async { print('PatrolAppService.runDartTest(${request.name}) called'); - assert(_testExecutionCompleted.isCompleted == false); + assert(_didCompleteTestExecution.isCompleted == false); - _testExecutionRequested.complete(request.name); + _didRequestTestExecution.complete(request.name); - final testExecutionResult = await testExecutionCompleted; + final testExecutionResult = await didCompleteTestExecution; return RunDartTestResponse( result: testExecutionResult.passed ? RunDartTestResponseResult.success @@ -223,9 +225,9 @@ class PatrolAppService extends PatrolAppServiceServer { listDartLifecycleCallbacks() async { print('PatrolAppService.listDartLifecycleCallbacks() called'); - assert(_testExecutionRequested.isCompleted == false); - assert(_testExecutionCompleted.isCompleted == false); - assert(_callbacksStateSet.isCompleted == false); + assert(_didRequestTestExecution.isCompleted == false); + assert(_didCompleteTestExecution.isCompleted == false); + assert(_didReceiveLifecycleCallbacksState.isCompleted == false); return ListDartLifecycleCallbacksResponse( setUpAlls: setUpAlls, @@ -238,16 +240,16 @@ class PatrolAppService extends PatrolAppServiceServer { SetLifecycleCallbacksStateRequest request, ) async { print('PatrolAppService.setLifecycleCallbacksState() called'); - assert(_testExecutionRequested.isCompleted == false); - assert(_testExecutionCompleted.isCompleted == false); - assert(_callbacksStateSet.isCompleted == false); + assert(_didRequestTestExecution.isCompleted == false); + assert(_didCompleteTestExecution.isCompleted == false); + assert(_didReceiveLifecycleCallbacksState.isCompleted == false); final state = {}; for (final e in request.state.entries) { state[e.key as String] = (e.value as String).toLowerCase() == 'true'; } - _callbacksStateSet.complete(state); + _didReceiveLifecycleCallbacksState.complete(state); return Empty(); } } diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 3c44aeed2..20df3e4c9 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -78,8 +78,8 @@ Future main() async { PatrolBinding.ensureInitialized(appService, nativeAutomator); - final testsExplorationCompleter = Completer(); - final callbacksExplorationCompleter = Completer(); + final didExploreTests = Completer(); + final didExploreLifecycleCallbacks = Completer(); // A special test to expore the hierarchy of groups and tests. This is a hack // around https://github.com/dart-lang/test/issues/1998. @@ -91,7 +91,7 @@ Future main() async { // to group() below. final topLevelGroup = Invoker.current!.liveTest.groups.first; final dartTestGroup = createDartTestGroup(topLevelGroup); - testsExplorationCompleter.complete(dartTestGroup); + didExploreTests.complete(dartTestGroup); print('patrol_test_explorer: obtained Dart-side test hierarchy:'); printGroupStructure(dartTestGroup); }); @@ -103,14 +103,14 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // An additional callback to discover setUpAlls. tearDownAll(() async { if (await global_state.isInitialRun) { - callbacksExplorationCompleter.complete(); + didExploreLifecycleCallbacks.complete(); } }); - appService.topLevelDartTestGroup = await testsExplorationCompleter.future; + appService.topLevelDartTestGroup = await didExploreTests.future; if (await global_state.isInitialRun) { - await callbacksExplorationCompleter.future; + await didExploreLifecycleCallbacks.future; } // Until now, the native test runner was waiting for us, the Dart side, to From d50706d5bff566766ab2b3ee5975f9f88ec1bdaa Mon Sep 17 00:00:00 2001 From: Damian Cudzik Date: Tue, 17 Oct 2023 14:46:43 +0200 Subject: [PATCH 46/77] Fix endpoints that do not return results --- .../PatrolAppServiceClient.swift | 44 +++++++++++--- .../android_http4k_client_generator.dart | 2 +- .../dart/dart_shelf_server_generator.dart | 18 +++--- .../ios/ios_url_session_client_generator.dart | 57 +++++++++++++++---- 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift index f6da06cbe..f48791431 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift @@ -17,20 +17,51 @@ class PatrolAppServiceClient { } func listDartTests(completion: @escaping (Result) -> Void) { - performRequest(requestName: "listDartTests", completion: completion) + performRequestWithResult(requestName: "listDartTests", completion: completion) } func runDartTest(request: RunDartTestRequest, completion: @escaping (Result) -> Void) { do { let body = try JSONEncoder().encode(request) - performRequest(requestName: "runDartTest", body: body, completion: completion) + performRequestWithResult(requestName: "runDartTest", body: body, completion: completion) } catch let err { completion(.failure(err)) } } - private func performRequest( + private func performRequestWithResult( requestName: String, body: Data? = nil, completion: @escaping (Result) -> Void + ) { + performRequest(requestName: requestName, body: body) { result in + switch result { + case .success(let data): + do { + let object = try JSONDecoder().decode(TResult.self, from: data) + completion(.success(object)) + } catch let err { + completion(.failure(err)) + } + case .failure(let error): + completion(.failure(error)) + } + } + } + + private func performRequestWithEmptyResult( + requestName: String, body: Data? = nil, completion: @escaping (Error?) -> Void + ) { + performRequest(requestName: requestName, body: body) { result in + switch result { + case .success(_): + completion(nil) + case .failure(let error): + completion(error) + } + } + } + + private func performRequest( + requestName: String, body: Data? = nil, completion: @escaping (Result) -> Void ) { let url = URL(string: "http://\(address):\(port)/\(requestName)")! @@ -47,12 +78,7 @@ class PatrolAppServiceClient { session.dataTask(with: request) { data, response, error in if (response as? HTTPURLResponse)?.statusCode == 200 { - do { - let object = try JSONDecoder().decode(TResult.self, from: data!) - completion(.success(object)) - } catch let err { - completion(.failure(err)) - } + completion(.success(data!)) } else { let message = "Invalid response: \(String(describing: response)) \(String(describing: data))" diff --git a/packages/patrol_gen/lib/src/generators/android/android_http4k_client_generator.dart b/packages/patrol_gen/lib/src/generators/android/android_http4k_client_generator.dart index d4ce57e3a..e1e42b9c0 100644 --- a/packages/patrol_gen/lib/src/generators/android/android_http4k_client_generator.dart +++ b/packages/patrol_gen/lib/src/generators/android/android_http4k_client_generator.dart @@ -98,7 +98,7 @@ $endpoints val response = performRequest("${endpoint.name}"$serializeParameter) return json.fromJson(response, Contracts.${endpoint.response!.name}::class.java)''' : ''' - return performRequest("${endpoint.name}"$serializeParameter)'''; + performRequest("${endpoint.name}"$serializeParameter)'''; return ''' fun ${endpoint.name}($parameterDef)$returnDef { diff --git a/packages/patrol_gen/lib/src/generators/dart/dart_shelf_server_generator.dart b/packages/patrol_gen/lib/src/generators/dart/dart_shelf_server_generator.dart index a4a06da53..861816b9a 100644 --- a/packages/patrol_gen/lib/src/generators/dart/dart_shelf_server_generator.dart +++ b/packages/patrol_gen/lib/src/generators/dart/dart_shelf_server_generator.dart @@ -39,7 +39,6 @@ import '${path.basename(config.contractsFilename)}'; String _generateHandlerCalls(Service service) { return service.endpoints.map((e) { var requestDeserialization = ''; - var responseSerialization = ''; if (e.request != null) { requestDeserialization = ''' @@ -49,16 +48,19 @@ final requestObj = ${e.request!.name}.fromJson(json as Map); '''; } - final handlerCall = e.request != null - ? 'final result = await ${e.name}(requestObj);' - : 'final result = await ${e.name}();'; - + var handlerCall = e.request != null + ? 'await ${e.name}(requestObj);' + : 'await ${e.name}();'; if (e.response != null) { - responseSerialization = ''' -final body = jsonEncode(result.toJson()); -return Response.ok(body);'''; + handlerCall = 'final result = $handlerCall'; } + final responseSerialization = e.response != null + ? ''' +final body = jsonEncode(result.toJson()); +return Response.ok(body);''' + : "return Response.ok('');"; + final elseKeyword = e == service.endpoints.first ? '' : 'else'; return ''' diff --git a/packages/patrol_gen/lib/src/generators/ios/ios_url_session_client_generator.dart b/packages/patrol_gen/lib/src/generators/ios/ios_url_session_client_generator.dart index 037b5ee18..69aacd46d 100644 --- a/packages/patrol_gen/lib/src/generators/ios/ios_url_session_client_generator.dart +++ b/packages/patrol_gen/lib/src/generators/ios/ios_url_session_client_generator.dart @@ -48,8 +48,39 @@ class ${service.name}Client { $endpoints - private func performRequest( + private func performRequestWithResult( requestName: String, body: Data? = nil, completion: @escaping (Result) -> Void + ) { + performRequest(requestName: requestName, body: body) { result in + switch result { + case .success(let data): + do { + let object = try JSONDecoder().decode(TResult.self, from: data) + completion(.success(object)) + } catch let err { + completion(.failure(err)) + } + case .failure(let error): + completion(.failure(error)) + } + } + } + + private func performRequestWithEmptyResult( + requestName: String, body: Data? = nil, completion: @escaping (Error?) -> Void + ) { + performRequest(requestName: requestName, body: body) { result in + switch result { + case .success(_): + completion(nil) + case .failure(let error): + completion(error) + } + } + } + + private func performRequest( + requestName: String, body: Data? = nil, completion: @escaping (Result) -> Void ) { let url = URL(string: "$url")! @@ -66,12 +97,7 @@ $endpoints session.dataTask(with: request) { data, response, error in if (response as? HTTPURLResponse)?.statusCode == 200 { - do { - let object = try JSONDecoder().decode(TResult.self, from: data!) - completion(.success(object)) - } catch let err { - completion(.failure(err)) - } + completion(.success(data!)) } else { let message = "$exceptionMessage" @@ -89,7 +115,7 @@ $endpoints final completionDef = endpoint.response != null ? 'completion: @escaping (Result<${endpoint.response!.name}, Error>) -> Void' - : 'completion: @escaping (Result) -> Void'; + : 'completion: @escaping (Error?) -> Void'; final parameters = endpoint.request != null ? '$requestDef, $completionDef' @@ -98,18 +124,25 @@ $endpoints final arguments = [ 'requestName: "${endpoint.name}"', if (endpoint.request != null) 'body: body', - 'completion: completion' + 'completion: completion', ].join(', '); + final performRequest = endpoint.response != null + ? 'performRequestWithResult' + : 'performRequestWithEmptyResult'; + final failureCompletion = endpoint.response != null + ? 'completion(.failure(err))' + : 'completion(err)'; + final bodyCode = endpoint.request != null ? ''' do { let body = try JSONEncoder().encode(request) - performRequest($arguments) + $performRequest($arguments) } catch let err { - completion(.failure(err)) + $failureCompletion }''' - : ' performRequest($arguments)'; + : ' $performRequest($arguments)'; return ''' func ${endpoint.name}($parameters) { From 957c4d11c424c58878e02b3815e49190695455cd Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 14:47:36 +0200 Subject: [PATCH 47/77] MainActivityTest.java: care about backward compatibility --- .../pl/leancode/patrol/PatrolJUnitRunner.java | 16 ++++++++++++++-- .../patrol/example/MainActivityTest.java | 11 +---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index 22e8caef6..b36299654 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -13,7 +13,6 @@ import android.os.Bundle; import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.runner.AndroidJUnitRunner; -import androidx.test.services.storage.TestStorage; import androidx.test.services.storage.file.HostedFile; import androidx.test.services.storage.internal.TestStorageUtil; import com.google.gson.Gson; @@ -142,6 +141,10 @@ public void waitForPatrolAppService() { public Object[] listDartTests() { final String TAG = "PatrolJUnitRunner.listDartTests(): "; + // This call is here for backward compatibility. + // It should be in MainActivityTest.java. + handleLifecycleCallbacks(); + try { final DartGroupEntry dartTestGroup = patrolAppServiceClient.listDartTests(); List dartTestCases = ContractsExtensionsKt.listTestsFlat(dartTestGroup, ""); @@ -158,6 +161,15 @@ public Object[] listDartTests() { } } + private void handleLifecycleCallbacks() { + if (isInitialRun()) { + Object[] lifecycleCallbacks = listLifecycleCallbacks(); + saveLifecycleCallbacks(lifecycleCallbacks); + } else { + setLifecycleCallbacksState(); + } + } + public Object[] listLifecycleCallbacks() { final String TAG = "PatrolJUnitRunner.listLifecycleCallbacks(): "; @@ -222,7 +234,7 @@ static String convertStreamToString(InputStream inputStream) { /** * Sets the state of lifecycle callbacks in the app. - * + *

* This is required because the app is launched in a new process for each test. */ public void setLifecycleCallbacksState() { diff --git a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java index a419edebc..675e46464 100644 --- a/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java +++ b/packages/patrol/example/android/app/src/androidTest/java/pl/leancode/patrol/example/MainActivityTest.java @@ -14,16 +14,7 @@ public static Object[] testCases() { PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); instrumentation.setUp(MainActivity.class); instrumentation.waitForPatrolAppService(); - Object[] dartTests = instrumentation.listDartTests(); - - if (instrumentation.isInitialRun()) { - Object[] lifecycleCallbacks = instrumentation.listLifecycleCallbacks(); - instrumentation.saveLifecycleCallbacks(lifecycleCallbacks); - } else { - instrumentation.setLifecycleCallbacksState(); - } - - return dartTests; + return instrumentation.listDartTests(); } public MainActivityTest(String dartTestName) { From 68b969b291b2b6be3b57d4afe94ea5471299c866 Mon Sep 17 00:00:00 2001 From: Albert Wolszon Date: Tue, 17 Oct 2023 17:00:39 +0200 Subject: [PATCH 48/77] Provide redirects for link changes from 2138733 --- docs/patrol/finders/advanced.mdx | 3 +++ docs/patrol/finders/finders-setup.mdx | 3 +++ docs/patrol/finders/overview.mdx | 3 +++ docs/patrol/finders/usage.mdx | 3 +++ docs/patrol/native/advanced.mdx | 3 +++ docs/patrol/native/feature-parity.mdx | 3 +++ docs/patrol/native/overview.mdx | 3 +++ docs/patrol/native/usage.mdx | 3 +++ 8 files changed, 24 insertions(+) create mode 100644 docs/patrol/finders/advanced.mdx create mode 100644 docs/patrol/finders/finders-setup.mdx create mode 100644 docs/patrol/finders/overview.mdx create mode 100644 docs/patrol/finders/usage.mdx create mode 100644 docs/patrol/native/advanced.mdx create mode 100644 docs/patrol/native/feature-parity.mdx create mode 100644 docs/patrol/native/overview.mdx create mode 100644 docs/patrol/native/usage.mdx diff --git a/docs/patrol/finders/advanced.mdx b/docs/patrol/finders/advanced.mdx new file mode 100644 index 000000000..9a8aba064 --- /dev/null +++ b/docs/patrol/finders/advanced.mdx @@ -0,0 +1,3 @@ +--- +redirect: /finders/advanced +--- diff --git a/docs/patrol/finders/finders-setup.mdx b/docs/patrol/finders/finders-setup.mdx new file mode 100644 index 000000000..ee38c9573 --- /dev/null +++ b/docs/patrol/finders/finders-setup.mdx @@ -0,0 +1,3 @@ +--- +redirect: /finders/finders-setup +--- diff --git a/docs/patrol/finders/overview.mdx b/docs/patrol/finders/overview.mdx new file mode 100644 index 000000000..ff33c72bc --- /dev/null +++ b/docs/patrol/finders/overview.mdx @@ -0,0 +1,3 @@ +--- +redirect: /finders/overview +--- diff --git a/docs/patrol/finders/usage.mdx b/docs/patrol/finders/usage.mdx new file mode 100644 index 000000000..df9cd5211 --- /dev/null +++ b/docs/patrol/finders/usage.mdx @@ -0,0 +1,3 @@ +--- +redirect: /finders/usage +--- diff --git a/docs/patrol/native/advanced.mdx b/docs/patrol/native/advanced.mdx new file mode 100644 index 000000000..f56697249 --- /dev/null +++ b/docs/patrol/native/advanced.mdx @@ -0,0 +1,3 @@ +--- +redirect: /native/advanced +--- diff --git a/docs/patrol/native/feature-parity.mdx b/docs/patrol/native/feature-parity.mdx new file mode 100644 index 000000000..ddcb4c0d4 --- /dev/null +++ b/docs/patrol/native/feature-parity.mdx @@ -0,0 +1,3 @@ +--- +redirect: /native/feature-parity +--- diff --git a/docs/patrol/native/overview.mdx b/docs/patrol/native/overview.mdx new file mode 100644 index 000000000..1d9a5ad43 --- /dev/null +++ b/docs/patrol/native/overview.mdx @@ -0,0 +1,3 @@ +--- +redirect: /native/overview +--- diff --git a/docs/patrol/native/usage.mdx b/docs/patrol/native/usage.mdx new file mode 100644 index 000000000..ed1b1adaf --- /dev/null +++ b/docs/patrol/native/usage.mdx @@ -0,0 +1,3 @@ +--- +redirect: /native/usage +--- From 54b2715addbee14358bcc4a5ab3c47ff57fa2921 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 20:23:58 +0200 Subject: [PATCH 49/77] Revert "Provide redirects for link changes from 2138733" --- docs/patrol/finders/advanced.mdx | 3 --- docs/patrol/finders/finders-setup.mdx | 3 --- docs/patrol/finders/overview.mdx | 3 --- docs/patrol/finders/usage.mdx | 3 --- docs/patrol/native/advanced.mdx | 3 --- docs/patrol/native/feature-parity.mdx | 3 --- docs/patrol/native/overview.mdx | 3 --- docs/patrol/native/usage.mdx | 3 --- 8 files changed, 24 deletions(-) delete mode 100644 docs/patrol/finders/advanced.mdx delete mode 100644 docs/patrol/finders/finders-setup.mdx delete mode 100644 docs/patrol/finders/overview.mdx delete mode 100644 docs/patrol/finders/usage.mdx delete mode 100644 docs/patrol/native/advanced.mdx delete mode 100644 docs/patrol/native/feature-parity.mdx delete mode 100644 docs/patrol/native/overview.mdx delete mode 100644 docs/patrol/native/usage.mdx diff --git a/docs/patrol/finders/advanced.mdx b/docs/patrol/finders/advanced.mdx deleted file mode 100644 index 9a8aba064..000000000 --- a/docs/patrol/finders/advanced.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /finders/advanced ---- diff --git a/docs/patrol/finders/finders-setup.mdx b/docs/patrol/finders/finders-setup.mdx deleted file mode 100644 index ee38c9573..000000000 --- a/docs/patrol/finders/finders-setup.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /finders/finders-setup ---- diff --git a/docs/patrol/finders/overview.mdx b/docs/patrol/finders/overview.mdx deleted file mode 100644 index ff33c72bc..000000000 --- a/docs/patrol/finders/overview.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /finders/overview ---- diff --git a/docs/patrol/finders/usage.mdx b/docs/patrol/finders/usage.mdx deleted file mode 100644 index df9cd5211..000000000 --- a/docs/patrol/finders/usage.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /finders/usage ---- diff --git a/docs/patrol/native/advanced.mdx b/docs/patrol/native/advanced.mdx deleted file mode 100644 index f56697249..000000000 --- a/docs/patrol/native/advanced.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /native/advanced ---- diff --git a/docs/patrol/native/feature-parity.mdx b/docs/patrol/native/feature-parity.mdx deleted file mode 100644 index ddcb4c0d4..000000000 --- a/docs/patrol/native/feature-parity.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /native/feature-parity ---- diff --git a/docs/patrol/native/overview.mdx b/docs/patrol/native/overview.mdx deleted file mode 100644 index 1d9a5ad43..000000000 --- a/docs/patrol/native/overview.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /native/overview ---- diff --git a/docs/patrol/native/usage.mdx b/docs/patrol/native/usage.mdx deleted file mode 100644 index ed1b1adaf..000000000 --- a/docs/patrol/native/usage.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect: /native/usage ---- From ddcb70fa753e47f45147122d74e2b2219014b45f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 22:02:05 +0200 Subject: [PATCH 50/77] update guide --- dev_docs/GUIDE.md | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/dev_docs/GUIDE.md b/dev_docs/GUIDE.md index f021c39e6..1e1cc5d9b 100644 --- a/dev_docs/GUIDE.md +++ b/dev_docs/GUIDE.md @@ -1,15 +1,31 @@ # Working on the test bundling feature +_Test bundling_, also known as _native automation_, is a core feature of Patrol. +It bridges the native world of tests on Android and iOS with the Flutter/Dart +world of tests. + +It lives in the [patrol package](../packages/patrol). + +To learn more about test bundling, [read this article][test_bundling_article]. + +This document is a collection of tips and tricks to make it easier to work on +test bundling-related code. + +### Tools + `adb logcat` is your friend. Spice it up with `-v color`. If you need something more powerful, check out [`purr`](https://github.com/google/purr). +### Show Dart-side logs only + +Search for `flutter :`. + ### Find out when a test starts Search for `TestRunner: started`. ``` 09-21 12:24:09.223 23387 23406 I TestRunner: started: runDartTest[callbacks_test testA](pl.leancode.patrol.example.MainActivityTest) - ``` ### Find out when a test ends @@ -18,10 +34,20 @@ Search for `TestRunner: finished`. ### I made some changes to test bundling code that result in a deadlock -This is normal when editing this code. It's a mine field of shared global -mutable state and things happening in parallel. +This can often happen when editing test bundling code. Because of various +limitations of the `test` package, which Patrol has to base on, test bundling +code is full of shared global mutable state and unobvious things happening in +parallel. + +When trying to find the cause of a deadlock: + +- search for `await`s in custom functions provided by Patrol (e.g. + `patrolTest()` and `patrolSetUpAll()`) and global lifecycle callbacks + registered by the generated Dart test bundle or PatrolBinding (e.g. + `tearDown()`s) +- Use `print`s amply to pinpint where the code is stuck. + +In the future, we should think about how to refactor this code to be more +maintainable and simpler. -Look for `await`s in custom functions provided by Patrol (e.g. `patrolTest()` -and `patrolSetUpAll()`) and global lifecycle callbacks registered by the -generated Dart test bundle or PatrolBinding (e.g. `tearDown()`s). Use `print`s -amply to pinpint where the code is stuck. +[test_bundling_article]: https://leancode.co/blog/patrol-2-0-improved-flutter-ui-testing From 36ab2096b97a27fe35788d1c0710942bd07bbade Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 17 Oct 2023 22:05:48 +0200 Subject: [PATCH 51/77] PatrolJUnitRunner: update comment --- .../main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index b36299654..a085a65cd 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -141,8 +141,9 @@ public void waitForPatrolAppService() { public Object[] listDartTests() { final String TAG = "PatrolJUnitRunner.listDartTests(): "; - // This call is here for backward compatibility. - // It should be in MainActivityTest.java. + // This call should be in MainActivityTest.java, but that would require + // users to change that file in their projects, thus breaking backward + // compatibility. handleLifecycleCallbacks(); try { From 33caf18a876330020350cc7287aa14567b0a6b08 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 18 Oct 2023 17:46:47 +0200 Subject: [PATCH 52/77] fix generated test_bundle referencing Completers with changede names --- packages/patrol_cli/lib/src/test_bundler.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 20df3e4c9..a38afdffc 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -29,7 +29,6 @@ import 'dart:async'; import 'package:flutter_test/flutter_test.dart'; import 'package:patrol/patrol.dart'; import 'package:patrol/src/global_state.dart' as global_state; -import 'package:patrol/src/logs.dart'; import 'package:patrol/src/native/contracts/contracts.dart'; import 'package:test_api/src/backend/invoker.dart'; @@ -118,7 +117,7 @@ ${generateGroupsCode(testFilePaths).split('\n').map((e) => ' $e').join('\n')} // about Dart tests. await nativeAutomator.markPatrolAppServiceReady(); - await appService.testExecutionCompleted; + await appService.didCompleteTestExecution; } '''; From 16f0b8f14029edef83542aa5da1e963b5052f48a Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 18 Oct 2023 17:55:35 +0200 Subject: [PATCH 53/77] make Hot Restart work again --- packages/patrol_cli/lib/src/test_bundler.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index a38afdffc..656a15e86 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -153,7 +153,11 @@ ${generateImports([testFilePath])} Future main() async { final nativeAutomator = NativeAutomator(config: NativeAutomatorConfig()); await nativeAutomator.initialize(); - PatrolBinding.ensureInitialized(); + + final appService = PatrolAppService(); + await runAppService(appService); + + PatrolBinding.ensureInitialized(appService, nativeAutomator); // START: GENERATED TEST GROUPS ${generateGroupsCode([testFilePath]).split('\n').map((e) => ' $e').join('\n')} From cf42a719ef00d79333e0a01dcc43674875ab5752 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 18 Oct 2023 18:27:48 +0200 Subject: [PATCH 54/77] fix setUpAll and tearDownAll - don't block when Hot Restart is enabled --- packages/patrol/lib/src/common.dart | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 775a4fba9..9456acaed 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -23,6 +23,16 @@ typedef PatrolTesterCallback = Future Function(PatrolIntegrationTester $); /// A modification of [setUp] that works with Patrol's native automation. void patrolSetUp(Future Function() body) { setUp(() async { + if (global_state.hotRestartEnabled) { + await body(); + return; + } + + if (await global_state.isInitialRun) { + // Skip calling body if we're in test discovery phase + return; + } + final currentTest = global_state.currentTestFullName; final requestedToExecute = await PatrolBinding.instance.patrolAppService @@ -37,6 +47,16 @@ void patrolSetUp(Future Function() body) { /// A modification of [tearDown] that works with Patrol's native automation. void patrolTearDown(Future Function() body) { tearDown(() async { + if (global_state.hotRestartEnabled) { + await body(); + return; + } + + if (await global_state.isInitialRun) { + // Skip calling body if we're in test discovery phase + return; + } + final currentTest = global_state.currentTestFullName; final requestedToExecute = await PatrolBinding.instance.patrolAppService From 5e730371db25a62f010f16d4c5c82336b930f13a Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 10:26:46 +0200 Subject: [PATCH 55/77] regenerate contracts --- packages/patrol/ios/Classes/AutomatorServer/Contracts.swift | 2 +- .../ios/Classes/AutomatorServer/PatrolAppServiceClient.swift | 4 ++-- packages/patrol/lib/src/native/contracts/contracts.dart | 2 +- packages/patrol/lib/src/native/contracts/contracts.g.dart | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift index a413af830..543adcdc2 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/Contracts.swift @@ -61,7 +61,7 @@ struct RunDartTestResponse: Codable { } struct SetLifecycleCallbacksStateRequest: Codable { - var state: Map + var state: [String: Bool] } struct ConfigureRequest: Codable { diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift index f368fc8fa..66ae8b542 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolAppServiceClient.swift @@ -21,13 +21,13 @@ class PatrolAppServiceClient { } func listDartLifecycleCallbacks(completion: @escaping (Result) -> Void) { - performRequest(requestName: "listDartLifecycleCallbacks", completion: completion) + performRequestWithResult(requestName: "listDartLifecycleCallbacks", completion: completion) } func setLifecycleCallbacksState(request: SetLifecycleCallbacksStateRequest, completion: @escaping (Result) -> Void) { do { let body = try JSONEncoder().encode(request) - performRequest(requestName: "setLifecycleCallbacksState", body: body, completion: completion) + performRequestWithResult(requestName: "setLifecycleCallbacksState", body: body, completion: completion) } catch let err { completion(.failure(err)) } diff --git a/packages/patrol/lib/src/native/contracts/contracts.dart b/packages/patrol/lib/src/native/contracts/contracts.dart index 9c4e0101f..3c90d4c5e 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.dart @@ -179,7 +179,7 @@ class SetLifecycleCallbacksStateRequest with EquatableMixin { Map json) => _$SetLifecycleCallbacksStateRequestFromJson(json); - final Map state; + final Map state; Map toJson() => _$SetLifecycleCallbacksStateRequestToJson(this); diff --git a/packages/patrol/lib/src/native/contracts/contracts.g.dart b/packages/patrol/lib/src/native/contracts/contracts.g.dart index 4a5bad73e..7fec943ac 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.g.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.g.dart @@ -92,7 +92,7 @@ const _$RunDartTestResponseResultEnumMap = { SetLifecycleCallbacksStateRequest _$SetLifecycleCallbacksStateRequestFromJson( Map json) => SetLifecycleCallbacksStateRequest( - state: json['state'] as Map, + state: Map.from(json['state'] as Map), ); Map _$SetLifecycleCallbacksStateRequestToJson( From 102f95b46820eea604a03604b43df661d29cccc5 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 10:50:44 +0200 Subject: [PATCH 56/77] copy PATROL_INTEGRATION_TEST_IOS_RUNNER macro to RunnerUITests --- .../example/ios/RunnerUITests/RunnerUITests.m | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index bfc5dbda2..b09632d8d 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -2,4 +2,106 @@ @import patrol; @import ObjectiveC.runtime; +#define PATROL_INTEGRATION_TEST_IOS_RUNNER(__test_class) + @interface __test_class : XCTestCase + @end + +@implementation __test_class + ++(NSArray *)testInvocations { + /* Start native automation server */ + PatrolServer *server = [[PatrolServer alloc] init]; + + NSError *_Nullable __autoreleasing *_Nullable err = NULL; + [server startAndReturnError:err]; + if (err != NULL) { + NSLog(@"patrolServer.start(): failed, err: %@", err); + } + + /* Create a client for PatrolAppService, which lets us list and run Dart tests */ + __block ObjCPatrolAppServiceClient *appServiceClient = [[ObjCPatrolAppServiceClient alloc] init]; + + /* Allow the Local Network permission required by Dart Observatory */ + XCUIApplication *springboard = [[XCUIApplication alloc] initWithBundleIdentifier:@"com.apple.springboard"]; + XCUIElementQuery *systemAlerts = springboard.alerts; + if (systemAlerts.buttons[@"Allow"].exists) { + [systemAlerts.buttons[@"Allow"] tap]; + } + + /* Run the app for the first time to gather Dart tests */ + [[[XCUIApplication alloc] init] launch]; + + /* Spin the runloop waiting until the app reports that it is ready to report Dart tests */ + while (!server.appReady) { + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; + } + + __block NSArray *dartTests = NULL; + [appServiceClient listDartTestsWithCompletion:^(NSArray *_Nullable tests, NSError *_Nullable err) { + if (err != NULL) { + NSLog(@"listDartTests(): failed, err: %@", err); + } + + dartTests = tests; + }]; + + /* Spin the runloop waiting until the app reports the Dart tests it contains */ + while (!dartTests) { + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; + } + + NSLog(@"Got %lu Dart tests: %@", dartTests.count, dartTests); + + NSMutableArray *invocations = [[NSMutableArray alloc] init]; + + /** + * Once Dart tests are available, we: + * + * Step 1. Dynamically add test case methods that request execution of an individual Dart test file. + * + * Step 2. Create invocations to the generated methods and return them + */ + + for (NSString * dartTest in dartTests) { + /* Step 1 - dynamically create test cases */ + + IMP implementation = imp_implementationWithBlock(^(id _self) { + [[[XCUIApplication alloc] init] launch]; + + __block ObjCRunDartTestResponse *response = NULL; + [appServiceClient runDartTestWithName:dartTest + completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { + if (err != NULL) { + NSLog(@"runDartTestWithName(%@): failed, err: %@", dartTest, err); + } + + response = r; + }]; + + /* Wait until Dart test finishes */ + while (!response) { + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; + } + + XCTAssertTrue(response.passed, @"%@", response.details); + }); + NSString *selectorName = [PatrolUtils createMethodNameFromPatrolGeneratedGroup:dartTest]; + SEL selector = NSSelectorFromString(selectorName); + class_addMethod(self, selector, implementation, "v@:"); + + /* Step 2 – create invocations to the dynamically created methods */ + NSMethodSignature *signature = [self instanceMethodSignatureForSelector:selector]; + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.selector = selector; + + NSLog(@"RunnerUITests.testInvocations(): selectorName = %@, signature: %@", selectorName, signature); + + [invocations addObject:invocation]; + } + + return invocations; +} + +@end + PATROL_INTEGRATION_TEST_IOS_RUNNER(RunnerUITests) From bc8a7735424ef5fece6eef864d0ed4f7f7387c47 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 11:15:08 +0200 Subject: [PATCH 57/77] ObjCPatrolAppServiceClient: implement listDartLifecycleCallbacks and setLifecycleCallbacksState --- .../Classes/ObjCPatrolAppServiceClient.swift | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift index 9b2b3290a..147abecf0 100644 --- a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift @@ -45,6 +45,39 @@ } } + @objc public func listDartLifecycleCallbacks( + completion: @escaping ([String]?, [String]?, Error?) -> Void + ) { + NSLog("PatrolAppService.listDartLifecycleCallbacks()") + + client.listDartLifecycleCallbacks { + result in + switch result { + case .success(let result): + completion(result.setUpAlls, result.tearDownAlls, nil) + case .failure(let error): + completion(nil, nil, error) + } + } + } + + @objc public func setLifecycleCallbacksState( + state: [String: Bool], completion: @escaping (Error?) -> Void + ) { + NSLog("PatrolAppService.setLifecycleCallbacksState()") + + let request = SetLifecycleCallbacksStateRequest(state: state) + client.setLifecycleCallbacksState(request: request) { + result in + switch result { + case .success(_): + completion(nil) + case .failure(let error): + completion(error) + } + } + } + @objc public func runDartTest( name: String, completion: @escaping (ObjCRunDartTestResponse?, Error?) -> Void ) { @@ -66,7 +99,6 @@ case .failure(let error): completion(nil, error) } - } } } From dac690750e11e207fef4ccb52621583aa82206dc Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 11:24:15 +0200 Subject: [PATCH 58/77] AutomatorServer: fix compile errors --- .../patrol/ios/Classes/AutomatorServer/AutomatorServer.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift b/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift index 0f67dea3a..b66c4b4a8 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift @@ -296,6 +296,10 @@ func markPatrolAppServiceReady() throws { onAppReady(true) } + + func markLifecycleCallbackExecuted(request: MarkLifecycleCallbackExecutedRequest) throws { + // TODO: Implement + } } #endif From eb2f236fda6ca96337917dc0d97b5689a3713205 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 11:59:50 +0200 Subject: [PATCH 59/77] forward PATROL_INITIAL_RUN to app and expose it on method channel --- .../example/ios/RunnerUITests/RunnerUITests.m | 7 ++++-- .../ios/Classes/SwiftPatrolPlugin.swift | 22 +++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index b09632d8d..a727b1a26 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -29,9 +29,12 @@ @implementation __test_class } /* Run the app for the first time to gather Dart tests */ - [[[XCUIApplication alloc] init] launch]; + XCUIApplication *app = [[XCUIApplication alloc] init]; + NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"true" }; + app.launchEnvironment = args; + [app launch]; - /* Spin the runloop waiting until the app reports that it is ready to report Dart tests */ + /* Spin the runloop waiting until the app reports that PatrolAppService is up */ while (!server.appReady) { [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } diff --git a/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift b/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift index c07ab9da1..48fe6d327 100644 --- a/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift +++ b/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift @@ -2,10 +2,9 @@ import Flutter import UIKit let kChannelName = "pl.leancode.patrol/main" -let kMethodAllTestsFinished = "allTestsFinished" -let kErrorCreateChannelFailed = "create_channel_failed" -let kErrorCreateChannelFailedMsg = "Failed to create GRPC channel" +let kErrorInvalidValue = "invalid value" +let kErrorInvalidValueMsg = "isInitialRun env var is not a bool" /// A Flutter plugin that was responsible for communicating the test results back /// to iOS XCUITest. @@ -23,6 +22,21 @@ public class SwiftPatrolPlugin: NSObject, FlutterPlugin { } public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { - result(FlutterMethodNotImplemented) + switch (call.method) { + case "isInitialRun": + let rawInitialRun = ProcessInfo.processInfo.environment["PATROL_INITIAL_RUN"] + let initialRun = Bool(rawInitialRun ?? "invalid") + if initialRun == nil { + result(FlutterError( + code: kErrorInvalidValue, + message: "PATROL_INITIAL_RUN value is invalid: \(String(describing: rawInitialRun))", + details: nil) + ) + } else { + result(initialRun) + } + default: + result(FlutterMethodNotImplemented) + } } } From 7cb598499feb065e1cbeeb87bf3b30aa24628013 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 13:36:00 +0200 Subject: [PATCH 60/77] ObjCPatrolAppServiceClient.swift: add 1 sec timeout to wait for patrolAppService --- .../Classes/ObjCPatrolAppServiceClient.swift | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift index 147abecf0..916e8376c 100644 --- a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift @@ -66,14 +66,16 @@ ) { NSLog("PatrolAppService.setLifecycleCallbacksState()") - let request = SetLifecycleCallbacksStateRequest(state: state) - client.setLifecycleCallbacksState(request: request) { - result in - switch result { - case .success(_): - completion(nil) - case .failure(let error): - completion(error) + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { + let request = SetLifecycleCallbacksStateRequest(state: state) + self.client.setLifecycleCallbacksState(request: request) { + result in + switch result { + case .success(_): + completion(nil) + case .failure(let error): + completion(error) + } } } } From 5f61d5c14ad642e7e54ec996a3d1826e6d6228c6 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 13:36:16 +0200 Subject: [PATCH 61/77] common.dart: print before setUpAll() - debugging aid --- packages/patrol/lib/src/common.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 9456acaed..75bd89284 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -85,6 +85,7 @@ void patrolSetUpAll(Future Function() body) { return; } + patrolDebug('Waiting for lifecycle callbacks state...'); final callbacksState = await patrolAppService.didReceiveLifecycleCallbacksState; From 289aaaa74a7eb9bab078a9ebe2f12b088b0da203 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 13:37:03 +0200 Subject: [PATCH 62/77] RunnerUITests: call listDartLifecycleCallbacks and setDartLifecycleCallbacksState Unfortunately it's failing for unknown reason (yet) --- .../example/ios/RunnerUITests/RunnerUITests.m | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index a727b1a26..9613ac0f1 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -39,6 +39,20 @@ @implementation __test_class [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } + // MARK: List Dart lifecycle callbacks + + __block NSMutableDictionary *callbacksState = [[NSMutableDictionary alloc] init]; + [appServiceClient + listDartLifecycleCallbacksWithCompletion:^(NSArray * _Nullable setUpAlls, + NSArray * _Nullable tearDownAlls, + NSError * _Nullable err) { + for (NSString* setUpAll in setUpAlls) { + [callbacksState setObject:@NO forKey:setUpAll]; + } + }]; + + // MARK: List Dart tests + __block NSArray *dartTests = NULL; [appServiceClient listDartTestsWithCompletion:^(NSArray *_Nullable tests, NSError *_Nullable err) { if (err != NULL) { @@ -55,6 +69,8 @@ @implementation __test_class NSLog(@"Got %lu Dart tests: %@", dartTests.count, dartTests); + // MARK: Dynamically create test case methods + NSMutableArray *invocations = [[NSMutableArray alloc] init]; /** @@ -69,7 +85,26 @@ @implementation __test_class /* Step 1 - dynamically create test cases */ IMP implementation = imp_implementationWithBlock(^(id _self) { - [[[XCUIApplication alloc] init] launch]; + XCUIApplication *app = [[XCUIApplication alloc] init]; + NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"false" }; + app.launchEnvironment = args; + [app launch]; + + // TODO: wait for patrolAppService to be ready + + __block BOOL callbacksSet = NO; + [appServiceClient setLifecycleCallbacksStateWithState:callbacksState completion:^(NSError * _Nullable err) { + if (err != NULL) { + NSLog(@"setLifecycleCallbacksStateWithState(): failed, err: %@", err); + } + + callbacksSet = YES; + }]; + + /*Wait until lifecycle callbacks are set*/ + while (!callbacksSet) { + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; + } __block ObjCRunDartTestResponse *response = NULL; [appServiceClient runDartTestWithName:dartTest From f571f2b010b4a27a5047c490aa1405509867fa1f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 16:26:40 +0200 Subject: [PATCH 63/77] PatrolJUnitRunner: fix sending lifecycle callback status as String instead of Bool --- .../main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java | 4 ++-- packages/patrol/lib/src/native/patrol_app_service.dart | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java index a085a65cd..91ea9bcef 100644 --- a/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java +++ b/packages/patrol/android/src/main/kotlin/pl/leancode/patrol/PatrolJUnitRunner.java @@ -207,7 +207,7 @@ private Map readStateFile() { InputStream inputStream = TestStorageUtil.getInputStream(stateFileUri, getContentResolver()); String content = convertStreamToString(inputStream); Gson gson = new Gson(); - Type typeOfHashMap = new TypeToken>() {}.getType(); + Type typeOfHashMap = new TypeToken>() {}.getType(); Map data = gson.fromJson(content, typeOfHashMap); return data; } catch (FileNotFoundException e) { @@ -219,7 +219,7 @@ private void writeStateFile(Map data) { try { OutputStream outputStream = TestStorageUtil.getOutputStream(stateFileUri, getContentResolver()); Gson gson = new Gson(); - Type typeOfHashMap = new TypeToken>() {}.getType(); + Type typeOfHashMap = new TypeToken>() {}.getType(); String json = gson.toJson(data, typeOfHashMap); outputStream.write(json.getBytes()); outputStream.write("\n".getBytes()); diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index b32a56725..0222036dd 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -244,12 +244,7 @@ class PatrolAppService extends PatrolAppServiceServer { assert(_didCompleteTestExecution.isCompleted == false); assert(_didReceiveLifecycleCallbacksState.isCompleted == false); - final state = {}; - for (final e in request.state.entries) { - state[e.key as String] = (e.value as String).toLowerCase() == 'true'; - } - - _didReceiveLifecycleCallbacksState.complete(state); + _didReceiveLifecycleCallbacksState.complete(request.state); return Empty(); } } From 00c9178e3d7b0a9c5c7ebe3925c94f92cb6cb262 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 19 Oct 2023 16:27:00 +0200 Subject: [PATCH 64/77] RunnerUITests.m: minor improvements --- .../example/ios/RunnerUITests/RunnerUITests.m | 20 +++++++++++++++---- .../Classes/ObjCPatrolAppServiceClient.swift | 8 +++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index 9613ac0f1..9adb557e4 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -40,17 +40,28 @@ @implementation __test_class } // MARK: List Dart lifecycle callbacks - - __block NSMutableDictionary *callbacksState = [[NSMutableDictionary alloc] init]; + + __block NSMutableDictionary *callbacksState = NULL; [appServiceClient listDartLifecycleCallbacksWithCompletion:^(NSArray * _Nullable setUpAlls, NSArray * _Nullable tearDownAlls, NSError * _Nullable err) { + if (err != NULL) { + NSLog(@"listDartLifecycleCallbacks(): failed, err: %@", err); + } + + callbacksState = [[NSMutableDictionary alloc] init]; for (NSString* setUpAll in setUpAlls) { [callbacksState setObject:@NO forKey:setUpAll]; } }]; + /* Spin the runloop waiting until the app reports the Dart lifecycle callbacks it contains */ + while (!callbacksState) { + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; + } + NSLog(@"Got %lu Dart lifecycle callbacks: %@", callbacksState.count, callbacksState); + // MARK: List Dart tests __block NSArray *dartTests = NULL; @@ -66,7 +77,6 @@ @implementation __test_class while (!dartTests) { [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } - NSLog(@"Got %lu Dart tests: %@", dartTests.count, dartTests); // MARK: Dynamically create test case methods @@ -87,7 +97,7 @@ @implementation __test_class IMP implementation = imp_implementationWithBlock(^(id _self) { XCUIApplication *app = [[XCUIApplication alloc] init]; NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"false" }; - app.launchEnvironment = args; + [app setLaunchEnvironment:args]; [app launch]; // TODO: wait for patrolAppService to be ready @@ -97,6 +107,7 @@ @implementation __test_class if (err != NULL) { NSLog(@"setLifecycleCallbacksStateWithState(): failed, err: %@", err); } + callbacksSet = YES; }]; @@ -123,6 +134,7 @@ @implementation __test_class XCTAssertTrue(response.passed, @"%@", response.details); }); + NSString *selectorName = [PatrolUtils createMethodNameFromPatrolGeneratedGroup:dartTest]; SEL selector = NSSelectorFromString(selectorName); class_addMethod(self, selector, implementation, "v@:"); diff --git a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift index 916e8376c..95d51d487 100644 --- a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift @@ -50,8 +50,7 @@ ) { NSLog("PatrolAppService.listDartLifecycleCallbacks()") - client.listDartLifecycleCallbacks { - result in + client.listDartLifecycleCallbacks { result in switch result { case .success(let result): completion(result.setUpAlls, result.tearDownAlls, nil) @@ -64,12 +63,11 @@ @objc public func setLifecycleCallbacksState( state: [String: Bool], completion: @escaping (Error?) -> Void ) { - NSLog("PatrolAppService.setLifecycleCallbacksState()") + NSLog("PatrolAppService.setLifecycleCallbacksState(\(state)") DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { let request = SetLifecycleCallbacksStateRequest(state: state) - self.client.setLifecycleCallbacksState(request: request) { - result in + self.client.setLifecycleCallbacksState(request: request) { result in switch result { case .success(_): completion(nil) From b77e74bdfdd50dafbe37289d2f8c99811d02cf82 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 23 Oct 2023 11:01:31 +0200 Subject: [PATCH 65/77] update podfile --- packages/patrol/example/ios/Podfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/patrol/example/ios/Podfile.lock b/packages/patrol/example/ios/Podfile.lock index 18e033a3c..5bd9677ea 100644 --- a/packages/patrol/example/ios/Podfile.lock +++ b/packages/patrol/example/ios/Podfile.lock @@ -70,4 +70,4 @@ SPEC CHECKSUMS: PODFILE CHECKSUM: b2bb71756d032256bcb4043384dd40772d5e6a93 -COCOAPODS: 1.12.1 +COCOAPODS: 1.13.0 From ae51ac218d7dd5bf79479224a57300cafce0cf3c Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 23 Oct 2023 12:24:51 +0200 Subject: [PATCH 66/77] logs.dart: format _runKey as 4-digit hex --- packages/patrol/lib/src/logs.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/patrol/lib/src/logs.dart b/packages/patrol/lib/src/logs.dart index cec31afa6..9caabeef5 100644 --- a/packages/patrol/lib/src/logs.dart +++ b/packages/patrol/lib/src/logs.dart @@ -1,6 +1,6 @@ import 'package:meta/meta.dart'; -final _runKey = Object().hashCode; +final _runKey = Object().hashCode.toRadixString(16).padLeft(4, '0'); /// Logs a message with a tag that's constant in the single app process /// instance. From 004fddc4e2b5f43109dd4838e9d8bea8ec75b7df Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 23 Oct 2023 12:28:54 +0200 Subject: [PATCH 67/77] format code in SwiftPatrolPlugin.swift --- .../patrol/ios/Classes/SwiftPatrolPlugin.swift | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift b/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift index 48fe6d327..0e99b7fff 100644 --- a/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift +++ b/packages/patrol/ios/Classes/SwiftPatrolPlugin.swift @@ -22,21 +22,22 @@ public class SwiftPatrolPlugin: NSObject, FlutterPlugin { } public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { - switch (call.method) { - case "isInitialRun": + switch call.method { + case "isInitialRun": let rawInitialRun = ProcessInfo.processInfo.environment["PATROL_INITIAL_RUN"] let initialRun = Bool(rawInitialRun ?? "invalid") if initialRun == nil { - result(FlutterError( - code: kErrorInvalidValue, - message: "PATROL_INITIAL_RUN value is invalid: \(String(describing: rawInitialRun))", - details: nil) + result( + FlutterError( + code: kErrorInvalidValue, + message: "PATROL_INITIAL_RUN value is invalid: \(String(describing: rawInitialRun))", + details: nil) ) } else { result(initialRun) } - default: - result(FlutterMethodNotImplemented) + default: + result(FlutterMethodNotImplemented) } } } From 4cb5a3907cb0e3a1df4e2cf16cbd3828176eea8d Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 23 Oct 2023 12:36:58 +0200 Subject: [PATCH 68/77] implement NativeAutomator.markLifecycleCallbackExecuted --- .../example/ios/RunnerUITests/RunnerUITests.m | 24 ++++++++++++------- .../AutomatorServer/AutomatorServer.swift | 11 +++++++-- .../AutomatorServer/PatrolServer.swift | 17 +++++++++---- .../Classes/ObjCPatrolAppServiceClient.swift | 8 +++---- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index 9adb557e4..a6c7c4b64 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -9,8 +9,15 @@ @implementation __test_class +(NSArray *)testInvocations { + __block NSMutableDictionary *callbacksState = NULL; + __block NSArray *dartTests = NULL; + /* Start native automation server */ - PatrolServer *server = [[PatrolServer alloc] init]; + PatrolServer *server = [[PatrolServer alloc] initOnLifecycleCallbackExecuted:^(NSString * _Nonnull callbackName) { + /* callbacksState dictionary will have been already initialized when this callback is executed */ + NSLog(@"onLifecycleCallbackExecuted for %@", callbackName); + [callbacksState setObject:@YES forKey:callbackName]; + }]; NSError *_Nullable __autoreleasing *_Nullable err = NULL; [server startAndReturnError:err]; @@ -41,7 +48,6 @@ @implementation __test_class // MARK: List Dart lifecycle callbacks - __block NSMutableDictionary *callbacksState = NULL; [appServiceClient listDartLifecycleCallbacksWithCompletion:^(NSArray * _Nullable setUpAlls, NSArray * _Nullable tearDownAlls, @@ -64,7 +70,6 @@ @implementation __test_class // MARK: List Dart tests - __block NSArray *dartTests = NULL; [appServiceClient listDartTestsWithCompletion:^(NSArray *_Nullable tests, NSError *_Nullable err) { if (err != NULL) { NSLog(@"listDartTests(): failed, err: %@", err); @@ -103,12 +108,12 @@ @implementation __test_class // TODO: wait for patrolAppService to be ready __block BOOL callbacksSet = NO; - [appServiceClient setLifecycleCallbacksStateWithState:callbacksState completion:^(NSError * _Nullable err) { + [appServiceClient setDartLifecycleCallbacksState:callbacksState completion:^(NSError * _Nullable err) { if (err != NULL) { - NSLog(@"setLifecycleCallbacksStateWithState(): failed, err: %@", err); + NSLog(@"setDartLifecycleCallbacksState(): failed, err: %@", err); } - + NSLog(@"setDartLifecycleCallbacksState(): succeeded"); callbacksSet = YES; }]; @@ -118,12 +123,13 @@ @implementation __test_class } __block ObjCRunDartTestResponse *response = NULL; - [appServiceClient runDartTestWithName:dartTest - completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { + [appServiceClient runDartTest:dartTest + completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { if (err != NULL) { - NSLog(@"runDartTestWithName(%@): failed, err: %@", dartTest, err); + NSLog(@"runDartTest(%@): failed, err: %@", dartTest, err); } + NSLog(@"runDartTest(%@): succeeded", dartTest); response = r; }]; diff --git a/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift b/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift index b66c4b4a8..ff58b9f2e 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift @@ -7,9 +7,16 @@ private let onAppReady: (Bool) -> Void - init(automator: Automator, onAppReady: @escaping (Bool) -> Void) { + private let onLifecycleCallbackExecuted: (String) -> Void + + init( + automator: Automator, + onAppReady: @escaping (Bool) -> Void, + onLifecycleCallbackExecuted: @escaping (String) -> Void + ) { self.automator = automator self.onAppReady = onAppReady + self.onLifecycleCallbackExecuted = onLifecycleCallbackExecuted } func initialize() throws {} @@ -298,7 +305,7 @@ } func markLifecycleCallbackExecuted(request: MarkLifecycleCallbackExecutedRequest) throws { - // TODO: Implement + onLifecycleCallbackExecuted(request.name) } } diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift index 9c9b28d61..bbac95b1a 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift @@ -15,6 +15,8 @@ import Telegraph @objc public private(set) var appReady = false + private let onLifecycleCallbackExecuted: (String) -> Void + private var passedPort: Int = { guard let portStr = ProcessInfo.processInfo.environment[envPortKey] else { Logger.shared.i("\(envPortKey) is null, falling back to default (\(defaultPort))") @@ -31,7 +33,8 @@ import Telegraph return portInt }() - @objc public override init() { + @objc public init(onLifecycleCallbackExecuted: @escaping (String) -> Void) { + self.onLifecycleCallbackExecuted = onLifecycleCallbackExecuted Logger.shared.i("PatrolServer constructor called") #if PATROL_ENABLED @@ -48,10 +51,14 @@ import Telegraph #if PATROL_ENABLED Logger.shared.i("Starting server...") - let provider = AutomatorServer(automator: automator) { appReady in - Logger.shared.i("App reported that it is ready") - self.appReady = appReady - } + let provider = AutomatorServer( + automator: automator, + onAppReady: { appReady in + Logger.shared.i("App reported that it is ready") + self.appReady = appReady + }, + onLifecycleCallbackExecuted: onLifecycleCallbackExecuted + ) provider.setupRoutes(server: server) diff --git a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift index 95d51d487..b8c7b0b03 100644 --- a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift @@ -60,10 +60,10 @@ } } - @objc public func setLifecycleCallbacksState( - state: [String: Bool], completion: @escaping (Error?) -> Void + @objc public func setDartLifecycleCallbacksState( + _ state: [String: Bool], completion: @escaping (Error?) -> Void ) { - NSLog("PatrolAppService.setLifecycleCallbacksState(\(state)") + NSLog("PatrolAppService.setDartLifecycleCallbacksState(\(state)") DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { let request = SetLifecycleCallbacksStateRequest(state: state) @@ -79,7 +79,7 @@ } @objc public func runDartTest( - name: String, completion: @escaping (ObjCRunDartTestResponse?, Error?) -> Void + _ name: String, completion: @escaping (ObjCRunDartTestResponse?, Error?) -> Void ) { // TODO: simple workaround - patrolAppService starts running too slowly. // We should wait for appReady in the dynamically created test case method, From 6f9e68e945f0ea78f8db855e2562cdfa0a015668 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 23 Oct 2023 14:42:24 +0200 Subject: [PATCH 69/77] remove too much docs --- .../patrol/lib/src/native/patrol_app_service.dart | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/patrol/lib/src/native/patrol_app_service.dart b/packages/patrol/lib/src/native/patrol_app_service.dart index 0222036dd..eb8a83f31 100644 --- a/packages/patrol/lib/src/native/patrol_app_service.dart +++ b/packages/patrol/lib/src/native/patrol_app_service.dart @@ -49,18 +49,6 @@ Future runAppService(PatrolAppService service) async { /// /// This is an internal class and you don't want to use it. It's public so that /// the generated code can access it. -/// -/// PatrolAppService lifecycle during initial run: -/// -/// 1. Initial -/// 2. Has Dart tests -/// 3. Has Dart lifecycle callbacks -/// -/// PatrolAppService lifecycle during test run: -/// -/// 1. Initial -/// 2. Has Dart tests -/// 3. Has Dart test to execute class PatrolAppService extends PatrolAppServiceServer { /// Creates a new [PatrolAppService]. PatrolAppService(); From 60c53d83b5541ccecc42f9ea042aa3d25cf7c2f7 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 23 Oct 2023 14:43:05 +0200 Subject: [PATCH 70/77] update todo --- schema.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.dart b/schema.dart index e469b63b1..23648608f 100644 --- a/schema.dart +++ b/schema.dart @@ -231,7 +231,7 @@ abstract class NativeAutomator { // other void debug(); -// TODO(bartekpacia): Move this RPC into a new PatrolNativeTestService service because it doesn't fit here +// TODO(bartekpacia): Move these RPCc into a new service (PatrolNativeTestService) because it doesn't fit here void markPatrolAppServiceReady(); void markLifecycleCallbackExecuted( MarkLifecycleCallbackExecutedRequest request); From 7e7e9ef971fd2d8a2e00fa3600cafbb2bb6e6b02 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 24 Oct 2023 13:41:44 +0200 Subject: [PATCH 71/77] patrolTest(): remove 2 sec wait --- packages/patrol/lib/src/common.dart | 5 +++-- packages/patrol/lib/src/logs.dart | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 75bd89284..4ed80dc7a 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -189,8 +189,9 @@ void patrolTest( (widgetTester) async { if (!global_state.hotRestartEnabled) { if (await global_state.isInitialRun) { - await Future.delayed(const Duration(seconds: 2)); - patrolDebug('fallthrough test "${global_state.currentTestFullName}"'); + patrolDebug( + 'skippng test "${global_state.currentTestFullName}" because this is the initial run', + ); // Fall through tests during the initial run that discovers tests. // // This is required to be able to find all setUpAll callbacks. diff --git a/packages/patrol/lib/src/logs.dart b/packages/patrol/lib/src/logs.dart index 9caabeef5..474b7d31f 100644 --- a/packages/patrol/lib/src/logs.dart +++ b/packages/patrol/lib/src/logs.dart @@ -8,6 +8,7 @@ final _runKey = Object().hashCode.toRadixString(16).padLeft(4, '0'); /// Helps to differentiate same logs that happen in different app process runs. @internal void patrolDebug(String message) { + // TODO: Enable only if debug dart define is passed // ignore: avoid_print print('PATROL_DEBUG($_runKey): $message'); } From 5ba8427bd541b0079db02cb1532cce395aa610b7 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 24 Oct 2023 13:48:07 +0200 Subject: [PATCH 72/77] callbacks_all_test: remove 2 sec wait --- .../example/integration_test/internal/callbacks_all_test.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart index 8d3d0b7b5..f5b1fa5b3 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -37,6 +37,4 @@ Future _body(PatrolTester $) async { expect($(#counterText).text, '1'); await $(#textField).enterText(testName); - - await $.pumpAndSettle(duration: Duration(seconds: 2)); } From d205d7acd194066e33b404eee840fe7662596f9c Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 24 Oct 2023 20:11:18 +0200 Subject: [PATCH 73/77] fix possible race condition Before, PatrolServer.appReady was set once to true during the initial run, and it stayed like that in subsequent runs. This was a race conditions, and a temporary solution with 1 sec timeout was applied. Now the timeout is removed, and the logic should be fixed. --- .../example/ios/RunnerUITests/RunnerUITests.m | 32 ++++++++---- .../AutomatorServer/PatrolServer.swift | 2 +- .../Classes/ObjCPatrolAppServiceClient.swift | 49 ++++++++----------- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index a6c7c4b64..916c6c219 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -100,20 +100,26 @@ @implementation __test_class /* Step 1 - dynamically create test cases */ IMP implementation = imp_implementationWithBlock(^(id _self) { + /* Reset server's appReady state, because new app process will be started */ + server.appReady = NO; + XCUIApplication *app = [[XCUIApplication alloc] init]; NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"false" }; [app setLaunchEnvironment:args]; [app launch]; - // TODO: wait for patrolAppService to be ready + /* Spin the runloop waiting until the app reports that PatrolAppService is up */ + while (!server.appReady) { + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; + } __block BOOL callbacksSet = NO; [appServiceClient setDartLifecycleCallbacksState:callbacksState completion:^(NSError * _Nullable err) { if (err != NULL) { - NSLog(@"setDartLifecycleCallbacksState(): failed, err: %@", err); + NSLog(@"setDartLifecycleCallbacksState(): call failed, err: %@", err); } - NSLog(@"setDartLifecycleCallbacksState(): succeeded"); + NSLog(@"setDartLifecycleCallbacksState(): call succeeded"); callbacksSet = YES; }]; @@ -123,22 +129,30 @@ @implementation __test_class } __block ObjCRunDartTestResponse *response = NULL; + __block NSError *error; [appServiceClient runDartTest:dartTest completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { + NSString *status; if (err != NULL) { - NSLog(@"runDartTest(%@): failed, err: %@", dartTest, err); + error = err; + status = @"CRASHED"; + } else { + response = r; + status = response.passed ? @"PASSED" : @"FAILED"; } - NSLog(@"runDartTest(%@): succeeded", dartTest); - response = r; + NSLog(@"runDartTest(\"%@\"): call finished, test result: %@", dartTest, status); }]; - /* Wait until Dart test finishes */ - while (!response) { + /* Wait until Dart test finishes (either fails or passes) or crashes */ + while (!response && !error) { [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } - XCTAssertTrue(response.passed, @"%@", response.details); + BOOL passed = response ? response.passed : NO; + NSString *details = response ? response.details : @"(no details - app likely crashed)"; + + XCTAssertTrue(passed, @"%@", details); }); NSString *selectorName = [PatrolUtils createMethodNameFromPatrolGeneratedGroup:dartTest]; diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift index bbac95b1a..c116498f7 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift @@ -13,7 +13,7 @@ import Telegraph #endif @objc - public private(set) var appReady = false + public var appReady = false private let onLifecycleCallbackExecuted: (String) -> Void diff --git a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift index b8c7b0b03..1c5c20c02 100644 --- a/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift +++ b/packages/patrol/ios/Classes/ObjCPatrolAppServiceClient.swift @@ -65,15 +65,13 @@ ) { NSLog("PatrolAppService.setDartLifecycleCallbacksState(\(state)") - DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { - let request = SetLifecycleCallbacksStateRequest(state: state) - self.client.setLifecycleCallbacksState(request: request) { result in - switch result { - case .success(_): - completion(nil) - case .failure(let error): - completion(error) - } + let request = SetLifecycleCallbacksStateRequest(state: state) + self.client.setLifecycleCallbacksState(request: request) { result in + switch result { + case .success(_): + completion(nil) + case .failure(let error): + completion(error) } } } @@ -81,24 +79,19 @@ @objc public func runDartTest( _ name: String, completion: @escaping (ObjCRunDartTestResponse?, Error?) -> Void ) { - // TODO: simple workaround - patrolAppService starts running too slowly. - // We should wait for appReady in the dynamically created test case method, - // before calling runDartTest() (in PATROL_INTEGRATION_TEST_IOS_MACRO) - DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { - NSLog("PatrolAppServiceClient.runDartTest(\(name))") - - let request = RunDartTestRequest(name: name) - self.client.runDartTest(request: request) { result in - switch result { - case .success(let result): - let testRespone = ObjCRunDartTestResponse( - passed: result.result == .success, - details: result.details - ) - completion(testRespone, nil) - case .failure(let error): - completion(nil, error) - } + NSLog("PatrolAppServiceClient.runDartTest(\(format: name))") + + let request = RunDartTestRequest(name: name) + self.client.runDartTest(request: request) { result in + switch result { + case .success(let result): + let testRespone = ObjCRunDartTestResponse( + passed: result.result == .success, + details: result.details + ) + completion(testRespone, nil) + case .failure(let error): + completion(nil, error) } } } @@ -117,7 +110,7 @@ extension DartGroupEntry { // This case is invalid, because every test will have at least // 1 named group - its filename. - fatalError("Invariant violated: test \(test.name) has no named parent group") + fatalError("Invariant violated: test \(format: test.name) has no named parent group") } test.name = "\(parentGroupName) \(test.name)" From aef572cc97fe5e2076e10f98458e86f43c8ed4b1 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 24 Oct 2023 21:17:54 +0200 Subject: [PATCH 74/77] remove PatrolServer.appReady, replace with onAppReady callback --- .../example/ios/RunnerUITests/RunnerUITests.m | 11 +++++---- .../AutomatorServer/AutomatorServer.swift | 14 +++++------ .../AutomatorServer/PatrolServer.swift | 23 ++++++++++++------- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index 916c6c219..4f6a399a6 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -11,9 +11,12 @@ @implementation __test_class +(NSArray *)testInvocations { __block NSMutableDictionary *callbacksState = NULL; __block NSArray *dartTests = NULL; + __block BOOL appReady = NO; /* Start native automation server */ - PatrolServer *server = [[PatrolServer alloc] initOnLifecycleCallbackExecuted:^(NSString * _Nonnull callbackName) { + PatrolServer *server = [[PatrolServer alloc] initWithOnAppReadyCallback:^{ + appReady = YES; + } onDartLifecycleCallbackExecuted:^(NSString * _Nonnull callbackName) { /* callbacksState dictionary will have been already initialized when this callback is executed */ NSLog(@"onLifecycleCallbackExecuted for %@", callbackName); [callbacksState setObject:@YES forKey:callbackName]; @@ -42,7 +45,7 @@ @implementation __test_class [app launch]; /* Spin the runloop waiting until the app reports that PatrolAppService is up */ - while (!server.appReady) { + while (!appReady) { [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } @@ -101,7 +104,7 @@ @implementation __test_class IMP implementation = imp_implementationWithBlock(^(id _self) { /* Reset server's appReady state, because new app process will be started */ - server.appReady = NO; + appReady = NO; XCUIApplication *app = [[XCUIApplication alloc] init]; NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"false" }; @@ -109,7 +112,7 @@ @implementation __test_class [app launch]; /* Spin the runloop waiting until the app reports that PatrolAppService is up */ - while (!server.appReady) { + while (!appReady) { [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } diff --git a/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift b/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift index ff58b9f2e..8c4cd1a36 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/AutomatorServer.swift @@ -5,18 +5,18 @@ final class AutomatorServer: NativeAutomatorServer { private let automator: Automator - private let onAppReady: (Bool) -> Void + private let onAppReady: () -> Void - private let onLifecycleCallbackExecuted: (String) -> Void + private let onDartLifecycleCallbackExecuted: (String) -> Void init( automator: Automator, - onAppReady: @escaping (Bool) -> Void, - onLifecycleCallbackExecuted: @escaping (String) -> Void + onAppReady: @escaping () -> Void, + onDartLifecycleCallbackExecuted: @escaping (String) -> Void ) { self.automator = automator self.onAppReady = onAppReady - self.onLifecycleCallbackExecuted = onLifecycleCallbackExecuted + self.onDartLifecycleCallbackExecuted = onDartLifecycleCallbackExecuted } func initialize() throws {} @@ -301,11 +301,11 @@ } func markPatrolAppServiceReady() throws { - onAppReady(true) + onAppReady() } func markLifecycleCallbackExecuted(request: MarkLifecycleCallbackExecutedRequest) throws { - onLifecycleCallbackExecuted(request.name) + onDartLifecycleCallbackExecuted(request.name) } } diff --git a/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift b/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift index c116498f7..8841ae504 100644 --- a/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift +++ b/packages/patrol/ios/Classes/AutomatorServer/PatrolServer.swift @@ -12,10 +12,9 @@ import Telegraph private let server: Server #endif - @objc - public var appReady = false + private let onAppReady: () -> Void - private let onLifecycleCallbackExecuted: (String) -> Void + private let onDartLifecycleCallbackExecuted: (String) -> Void private var passedPort: Int = { guard let portStr = ProcessInfo.processInfo.environment[envPortKey] else { @@ -33,8 +32,12 @@ import Telegraph return portInt }() - @objc public init(onLifecycleCallbackExecuted: @escaping (String) -> Void) { - self.onLifecycleCallbackExecuted = onLifecycleCallbackExecuted + @objc public init( + withOnAppReadyCallback onAppReady: @escaping () -> Void, + onDartLifecycleCallbackExecuted: @escaping (String) -> Void + ) { + self.onDartLifecycleCallbackExecuted = onDartLifecycleCallbackExecuted + self.onAppReady = onAppReady Logger.shared.i("PatrolServer constructor called") #if PATROL_ENABLED @@ -53,11 +56,15 @@ import Telegraph let provider = AutomatorServer( automator: automator, - onAppReady: { appReady in + onAppReady: { Logger.shared.i("App reported that it is ready") - self.appReady = appReady + self.onAppReady() }, - onLifecycleCallbackExecuted: onLifecycleCallbackExecuted + onDartLifecycleCallbackExecuted: { callback in + Logger.shared.i( + "App reported that Dart lifecycle callback \(format: callback) was executed") + self.onDartLifecycleCallbackExecuted(callback) + } ) provider.setupRoutes(server: server) From 2d2a7e339b06f3fdd44dea6327b95053f1bc4ddf Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 24 Oct 2023 21:25:15 +0200 Subject: [PATCH 75/77] PATROL_INTEGRATION_TEST_IOS_RUNNER: improve comments --- packages/patrol/example/ios/RunnerUITests/RunnerUITests.m | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index 4f6a399a6..1e60fc34a 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -38,6 +38,7 @@ @implementation __test_class [systemAlerts.buttons[@"Allow"] tap]; } + /* MARK: Start initial run */ /* Run the app for the first time to gather Dart tests */ XCUIApplication *app = [[XCUIApplication alloc] init]; NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"true" }; @@ -49,7 +50,7 @@ @implementation __test_class [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; } - // MARK: List Dart lifecycle callbacks + /* MARK: List Dart lifecycle callbacks */ [appServiceClient listDartLifecycleCallbacksWithCompletion:^(NSArray * _Nullable setUpAlls, @@ -71,7 +72,7 @@ @implementation __test_class } NSLog(@"Got %lu Dart lifecycle callbacks: %@", callbacksState.count, callbacksState); - // MARK: List Dart tests + /* MARK: List Dart tests */ [appServiceClient listDartTestsWithCompletion:^(NSArray *_Nullable tests, NSError *_Nullable err) { if (err != NULL) { @@ -87,11 +88,12 @@ @implementation __test_class } NSLog(@"Got %lu Dart tests: %@", dartTests.count, dartTests); - // MARK: Dynamically create test case methods + /* MARK: Create tests at runtime */ NSMutableArray *invocations = [[NSMutableArray alloc] init]; /** + * * Once Dart tests are available, we: * * Step 1. Dynamically add test case methods that request execution of an individual Dart test file. From ae868ab5da0415d589f66b2106a73b76a4063a7f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 24 Oct 2023 21:30:32 +0200 Subject: [PATCH 76/77] move code from RunnerUITests.m to PatrolIntegrationTestRunner.h --- .../example/ios/RunnerUITests/RunnerUITests.m | 177 ------------------ .../ios/Classes/PatrolIntegrationTestRunner.h | 118 ++++++++++-- 2 files changed, 98 insertions(+), 197 deletions(-) diff --git a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m index 1e60fc34a..bfc5dbda2 100644 --- a/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m +++ b/packages/patrol/example/ios/RunnerUITests/RunnerUITests.m @@ -2,181 +2,4 @@ @import patrol; @import ObjectiveC.runtime; -#define PATROL_INTEGRATION_TEST_IOS_RUNNER(__test_class) - @interface __test_class : XCTestCase - @end - -@implementation __test_class - -+(NSArray *)testInvocations { - __block NSMutableDictionary *callbacksState = NULL; - __block NSArray *dartTests = NULL; - __block BOOL appReady = NO; - - /* Start native automation server */ - PatrolServer *server = [[PatrolServer alloc] initWithOnAppReadyCallback:^{ - appReady = YES; - } onDartLifecycleCallbackExecuted:^(NSString * _Nonnull callbackName) { - /* callbacksState dictionary will have been already initialized when this callback is executed */ - NSLog(@"onLifecycleCallbackExecuted for %@", callbackName); - [callbacksState setObject:@YES forKey:callbackName]; - }]; - - NSError *_Nullable __autoreleasing *_Nullable err = NULL; - [server startAndReturnError:err]; - if (err != NULL) { - NSLog(@"patrolServer.start(): failed, err: %@", err); - } - - /* Create a client for PatrolAppService, which lets us list and run Dart tests */ - __block ObjCPatrolAppServiceClient *appServiceClient = [[ObjCPatrolAppServiceClient alloc] init]; - - /* Allow the Local Network permission required by Dart Observatory */ - XCUIApplication *springboard = [[XCUIApplication alloc] initWithBundleIdentifier:@"com.apple.springboard"]; - XCUIElementQuery *systemAlerts = springboard.alerts; - if (systemAlerts.buttons[@"Allow"].exists) { - [systemAlerts.buttons[@"Allow"] tap]; - } - - /* MARK: Start initial run */ - /* Run the app for the first time to gather Dart tests */ - XCUIApplication *app = [[XCUIApplication alloc] init]; - NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"true" }; - app.launchEnvironment = args; - [app launch]; - - /* Spin the runloop waiting until the app reports that PatrolAppService is up */ - while (!appReady) { - [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; - } - - /* MARK: List Dart lifecycle callbacks */ - - [appServiceClient - listDartLifecycleCallbacksWithCompletion:^(NSArray * _Nullable setUpAlls, - NSArray * _Nullable tearDownAlls, - NSError * _Nullable err) { - if (err != NULL) { - NSLog(@"listDartLifecycleCallbacks(): failed, err: %@", err); - } - - callbacksState = [[NSMutableDictionary alloc] init]; - for (NSString* setUpAll in setUpAlls) { - [callbacksState setObject:@NO forKey:setUpAll]; - } - }]; - - /* Spin the runloop waiting until the app reports the Dart lifecycle callbacks it contains */ - while (!callbacksState) { - [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; - } - NSLog(@"Got %lu Dart lifecycle callbacks: %@", callbacksState.count, callbacksState); - - /* MARK: List Dart tests */ - - [appServiceClient listDartTestsWithCompletion:^(NSArray *_Nullable tests, NSError *_Nullable err) { - if (err != NULL) { - NSLog(@"listDartTests(): failed, err: %@", err); - } - - dartTests = tests; - }]; - - /* Spin the runloop waiting until the app reports the Dart tests it contains */ - while (!dartTests) { - [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; - } - NSLog(@"Got %lu Dart tests: %@", dartTests.count, dartTests); - - /* MARK: Create tests at runtime */ - - NSMutableArray *invocations = [[NSMutableArray alloc] init]; - - /** - * - * Once Dart tests are available, we: - * - * Step 1. Dynamically add test case methods that request execution of an individual Dart test file. - * - * Step 2. Create invocations to the generated methods and return them - */ - - for (NSString * dartTest in dartTests) { - /* Step 1 - dynamically create test cases */ - - IMP implementation = imp_implementationWithBlock(^(id _self) { - /* Reset server's appReady state, because new app process will be started */ - appReady = NO; - - XCUIApplication *app = [[XCUIApplication alloc] init]; - NSDictionary *args = @{ @"PATROL_INITIAL_RUN" : @"false" }; - [app setLaunchEnvironment:args]; - [app launch]; - - /* Spin the runloop waiting until the app reports that PatrolAppService is up */ - while (!appReady) { - [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; - } - - __block BOOL callbacksSet = NO; - [appServiceClient setDartLifecycleCallbacksState:callbacksState completion:^(NSError * _Nullable err) { - if (err != NULL) { - NSLog(@"setDartLifecycleCallbacksState(): call failed, err: %@", err); - } - - NSLog(@"setDartLifecycleCallbacksState(): call succeeded"); - callbacksSet = YES; - }]; - - /*Wait until lifecycle callbacks are set*/ - while (!callbacksSet) { - [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; - } - - __block ObjCRunDartTestResponse *response = NULL; - __block NSError *error; - [appServiceClient runDartTest:dartTest - completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { - NSString *status; - if (err != NULL) { - error = err; - status = @"CRASHED"; - } else { - response = r; - status = response.passed ? @"PASSED" : @"FAILED"; - } - - NSLog(@"runDartTest(\"%@\"): call finished, test result: %@", dartTest, status); - }]; - - /* Wait until Dart test finishes (either fails or passes) or crashes */ - while (!response && !error) { - [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; - } - - BOOL passed = response ? response.passed : NO; - NSString *details = response ? response.details : @"(no details - app likely crashed)"; - - XCTAssertTrue(passed, @"%@", details); - }); - - NSString *selectorName = [PatrolUtils createMethodNameFromPatrolGeneratedGroup:dartTest]; - SEL selector = NSSelectorFromString(selectorName); - class_addMethod(self, selector, implementation, "v@:"); - - /* Step 2 – create invocations to the dynamically created methods */ - NSMethodSignature *signature = [self instanceMethodSignatureForSelector:selector]; - NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; - invocation.selector = selector; - - NSLog(@"RunnerUITests.testInvocations(): selectorName = %@, signature: %@", selectorName, signature); - - [invocations addObject:invocation]; - } - - return invocations; -} - -@end - PATROL_INTEGRATION_TEST_IOS_RUNNER(RunnerUITests) diff --git a/packages/patrol/ios/Classes/PatrolIntegrationTestRunner.h b/packages/patrol/ios/Classes/PatrolIntegrationTestRunner.h index d7f6c0b05..172811a02 100644 --- a/packages/patrol/ios/Classes/PatrolIntegrationTestRunner.h +++ b/packages/patrol/ios/Classes/PatrolIntegrationTestRunner.h @@ -1,9 +1,9 @@ // This file is a one giant macro to make the setup as easy as possible for the developer. // To edit it: -// 1. Remove the trailing backslashes: $ sed 's/\\$//' ios/Classes/PatrolIntegrationTestRunner.h +// 1. Remove the trailing backslashes: $ sed 's/\\$//' PatrolIntegrationTestRunner.h // 2. Paste its contents into the RunnerUITests.m in the RunnerUITests target // 3. Make the changes, make sure it works -// 4. Re-add trailing backslashes: $ sed 's/$/\\/' ios/Classes/PatrolIntegrationTestRunner.h +// 4. Re-add trailing backslashes: $ sed 's/$/\\/' RunnerUITests.m // 5. Copy the contents from RunnerUITests.m back here // 6. Go back to using a macro in RunnerTests.m @@ -16,8 +16,20 @@ @implementation __test_class \ \ +(NSArray *)testInvocations { \ + __block NSMutableDictionary *callbacksState = NULL; \ + __block NSArray *dartTests = NULL; \ + __block BOOL appReady = NO; \ + \ /* Start native automation server */ \ - PatrolServer *server = [[PatrolServer alloc] init]; \ + PatrolServer *server = [[PatrolServer alloc] \ + initWithOnAppReadyCallback:^{ \ + appReady = YES; \ + } \ + onDartLifecycleCallbackExecuted:^(NSString *_Nonnull callbackName) { \ + /* callbacksState dictionary will have been already initialized when this callback is executed */ \ + NSLog(@"onLifecycleCallbackExecuted for %@", callbackName); \ + [callbacksState setObject:@YES forKey:callbackName]; \ + }]; \ \ NSError *_Nullable __autoreleasing *_Nullable err = NULL; \ [server startAndReturnError:err]; \ @@ -35,15 +47,41 @@ [systemAlerts.buttons[@"Allow"] tap]; \ } \ \ + /* MARK: Start initial run */ \ /* Run the app for the first time to gather Dart tests */ \ - [[[XCUIApplication alloc] init] launch]; \ + XCUIApplication *app = [[XCUIApplication alloc] init]; \ + NSDictionary *args = @{@"PATROL_INITIAL_RUN" : @"true"}; \ + app.launchEnvironment = args; \ + [app launch]; \ \ - /* Spin the runloop waiting until the app reports that it is ready to report Dart tests */ \ - while (!server.appReady) { \ + /* Spin the runloop waiting until the app reports that PatrolAppService is up */ \ + while (!appReady) { \ [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; \ } \ \ - __block NSArray *dartTests = NULL; \ + /* MARK: List Dart lifecycle callbacks */ \ + \ + [appServiceClient listDartLifecycleCallbacksWithCompletion:^(NSArray *_Nullable setUpAlls, \ + NSArray *_Nullable tearDownAlls, \ + NSError *_Nullable err) { \ + if (err != NULL) { \ + NSLog(@"listDartLifecycleCallbacks(): failed, err: %@", err); \ + } \ + \ + callbacksState = [[NSMutableDictionary alloc] init]; \ + for (NSString * setUpAll in setUpAlls) { \ + [callbacksState setObject:@NO forKey:setUpAll]; \ + } \ + }]; \ + \ + /* Spin the runloop waiting until the app reports the Dart lifecycle callbacks it contains */ \ + while (!callbacksState) { \ + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; \ + } \ + NSLog(@"Got %lu Dart lifecycle callbacks: %@", callbacksState.count, callbacksState); \ + \ + /* MARK: List Dart tests */ \ + \ [appServiceClient listDartTestsWithCompletion:^(NSArray *_Nullable tests, NSError *_Nullable err) { \ if (err != NULL) { \ NSLog(@"listDartTests(): failed, err: %@", err); \ @@ -56,12 +94,14 @@ while (!dartTests) { \ [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; \ } \ - \ NSLog(@"Got %lu Dart tests: %@", dartTests.count, dartTests); \ \ + /* MARK: Create tests at runtime */ \ + \ NSMutableArray *invocations = [[NSMutableArray alloc] init]; \ \ /** \ + * \ * Once Dart tests are available, we: \ * \ * Step 1. Dynamically add test case methods that request execution of an individual Dart test file. \ @@ -73,25 +113,63 @@ /* Step 1 - dynamically create test cases */ \ \ IMP implementation = imp_implementationWithBlock(^(id _self) { \ - [[[XCUIApplication alloc] init] launch]; \ + /* Reset server's appReady state, because new app process will be started */ \ + appReady = NO; \ \ - __block ObjCRunDartTestResponse *response = NULL; \ - [appServiceClient runDartTestWithName:dartTest \ - completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { \ - if (err != NULL) { \ - NSLog(@"runDartTestWithName(%@): failed, err: %@", dartTest, err); \ - } \ + XCUIApplication *app = [[XCUIApplication alloc] init]; \ + NSDictionary *args = @{@"PATROL_INITIAL_RUN" : @"false"}; \ + [app setLaunchEnvironment:args]; \ + [app launch]; \ + \ + /* Spin the runloop waiting until the app reports that PatrolAppService is up */ \ + while (!appReady) { \ + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; \ + } \ \ - response = r; \ - }]; \ + __block BOOL callbacksSet = NO; \ + [appServiceClient \ + setDartLifecycleCallbacksState:callbacksState \ + completion:^(NSError *_Nullable err) { \ + if (err != NULL) { \ + NSLog(@"setDartLifecycleCallbacksState(): call failed, err: %@", err); \ + } \ \ - /* Wait until Dart test finishes */ \ - while (!response) { \ + NSLog(@"setDartLifecycleCallbacksState(): call succeeded"); \ + callbacksSet = YES; \ + }]; \ + \ + /*Wait until lifecycle callbacks are set*/ \ + while (!callbacksSet) { \ [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; \ } \ \ - XCTAssertTrue(response.passed, @"%@", response.details); \ + __block ObjCRunDartTestResponse *response = NULL; \ + __block NSError *error; \ + [appServiceClient runDartTest:dartTest \ + completion:^(ObjCRunDartTestResponse *_Nullable r, NSError *_Nullable err) { \ + NSString *status; \ + if (err != NULL) { \ + error = err; \ + status = @"CRASHED"; \ + } else { \ + response = r; \ + status = response.passed ? @"PASSED" : @"FAILED"; \ + } \ + \ + NSLog(@"runDartTest(\"%@\"): call finished, test result: %@", dartTest, status); \ + }]; \ + \ + /* Wait until Dart test finishes (either fails or passes) or crashes */ \ + while (!response && !error) { \ + [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]]; \ + } \ + \ + BOOL passed = response ? response.passed : NO; \ + NSString *details = response ? response.details : @"(no details - app likely crashed)"; \ + \ + XCTAssertTrue(passed, @"%@", details); \ }); \ + \ NSString *selectorName = [PatrolUtils createMethodNameFromPatrolGeneratedGroup:dartTest]; \ SEL selector = NSSelectorFromString(selectorName); \ class_addMethod(self, selector, implementation, "v@:"); \ From 6a6615eacb17bfb603d5884a4193943dffede109 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 25 Oct 2023 10:03:39 +0200 Subject: [PATCH 77/77] final cleanup --- .../internal/callbacks_all_test.dart | 10 +++++----- .../internal/callbacks_test.dart | 17 +++++++++-------- .../lib/src/native/contracts/contracts.dart | 9 ++++++--- .../contracts/patrol_app_service_server.dart | 6 ++++-- packages/patrol_cli/lib/src/test_runner.dart | 2 +- schema.dart | 2 +- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart index f5b1fa5b3..b5b3654f2 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_all_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_all_test.dart @@ -1,7 +1,5 @@ import 'package:flutter/material.dart'; import 'package:patrol/src/extensions.dart'; -// ignore: implementation_imports -import 'package:patrol/src/logs.dart'; // ignore: depend_on_referenced_packages import 'package:test_api/src/backend/invoker.dart'; @@ -9,16 +7,18 @@ import '../common.dart'; String get currentTest => Invoker.current!.fullCurrentTestName(); +void _print(String text) => print('TEST_DEBUG: $text'); + void main() { group('parent', () { patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - patrolDebug('ran setUpAll (1) before "$currentTest"'); + _print('ran patrolSetUpAll (1) before "$currentTest"'); }); patrolSetUpAll(() async { await Future.delayed(Duration(seconds: 1)); - patrolDebug('ran setUpAll (2) before "$currentTest"'); + _print('ran patrolSetUpAll (2) before "$currentTest"'); }); patrolTest('testA', nativeAutomation: true, _body); @@ -29,7 +29,7 @@ void main() { Future _body(PatrolTester $) async { final testName = Invoker.current!.fullCurrentTestName(); - patrolDebug('ran body of test "$testName"'); + _print('ran body of test "$testName"'); await createApp($); diff --git a/packages/patrol/example/integration_test/internal/callbacks_test.dart b/packages/patrol/example/integration_test/internal/callbacks_test.dart index 43959dbfe..06fc129e8 100644 --- a/packages/patrol/example/integration_test/internal/callbacks_test.dart +++ b/packages/patrol/example/integration_test/internal/callbacks_test.dart @@ -7,31 +7,32 @@ import '../common.dart'; String get currentTest => Invoker.current!.fullCurrentTestName(); -void _print(String text) => print('PATROL_DEBUG: $text'); +void _print(String text) => print('TEST_DEBUG: $text'); void main() { patrolSetUp(() async { await Future.delayed(Duration(seconds: 1)); - _print('setting up before $currentTest'); + _print('ran patrolSetUp (1) up before "$currentTest"'); }); patrolTearDown(() async { await Future.delayed(Duration(seconds: 1)); - _print('tearing down after $currentTest'); + _print('ran patrolTearDown (1) after "$currentTest"'); }); patrolTest('testFirst', nativeAutomation: true, _body); group('groupA', () { patrolSetUp(() async { - if (currentTest == 'callbacks_test groupA testB') { - throw Exception('PATROL_DEBUG: Crashing testB on purpose!'); + if (currentTest == 'internal.callbacks_test groupA testB') { + throw Exception('TEST_DEBUG: "$currentTest" crashed on purpose'); } - _print('setting up before $currentTest'); + + _print('ran patrolSetUp (2) before "$currentTest"'); }); patrolTearDown(() async { - _print('tearing down after $currentTest'); + _print('ran patrolTearDown (2) after "$currentTest"'); }); patrolTest('testA', nativeAutomation: true, _body); @@ -44,7 +45,7 @@ void main() { Future _body(PatrolTester $) async { final testName = Invoker.current!.fullCurrentTestName(); - _print('test body: name=$testName'); + _print('ran body of test "$testName"'); await createApp($); diff --git a/packages/patrol/lib/src/native/contracts/contracts.dart b/packages/patrol/lib/src/native/contracts/contracts.dart index 3c90d4c5e..6bfbb5770 100644 --- a/packages/patrol/lib/src/native/contracts/contracts.dart +++ b/packages/patrol/lib/src/native/contracts/contracts.dart @@ -112,7 +112,8 @@ class ListDartLifecycleCallbacksResponse with EquatableMixin { }); factory ListDartLifecycleCallbacksResponse.fromJson( - Map json) => + Map json, + ) => _$ListDartLifecycleCallbacksResponseFromJson(json); final List setUpAlls; @@ -176,7 +177,8 @@ class SetLifecycleCallbacksStateRequest with EquatableMixin { }); factory SetLifecycleCallbacksStateRequest.fromJson( - Map json) => + Map json, + ) => _$SetLifecycleCallbacksStateRequestFromJson(json); final Map state; @@ -668,7 +670,8 @@ class MarkLifecycleCallbackExecutedRequest with EquatableMixin { }); factory MarkLifecycleCallbackExecutedRequest.fromJson( - Map json) => + Map json, + ) => _$MarkLifecycleCallbackExecutedRequestFromJson(json); final String name; diff --git a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart index 95e5b08d4..55b631b60 100644 --- a/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart +++ b/packages/patrol/lib/src/native/contracts/patrol_app_service_server.dart @@ -27,7 +27,8 @@ abstract class PatrolAppServiceServer { final stringContent = await request.readAsString(utf8); final json = jsonDecode(stringContent); final requestObj = SetLifecycleCallbacksStateRequest.fromJson( - json as Map); + json as Map, + ); final result = await setLifecycleCallbacksState(requestObj); @@ -51,6 +52,7 @@ abstract class PatrolAppServiceServer { Future listDartTests(); Future listDartLifecycleCallbacks(); Future setLifecycleCallbacksState( - SetLifecycleCallbacksStateRequest request); + SetLifecycleCallbacksStateRequest request, + ); Future runDartTest(RunDartTestRequest request); } diff --git a/packages/patrol_cli/lib/src/test_runner.dart b/packages/patrol_cli/lib/src/test_runner.dart index cf96acd44..74e3df8d5 100644 --- a/packages/patrol_cli/lib/src/test_runner.dart +++ b/packages/patrol_cli/lib/src/test_runner.dart @@ -3,7 +3,7 @@ import 'package:equatable/equatable.dart'; import 'package:path/path.dart' show basename; import 'package:patrol_cli/src/devices.dart'; -// TODO(bartekpacia): Lots of this code is not needed after #1004 is done. +// TODO: Lots of this code is not needed after #1004 is done. enum TargetRunStatus { failedToBuild, failedToExecute, passed, canceled } diff --git a/schema.dart b/schema.dart index 23648608f..68ff1d99c 100644 --- a/schema.dart +++ b/schema.dart @@ -231,7 +231,7 @@ abstract class NativeAutomator { // other void debug(); -// TODO(bartekpacia): Move these RPCc into a new service (PatrolNativeTestService) because it doesn't fit here +// TODO: Move these RPCc into a new service (PatrolNativeTestService) because it doesn't fit here void markPatrolAppServiceReady(); void markLifecycleCallbackExecuted( MarkLifecycleCallbackExecutedRequest request);