diff --git a/docs.json b/docs.json index 1792600d2..62e674bcf 100644 --- a/docs.json +++ b/docs.json @@ -1,106 +1,266 @@ { + "$schema": "https://docs.page/schema.json", "name": "Patrol", - "twitter": "leancodepl", - "theme": "#FFA03F", + "description": "Flutter-first UI testing framework", + "theme": { + "primary": "#FFA03F" + }, "favicon": "/assets/favicon-dark.png", - "logo": "/assets/favicon-light.png", - "logoDark": "/assets/favicon-dark.png", + "logo": { + "light": "/assets/favicon-light.png", + "dark": "/assets/favicon-dark.png" + }, "socialPreview": "https://raw.githubusercontent.com/leancodepl/patrol/master/docs/assets/preview.jpg", - "googleAnalytics": "G-2REFSQEXJZ", - "docsearch": { - "appId": "HFSGK31MT8", - "apiKey": "4d41430f0f5928c58db0acfdace04a8b", - "indexName": "patrol-leancode" + "scripts": { + "googleAnalytics": "G-2REFSQEXJZ" + }, + "social": { + "x": "leancodepl", + "github": "leancodepl/patrol", + "discord": "ukBK5t4EZg" + }, + "search": { + "docsearch": { + "appId": "HFSGK31MT8", + "apiKey": "4d41430f0f5928c58db0acfdace04a8b", + "indexName": "patrol-leancode" + } + }, + "content": { + "showPageTitle": false, + "zoomImages": true + }, + "header": { + "showName": true, + "showGitHubCard": true, + "showThemeToggle": true, + "links": [] }, "anchors": [ { - "title": "Join our Discord", - "icon": "discord", - "link": "https://discord.gg/ukBK5t4EZg" + "title": "Discord", + "href": "https://discord.gg/ukBK5t4EZg", + "icon": "discord" }, { "title": "patrol API reference", - "icon": "code", - "link": "https://pub.dev/documentation/patrol/latest/index.html" + "href": "https://pub.dev/documentation/patrol/latest/index.html", + "icon": "code" }, { "title": "patrol_finders API reference", - "icon": "code", - "link": "https://pub.dev/documentation/patrol_finders/latest/index.html" + "href": "https://pub.dev/documentation/patrol_finders/latest/index.html", + "icon": "code" + } + ], + "tabs": [ + { + "id": "root", + "title": "Overview", + "href": "/" + }, + { + "id": "documentation", + "title": "Documentation", + "href": "/documentation" + }, + { + "id": "cli", + "title": "CLI commands", + "href": "/cli-commands" } ], "sidebar": [ - ["Patrol", "/"], - ["Improved logging and reporting is here!", "logs-announcement"], - [ - "Introduction", - [ - ["Getting started", "/getting-started"], - ["Write your first test", "/write-your-first-test"], - ["Logs and test results", "/logs"], - ["Supported platforms", "/supported-platforms"], - ["Compatibility table", "/compatibility-table"] - ] - ], - [ - "Finders", - [ - ["Overview", "/finders/overview"], - ["Usage", "/finders/usage"], - ["Advanced", "/finders/advanced"], - ["Using Patrol finders in widget tests", "/finders/finders-setup"] - ] - ], - [ - "Native automation", - [ - ["Overview", "/native/overview"], - ["Usage", "/native/usage"], - ["Advanced", "/native/advanced"], - ["Feature parity", "/native/feature-parity"] - ] - ], - [ - "CLI commands", - [ - ["build", "/cli-commands/build"], - ["develop", "/cli-commands/develop"], - ["devices", "/cli-commands/devices"], - ["doctor", "/cli-commands/doctor"], - ["test", "/cli-commands/test"], - ["update", "/cli-commands/update"] + { + "tab": "root", + "pages": [ + { + "title": "Getting started", + "href": "/overview/getting-started" + }, + { + "title": "Improved logging and reporting is here!", + "href": "logs-announcement" + }, + { + "title": "LeanCode", + "href": "https://leancode.co" + } ] - ], - [ - "Patrol and CI", - [ - ["Overview", "/ci/overview"], - ["Platforms", "/ci/platforms"], - ["Firebase Test Lab", "/ci/firebase-test-lab"] + }, + { + "group": "Introduction", + "tab": "documentation", + "pages": [ + { + "title": "Install", + "href": "/documentation" + }, + { + "title": "Write your first test", + "href": "/documentation/write-your-first-test" + }, + { + "title": "Logs and test results", + "href": "/documentation/logs" + }, + { + "title": "Cheat sheet", + "href": "/documentation/cheatsheet" + }, + { + "title": "Supported platforms", + "href": "/documentation/supported-platforms" + }, + { + "title": "Compatibility table", + "href": "/documentation/compatibility-table" + } ] - ], - [ - "Integrations", - [ - ["Allure", "/integrations/allure"], - ["Browserstack", "/integrations/browserstack"] + }, + { + "group": "Advanced", + "tab": "documentation", + "pages": [ + { + "group": "Finders", + "tab": "documentation", + "pages": [ + { + "title": "Overview", + "href": "/documentation/finders/overview" + }, + { + "title": "Usage", + "href": "/documentation/finders/usage" + }, + { + "title": "Advanced", + "href": "/documentation/finders/advanced" + }, + { + "title": "Using Patrol finders in widget tests", + "href": "/documentation/finders/finders-setup" + } + ] + }, + { + "group": "Native automation", + "tab": "documentation", + "pages": [ + { + "title": "Overview", + "href": "/documentation/native/overview" + }, + { + "title": "Usage", + "href": "/documentation/native/usage" + }, + { + "title": "Advanced", + "href": "/documentation/native/advanced" + }, + { + "title": "Feature parity", + "href": "/documentation/native/feature-parity" + } + ] + }, + { + "group": "Patrol and CI", + "tab": "documentation", + "pages": [ + { + "title": "Overview", + "href": "/documentation/ci/overview" + }, + { + "title": "Platforms", + "href": "/documentation/ci/platforms" + }, + { + "title": "Firebase Test Lab", + "href": "/documentation/ci/firebase-test-lab" + } + ] + }, + { + "group": "Integrations", + "tab": "documentation", + "pages": [ + { + "title": "Allure", + "href": "/documentation/integrations/allure" + }, + { + "title": "Browserstack", + "href": "/documentation/integrations/browserstack" + } + ] + }, + { + "group": "Other", + "tab": "documentation", + "pages": [ + { + "title": "Effective Patrol", + "href": "/documentation/effective-patrol" + }, + { + "title": "Tips and tricks", + "href": "/documentation/tips-and-tricks" + }, + { + "title": "Debugging Patrol tests", + "href": "/documentation/debugging-patrol-tests" + } + ] + } ] - ], - [ - "Announcements", - [ - ["Patrol 3.0 is here!", "/v3"], - ["New package - patrol_finders", "/patrol-finders-release"] + }, + { + "group": "CLI commands", + "tab": "cli", + "pages": [ + { + "title": "develop", + "href": "/cli-commands" + }, + { + "title": "test", + "href": "/cli-commands/test" + }, + { + "title": "build", + "href": "/cli-commands/build" + }, + { + "title": "doctor", + "href": "/cli-commands/doctor" + }, + { + "title": "update", + "href": "/cli-commands/update" + }, + { + "title": "devices", + "href": "/cli-commands/devices" + } ] - ], - [ - "Other", - [ - ["Effective Patrol", "/effective-patrol"], - ["Tips and tricks", "/tips-and-tricks"], - ["Debugging Patrol tests", "/debugging-patrol-tests"], - ["LeanCode", "https://leancode.co"] + }, + { + "group": "Announcements", + "tab": "root", + "pages": [ + { + "title": "Patrol 3.0 is here!", + "href": "/v3" + }, + { + "title": "New package - patrol_finders", + "href": "/patrol-finders-release" + } ] - ] + } ] -} +} \ No newline at end of file diff --git a/docs/announcements/logs-announcement.mdx b/docs/announcements/logs-announcement.mdx new file mode 100644 index 000000000..8b5be4a65 --- /dev/null +++ b/docs/announcements/logs-announcement.mdx @@ -0,0 +1,14 @@ +--- +title: Improved logging and reporting is here! +--- + +We’ve made some major improvements to how you can monitor and analyze your tests! With Patrol 3.13.0 and later, you’ll get: + + - Verbose logging: Test names are now displayed in real time as they’re executed! + - Detailed step reporting: See every action Patrol takes during your test execution, giving you deeper insights into the process. + - Flutter logs in console: Now you can access Flutter logs directly within the patrol test output, streamlining debugging and analysis. +These enhancements will make it easier than ever to understand what's happening behind the code. + +For a full breakdown of these updates, check out the [Logs and test results][logs] page! + +[logs]: /documentation/logs \ No newline at end of file diff --git a/docs/announcements/patrol-finders-release.mdx b/docs/announcements/patrol-finders-release.mdx new file mode 100644 index 000000000..9557b3d14 --- /dev/null +++ b/docs/announcements/patrol-finders-release.mdx @@ -0,0 +1,27 @@ +--- +title: New package - patrol_finders +--- + +We're introducing [`patrol_finders`] - a new package in Patrol framework! It was +created to make it easier to use Patrol finders in widget tests. + +We decided to separate out our finders mechanism to another package, so +developers who would like to use Patrol's awesome finders in their widget tests +don't need to depend on whole Patrol package. This way you can conveniently use +Patrol finders in widget or golden tests, whichever platforms you need to +support in your project! + +### How to use it? + +We made a short tutorial on how to use `patrol_finders` package separately in +widget tests, you can find it in [Using Patrol finders in widget tests] section. + +### Does this change affect my Patrol tests? + +If you have already some Patrol tests in your project, there are no breaking +changes in this release - everything works the same as before. Though you may +see some deprecation warnings in your code - you can get your code aligned with +them to prepare for future changes. + +[Using Patrol finders in widget tests]: /documentation/finders/finders-setup +[`patrol_finders`]: https://pub.dev/packages/patrol_finders diff --git a/docs/announcements/v3.mdx b/docs/announcements/v3.mdx new file mode 100644 index 000000000..e66d019c0 --- /dev/null +++ b/docs/announcements/v3.mdx @@ -0,0 +1,80 @@ +--- +title: Patrol 3.0 is here +--- + +Patrol 3.0 is the new major version of Patrol. + +## `patrol v3` and DevTools extension + +The highlight of this release is the **Patrol DevTools Extension**. We created +it to enhance your UI test development experience with `patrol develop` by +making it much easier to explore the native view hierarchy. With Patrol's new +DevTools extension, you can effortlessly inspect the currently visible +Android/iOS views and discover their properties. This information can be then +used in native selectors like `$.native.tap()`, eliminating the need for +external tools. + +Patrol is one of the first packages in the whole Flutter ecosystem to have a +DevTools extension. We have started working on it as soon as the Flutter team +has announced that they're working on making DevTools extensible. We immediately +realized how powerful this feature is and how it can enable us to deliver better +UI testing experience. + +This is, of course, just the beginning, and we have plans to introduce more +features in future updates of our DevTools extension. + +### Changes in `patrol` v3: + +The DevTools extension is not the only new feature in this release. Other +changes include: + +- **Minimum Flutter version**: The minimum supported Flutter version has been + bumped to 3.16 to make it compatible with a few breaking changes that were + introduced to the `flutter_test` package that `patrol` and `patrol_finders` + depend on. We hope you'll have an easy time upgrading to 3.16, but if not, you + can always use Patrol v2 until you're ready to upgrade. + +- **A few breaking changes**: + - The `bindingType` parameter has been removed from the `patrolTest()` + function. Now, only `PatrolBinding` is used and it's also automatically + initialized. + - The `nativeAutomation` parameter has also been removed from the + `patrolTest()` function. Now `patrolTest()` implies native automation and + you can use `patrolWidgetTest()` if you don't need it. + - `PatrolTester` class has been renamed to `PatrolIntegrationTester`. Now + `PatrolTester` is used with `patrolWidgetTest()` *without* native automation + and `PatrolIntegrationTester` is used with `patrolTest()` *with* native + automation. +- **Patrol CLI version requirement**: Patrol v3 requires Patrol CLI v2.3.0 or + newer, so make sure to `patrol update`! + +## `patrol_finders` v2 + +Along with `patrol` v3, we are releasing the v2 of [patrol_finders]. In case you +missed it, we split `patrol_finders` from `patrol` a few months ago in response +to our community members who loved Patrol's lean finders syntax, but weren't +interested in developing integration tests. [Here's the docs +page][patrol_finders_docs] about `patrol_finders` in case you missed it. + +### Changes in `patrol_finders` v2: + +- **Minimum Flutter version**: The minimum supported Flutter version of + `patrol_finders` has been bumped to 3.16, just like in `patrol`'s case. +- The deprecated `andSettle` method has been removed from all `PatrolTester` and + `PatrolFinder` methods like `tap()`, `enterText()`, and so on. Developers + should now use `settlePolicy` as a replacement, which has been available since + June. +- The default `settlePolicy` has been changed to [SettlePolicy.trySettle]. + +## Wrapping up + +As you can see, these updates have a little bit of everything - a large new +feature, support for the latest Flutter version, and a clean-up of a few +deprecations. We encourage you to explore our new DevTools extension and look +forward to your feedback and ideas for new features as we continue to evolve the +Patrol ecosystem. Meanwhile, we're getting back to work on Patrol, with a single +goal in mind – to make it the go-to UI testing framework for Flutter apps. + +[patrol_finders]: https://pub.dev/packages/patrol_finders +[patrol_finders_docs]: https://patrol.leancode.co/patrol-finders-release +[SettlePolicy.trySettle]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/SettlePolicy.html#trySettle diff --git a/docs/assets/cheatsheet.png b/docs/assets/cheatsheet.png new file mode 100644 index 000000000..4a961e9a0 Binary files /dev/null and b/docs/assets/cheatsheet.png differ diff --git a/docs/cheatsheet.mdx b/docs/cheatsheet.mdx new file mode 100644 index 000000000..d4733dfd0 --- /dev/null +++ b/docs/cheatsheet.mdx @@ -0,0 +1,3 @@ +--- +redirect: /documentation/cheatsheet +--- diff --git a/docs/ci/firebase-test-lab.mdx b/docs/ci/firebase-test-lab.mdx index 962e31013..6160e804c 100644 --- a/docs/ci/firebase-test-lab.mdx +++ b/docs/ci/firebase-test-lab.mdx @@ -1,134 +1,3 @@ --- -title: CI - Firebase Test Lab +redirect: /documentation/ci/firebase-test-lab --- - -# Firebase Test Lab - -There are many device lab providers. Below we're showing how to run Patrol tests -on [Firebase Test Lab], because it's popular in the Flutter community, but the -instructions should be similar for other device farms, such as [AWS Device -Farm][aws_device_farm]. - - - Before you proceed with the steps listed below, make sure that you've - completed the [native setup] guide. - - - - - To run the integration tests on Android, you need 2 apps: the app itself - (often called the "app under test") and the test intrumentation app. - - To build these apps, run: - - ``` - patrol build android --target integration_test/example_test.dart - ``` - - Once you have built the apks, use the [gcloud] tool to run them on Firebase - Test Lab: - - ``` - gcloud firebase test android run \ - --type instrumentation \ - --use-orchestrator \ - --app build/app/outputs/apk/debug/app-debug.apk \ - --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk \ - --timeout 1m \ - --environment-variables clearPackageData=true \ - --results-bucket= \ - --results-dir= - ``` - - - You must [install the gcloud tool] first. [Here][gcloud_android] you can learn - more about all available options and flags. - - - - The environment variable `clearPackageData=true` tells orchestartor to clear the - package data between test runs. Keep in mind that it clears only the data of your - app, not other data on the device, e.g. Chrome. - - - It's convenient to create a shell script to avoid typing that long command - every time. You might want to take a look at Patrol example app's - [run_android_testlab script][example_android_script]. - - - On Android, all permissions are granted by default. This behavior can be - changed using the [alpha version of the gcloud tool]. - - - - - - To run the integration tests on iOS, you need 2 apps: the app itself - (often called the "app under test") and the test intrumentation app. - - First, build your Flutter app, choosing the integration test file as target: - - ``` - patrol build ios --target integration_test/example_test.dart - ``` - - `patrol build ios` outputs paths to the built app binaries, for example: - - ``` - $ patrol build ios -t integration_test/example_test.dart --release - • Building app with entrypoint example_test.dart for iOS device (release)... - ✓ Completed building app with entrypoint example_test.dart for iOS device (31.5s) - build/ios_integ/Build/Products/Release-iphoneos/Runner.app (app under test) - build/ios_integ/Build/Products/Release-iphoneos/RunnerUITests-Runner.app (test instrumentation app) - build/ios_integ/Build/Products/Runner_iphoneos16.2-arm64.xctestrun (xctestrun file) - ``` - - Firebase Test Lab requires these files to be packaged together in a zip - archive. To create the archive: - - ``` - pushd build/ios_integ/Build/Products - zip -r ios_tests.zip Release-iphoneos Runner_iphoneos16.2-arm64.xctestrun - popd - ``` - - Finally, upload the `ios_tests.zip` to Firebase Test Lab for execution: - - ``` - gcloud firebase test ios run \ - --test build/ios_integ/Build/Products/ios_tests.zip \ - --device model=iphone8,version=16.2,locale=en_US,orientation=portrait - ``` - - - You must [install the gcloud tool] first. [Here][gcloud_ios] you can learn - more about all available options and flags. - - - If your `.xctestrun` file has different iOS version in its name than the - device you're running on, simply rename the `.xctestrun` so that the version - matches. - - It's convenient to create a shell script to avoid typing that long command - every time. You might want to take a look at Patrol example app's - [run_ios_testlab script][example_ios_script]. - - - - -[native setup]: /getting-started -[gcloud]: https://cloud.google.com/sdk/gcloud -[example_android_script]: https://github.com/leancodepl/patrol/blob/master/dev/e2e_app/run_android_testlab -[example_ios_script]: https://github.com/leancodepl/patrol/blob/master/dev/e2e_app/run_ios_testlab -[firebase test lab]: https://firebase.google.com/products/test-lab -[aws_device_farm]: https://aws.amazon.com/device-farm -[install the gcloud tool]: https://cloud.google.com/sdk/docs/install -[gcloud_android]: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -[gcloud_ios]: https://cloud.google.com/sdk/gcloud/reference/firebase/test/ios/run -[alpha version of the gcloud tool]: https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/android/run#--grant-permissions diff --git a/docs/ci/overview.mdx b/docs/ci/overview.mdx index 6b6532fe6..150fb0284 100644 --- a/docs/ci/overview.mdx +++ b/docs/ci/overview.mdx @@ -1,12 +1,3 @@ --- -title: CI - Overview +redirect: /documentation/ci/overview --- - -# Continuous Integration - -This section of the documentation is focused on running Patrol tests as part of -your Continuous Integration workflows. - -Having tests doesn't bring you any benefits if you don't automatically verify -that they pass. We know this too well, and we're putting a lot of work into -making it easy to do so. diff --git a/docs/ci/platforms.mdx b/docs/ci/platforms.mdx index a2d68f310..ee1967aec 100644 --- a/docs/ci/platforms.mdx +++ b/docs/ci/platforms.mdx @@ -1,153 +1,3 @@ --- -title: CI - Platforms +redirect: /documentation/ci/platforms --- - -In this document, we'll outline a few ways to run Patrol UI tests of Flutter -apps. - -Generally, the solutions for running UI tests of mobile apps can be divided into -2 groups: - -- Device labs - platforms that provide access to mobile devices in the cloud. You - upload an app binary with tests to the device lab, which runs the tests and - reports the results back to you. - -- Traditional – containers or VMs, either managed or self-hosted. In this - approach, you get access to the shell, so everything is possible. You manually - script what you want to achieve, which is usually: installing the Android/iOS - SDK tools, creating a virtual device, running tests, and collecting results. - -There are quite a few solutions in each of these groups, and each is unique, but -generally, **device labs trade flexibility for ease of use**. They're a good fit -for most apps but make certain more complicated scenarios impossible. - -# Device labs - -### Firebase Test Lab - -[Firebase Test Lab] is one of the most popular device labs. It is a good choice -for most projects. - -You upload the app main app, the test app, select devices to run on, and after a -while, test results along with a video recording are available. - -Firebase Test Lab has a large pool of physical and virtual devices. - -See also: - -- [Firebase Test Lab pricing] - -### emulator.wtf - -[emulator.wtf] is a fairly new solution created by Madis Pink and Tauno Talimaa. It -claims to provide a 2-5x speedup compared to Firebase Test Lab, and 4-20x -speedup compared to spawning emulators on CI machines. It works similarly to -Firebase Test Lab - you upload your main apk, test apk, select emulators to run -on, and the rest is up to emulator.wtf - it runs the tests and outputs results. - -The emulators are indeed rock stable. Emulator.wtf automatically records videos -from test runs, and it presents the test results nicely. - -It's a solid choice if you can accept that your tests will run only on Android -emulator. - -Reports are available in JUnit. - -See also: - -- [emulator.wtf pricing] - -### Xcode Cloud - -[Xcode Cloud] is a CI/CD platform built into Xcode and designed expressly for -Apple developers. It doesn't support testing on Android. - -Since integration tests written with Patrol are also native `XCTest`s, it should -be possible to run Patrol on Xcode Cloud. We plan to research it soon and share -our findings here. - -### Other - -Another popular device lab is [AWS Device Farm]. - -If your use-case is highly specific, you might want to build an in-house device -farm. A project that helps with this is [Simple Test Farm]. - -### Limitations - -We mentioned above that device labs make certain scenarios impossible to -accomplish. - -An example of such a scenario scanning a QR code. One of the apps we worked on had -this feature, and we wanted to test it because it was a critical part of the user -flow. When you have access to the shell filesystem (which you do have in the -"manual" approach, and don't have in the "device lab" approach), you can easily -[replace the scene that is visible in the camera's viewfinder][so_viewfinder]. - -This is not possible on device labs. - -# Traditional - -### GitHub Actions - -[GitHub Actions] is a very popular CI/CD platform, especially among open-source -projects thanks to unlimited minutes. - -Unfortunately, running Flutter integration tests on GitHub Actions is not a -pleasant experience. - -**Android** - -We used the [ReactiveCircus/android-emulator-runner] GitHub Action to run -Android emulator on GitHub Actions. Our takeaway is this: Running an Android -emulator on the default GitHub Actions runner is a bad idea. It is slow to start and -unstable (apps crash randomly) and very slow. Really, really slow. We tried to -mitigate its instability by using [Test Butler], but it comes with its own -restrictions, most notably, it doesn't allow for Google Play Services. - -**iOS** - -We use the [futureware-tech/simulator-action] GitHub Action to run iOS simulator -on GitHub Actions is stable. But given that the iOS simulator is just that – a -simulator, not an emulator – the range of cases it can be used for is reduced. -For example, there's no easy way to disable an internet connection, which makes it -very hard to test the behavior of an app when offline. - -Bear in mind that to run an iOS simulator on GitHub Actions, you have to use a -macOS runner. 1 minute on macos-latest counts as 10 minutes on ubuntu-latest. -You can also use a custom runner – more on that below. - -Custom Runners Workflows on GitHub Actions can run on external runners, in -addition to default runners such as ubuntu-latest and macos-latest. - -One example of such a custom runner provider is BuildJet. We tried running -Android emulator on it, hoping that the performance benefits it brings would -help with the abysmal stability, but we've found that, even though the emulator -works faster and is more stable, it sometimes just crashes with no actionable -error message. - -### Other - -There are many more CI/CD platforms. Some of the most popular include -[CircleCI], [CirrusCI], and [GitLab CI/CD]. There are also CI providers that are -focused specifically on mobile apps, for example [Bitrise] and [Codemagic]. If -you used these platforms, we (and other Patrol users) will be happy to hear -about your experiences! - -[github actions]: https://github.com/features/actions -[aws device farm]: https://aws.amazon.com/device-farm -[emulator.wtf]: https://emulator.wtf -[emulator.wtf pricing]: https://emulator.wtf/pricing -[firebase test lab]: https://firebase.google.com/docs/test-lab -[firebase test lab pricing]: https://firebase.google.com/docs/test-lab/usage-quotas-pricing -[xcode cloud]: https://developer.apple.com/xcode-cloud -[test butler]: https://github.com/linkedin/test-butler -[reactivecircus/android-emulator-runner]: https://github.com/ReactiveCircus/android-emulator-runner -[futureware-tech/simulator-action]: https://github.com/futureware-tech/simulator-action -[simple test farm]: https://github.com/DeviceFarmer/stf -[so_viewfinder]: https://stackoverflow.com/questions/13818389/android-emulator-camera-custom-image -[circleci]: https://circleci.com -[cirrusci]: https://cirrus-ci.org -[gitlab ci/cd]: https://docs.gitlab.com/ee/ci -[bitrise]: https://bitrise.io -[codemagic]: https://codemagic.io/start diff --git a/docs/cli-commands/build.mdx b/docs/cli-commands/build.mdx index fa31b12f0..86d9f1111 100644 --- a/docs/cli-commands/build.mdx +++ b/docs/cli-commands/build.mdx @@ -2,8 +2,6 @@ title: Commands - build --- -# build - ### Synopsis Build app binaries for integration testing. @@ -17,7 +15,7 @@ To see all available options and flags, run `patrol build android --help` or `patrol build ios --help`. - For `patrol build` to work, you must complete [native setup](/native/setup). + For `patrol build` to work, you must complete [native setup]. ### Description @@ -86,3 +84,4 @@ glance). To learn more, read [the in-depth technical article][patrol_v2_article] explaining the nuts and bolts. [patrol_v2_article]: https://leancode.co/blog/patrol-2-0-improved-flutter-ui-testing +[native setup]: /documentation diff --git a/docs/cli-commands/develop.mdx b/docs/cli-commands/develop.mdx index ad41f7a24..74bdf560e 100644 --- a/docs/cli-commands/develop.mdx +++ b/docs/cli-commands/develop.mdx @@ -1,99 +1,3 @@ --- -title: Commands - develop +redirect: /cli-commands --- - -# develop - -### Synopsis - -Develop integration tests with Hot Restart. - -``` -patrol develop -``` - -To see all available options and flags, run `patrol develop --help`. - -### Description - -`patrol develop` makes the development of integration tests faster and more fun, -thanks to Flutter's Hot Restart feature. - -To run a test file with Hot Restart, specify the `--target` option: - -``` -patrol develop --target integration_test/example_test.dart -``` - -This performs a build of your app, so it's usually slow. - -When the build completes and the app starts, Hot Restart becomes active after a -short while. Once it is active, type **R** to trigger a Hot Restart. - -### Demo - - - -### Caveats - -`patrol develop` is powerful, but it has some limitations. It's important to -understand them to write correct tests. - -Flutter apps consist of 2 parts: the native part and the Flutter part. **Hot -Restart restarts only the Flutter part of your app**. - -When you press **R**, the Flutter part of your app is restarted – your `main()` -function is run again. - -It's important to note that: - -- The native part of your app is not restarted -- The app's data is not cleared -- The app is not uninstalled - -Below are a few common scenarios when state of your app will probably differ -between the first and later Hot Restarts. - -**No support for physical iOS** - -Patrol's Hot Restart is very unreliable when running on physical iOS devices, to -the point that we consider it completely broken. This is unfortunate, but it's a -[bug on the Flutter's side](https://github.com/flutter/flutter/issues/122698). - -**Permissions** - -Once granted, a permissions cannot be revoked. - -You have to work around this by handling both cases (of granted and not granted -permissions) in your tests. - -Removing all permissions at the beginning of each test won't work - both iOS and -Android kill the app when any permission is revoked, and when the app dies, Hot -Restart stops. - -**File system** - -The files your app creates in its internal storage aren't cleared between hot -restarts. - -For example, when some data is saved to SharedPreferences during the first -restart, it will stay around in subsequent restarts (unless it's manually -cleared). - -The same goes for any data the app creates during test - photos, documents, etc. -It's your responsibility to clean them up at the right time to have a stable -environment during `patrol develop`. - -**Native state** - -If your app has some native code that runs only when the app is first run, it -won't be re-executed on hot restarts. - -These cases are quite specific and it's hard to give advice without knowing the -context. - -See also: - -- [The difference between Hot Restart and Hot Reload in Flutter][so_question] - -[so_question]: https://stackoverflow.com/q/61787776/7009800 diff --git a/docs/cli-commands/devices.mdx b/docs/cli-commands/devices.mdx index 281b32a4d..d4c7624cb 100644 --- a/docs/cli-commands/devices.mdx +++ b/docs/cli-commands/devices.mdx @@ -2,8 +2,6 @@ title: Commands - devices --- -# devices - ### Synopsis List attached devices, simulators and emulators diff --git a/docs/cli-commands/doctor.mdx b/docs/cli-commands/doctor.mdx index af2ff59e3..3c94305a0 100644 --- a/docs/cli-commands/doctor.mdx +++ b/docs/cli-commands/doctor.mdx @@ -2,8 +2,6 @@ title: Commands - doctor --- -# doctor - ### Synopsis Show information about installed tooling. diff --git a/docs/cli-commands/index.mdx b/docs/cli-commands/index.mdx new file mode 100644 index 000000000..9e7cfc39d --- /dev/null +++ b/docs/cli-commands/index.mdx @@ -0,0 +1,97 @@ +--- +title: Commands - develop +--- + +### Synopsis + +Develop integration tests with Hot Restart. + +``` +patrol develop +``` + +To see all available options and flags, run `patrol develop --help`. + +### Description + +`patrol develop` makes the development of integration tests faster and more fun, +thanks to Flutter's Hot Restart feature. + +To run a test file with Hot Restart, specify the `--target` option: + +``` +patrol develop --target integration_test/example_test.dart +``` + +This performs a build of your app, so it's usually slow. + +When the build completes and the app starts, Hot Restart becomes active after a +short while. Once it is active, type **R** to trigger a Hot Restart. + +### Demo + + + +### Caveats + +`patrol develop` is powerful, but it has some limitations. It's important to +understand them to write correct tests. + +Flutter apps consist of 2 parts: the native part and the Flutter part. **Hot +Restart restarts only the Flutter part of your app**. + +When you press **R**, the Flutter part of your app is restarted – your `main()` +function is run again. + +It's important to note that: + +- The native part of your app is not restarted +- The app's data is not cleared +- The app is not uninstalled + +Below are a few common scenarios when state of your app will probably differ +between the first and later Hot Restarts. + +**No support for physical iOS** + +Patrol's Hot Restart is very unreliable when running on physical iOS devices, to +the point that we consider it completely broken. This is unfortunate, but it's a +[bug on the Flutter's side](https://github.com/flutter/flutter/issues/122698). + +**Permissions** + +Once granted, a permissions cannot be revoked. + +You have to work around this by handling both cases (of granted and not granted +permissions) in your tests. + +Removing all permissions at the beginning of each test won't work - both iOS and +Android kill the app when any permission is revoked, and when the app dies, Hot +Restart stops. + +**File system** + +The files your app creates in its internal storage aren't cleared between hot +restarts. + +For example, when some data is saved to SharedPreferences during the first +restart, it will stay around in subsequent restarts (unless it's manually +cleared). + +The same goes for any data the app creates during test - photos, documents, etc. +It's your responsibility to clean them up at the right time to have a stable +environment during `patrol develop`. + +**Native state** + +If your app has some native code that runs only when the app is first run, it +won't be re-executed on hot restarts. + +These cases are quite specific and it's hard to give advice without knowing the +context. + +See also: + +- [The difference between Hot Restart and Hot Reload in Flutter][so_question] + +[so_question]: https://stackoverflow.com/q/61787776/7009800 diff --git a/docs/cli-commands/test.mdx b/docs/cli-commands/test.mdx index 676c7cda0..415c757a5 100644 --- a/docs/cli-commands/test.mdx +++ b/docs/cli-commands/test.mdx @@ -2,8 +2,6 @@ title: Commands - test --- -# test - ### Synopsis Run integration tests. diff --git a/docs/cli-commands/update.mdx b/docs/cli-commands/update.mdx index a284ffb71..80a5c62a3 100644 --- a/docs/cli-commands/update.mdx +++ b/docs/cli-commands/update.mdx @@ -2,8 +2,6 @@ title: Commands - update --- -# update - ### Synopsis Update Patrol CLI to the latest version. diff --git a/docs/compatibility-table.mdx b/docs/compatibility-table.mdx index eea27570d..b48121033 100644 --- a/docs/compatibility-table.mdx +++ b/docs/compatibility-table.mdx @@ -1,27 +1,3 @@ --- -title: Compatibility table +redirect: /documentation/compatibility-table --- - -# Compatibility table - -The following table describes which versions of `patrol` -and `patrol_cli` are compatible with each other. -The simplest way to ensure that both packages are compatible -is by always using the latest version. However, -if for some reason that isn't possible, you can refer to -the table below to assess which version you should use. - -| patrol_cli | patrol | Min Flutter Version | -| -------------- | -------------- | ------------------- | -| 3.4.0+ | 3.13.0+ | 3.24.0 | -| 3.3.0 | 3.12.0 | 3.24.0 | -| 3.2.1 | 3.11.2 | 3.24.0 | -| 3.2.0 | 3.11.0 - 3.11.1| 3.22.0 | -| 3.1.0 - 3.1.1 | 3.10.0 | 3.22.0 | -| 2.6.5 - 3.0.1 | 3.6.0 - 3.10.0 | 3.16.0 | -| 2.6.0 - 2.6.4 | 3.4.0 - 3.5.2 | 3.16.0 | -| 2.3.0 - 2.5.0 | 3.0.0 - 3.3.0 | 3.16.0 | -| 2.2.0 - 2.2.2 | 2.3.0 - 2.3.2 | 3.3.0 | -| 2.0.1 - 2.1.5 | 2.0.1 - 2.2.5 | 3.3.0 | -| 2.0.0 | 2.0.0 | 3.3.0 | -| 1.1.4 - 1.1.11 | 1.0.9 - 1.1.11 | 3.3.0 | diff --git a/docs/debugging-patrol-tests.mdx b/docs/debugging-patrol-tests.mdx index 8cd321cee..1ca033f9e 100644 --- a/docs/debugging-patrol-tests.mdx +++ b/docs/debugging-patrol-tests.mdx @@ -1,38 +1,3 @@ --- -title: Debugging Patrol tests +redirect: /documentation/debugging-patrol-tests --- - -If you want to debug your application during patrol tests, -you can do in Visual Studio Code by attaching a debugger to the running process. -Here is how you can do it: - -1. In your `launch.json` file, add a new configuration for attaching debugger to a process: - -```json -{ - "name": "attach debugger", - "request": "attach", - "type": "dart", - "cwd": "integration_test", - "vmServiceUri": "${command:dart.promptForVmService}" -} -``` - -2. Run your patrol tests using `develop` command with the same arguments as you would normally do. - -3. When the tests will start running, at some point you will see a message with the VM service URI. - Copy the URI from the message. - -`The Dart VM service is listening on http://127.0.0.1:63725/57XmBI_pwSA=/` - -4. From "Run and Debug" tab in Visual Studio Code, select the configuration you have created in step 1. - You will be prompted to enter the VM service URI. Paste the URI you copied in step 3. - -5. Once the debugger is attached, you can set breakpoints and debug your application as you would normally do. - - - Intellij/Android Studio does not support attaching a debugger to a running process via Observatory Uri. - Therefore you cannot achieve the same behavior in those IDEs (See this [issue]). - - -[issue]: https://github.com/flutter/flutter-intellij/issues/2250 diff --git a/docs/documentation/cheatsheet.mdx b/docs/documentation/cheatsheet.mdx new file mode 100644 index 000000000..61e9b7f66 --- /dev/null +++ b/docs/documentation/cheatsheet.mdx @@ -0,0 +1,5 @@ +--- +title: Cheat sheet +--- + + \ No newline at end of file diff --git a/docs/documentation/ci/firebase-test-lab.mdx b/docs/documentation/ci/firebase-test-lab.mdx new file mode 100644 index 000000000..3e1e70536 --- /dev/null +++ b/docs/documentation/ci/firebase-test-lab.mdx @@ -0,0 +1,132 @@ +--- +title: CI - Firebase Test Lab +--- + +There are many device lab providers. Below we're showing how to run Patrol tests +on [Firebase Test Lab], because it's popular in the Flutter community, but the +instructions should be similar for other device farms, such as [AWS Device +Farm][aws_device_farm]. + + + Before you proceed with the steps listed below, make sure that you've + completed the [native setup] guide. + + + + + To run the integration tests on Android, you need 2 apps: the app itself + (often called the "app under test") and the test intrumentation app. + + To build these apps, run: + + ``` + patrol build android --target integration_test/example_test.dart + ``` + + Once you have built the apks, use the [gcloud] tool to run them on Firebase + Test Lab: + + ``` + gcloud firebase test android run \ + --type instrumentation \ + --use-orchestrator \ + --app build/app/outputs/apk/debug/app-debug.apk \ + --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk \ + --timeout 1m \ + --device model=MediumPhone.arm,version=34,locale=en,orientation=portrait \ + --record-video \ + --environment-variables clearPackageData=true + ``` + + + You must [install the gcloud tool] first. [Here][gcloud_android] you can learn + more about all available options and flags. + + + + The environment variable `clearPackageData=true` tells orchestartor to clear the + package data between test runs. Keep in mind that it clears only the data of your + app, not other data on the device, e.g. Chrome. + + + It's convenient to create a shell script to avoid typing that long command + every time. You might want to take a look at Patrol example app's + [run_android_testlab script][example_android_script]. + + + On Android, all permissions are granted by default. This behavior can be + changed using the [alpha version of the gcloud tool]. + + + + + + To run the integration tests on iOS, you need 2 apps: the app itself + (often called the "app under test") and the test intrumentation app. + + First, build your Flutter app, choosing the integration test file as target: + + ``` + patrol build ios --target integration_test/example_test.dart + ``` + + `patrol build ios` outputs paths to the built app binaries, for example: + + ``` + $ patrol build ios -t integration_test/example_test.dart --release + • Building app with entrypoint example_test.dart for iOS device (release)... + ✓ Completed building app with entrypoint example_test.dart for iOS device (31.5s) + build/ios_integ/Build/Products/Release-iphoneos/Runner.app (app under test) + build/ios_integ/Build/Products/Release-iphoneos/RunnerUITests-Runner.app (test instrumentation app) + build/ios_integ/Build/Products/Runner_iphoneos16.2-arm64.xctestrun (xctestrun file) + ``` + + Firebase Test Lab requires these files to be packaged together in a zip + archive. To create the archive: + + ``` + pushd build/ios_integ/Build/Products + zip -r ios_tests.zip Release-iphoneos Runner_iphoneos16.2-arm64.xctestrun + popd + ``` + + Finally, upload the `ios_tests.zip` to Firebase Test Lab for execution: + + ``` + gcloud firebase test ios run \ + --test build/ios_integ/Build/Products/ios_tests.zip \ + --device model=iphone8,version=16.2,locale=en_US,orientation=portrait + ``` + + + You must [install the gcloud tool] first. [Here][gcloud_ios] you can learn + more about all available options and flags. + + + If your `.xctestrun` file has different iOS version in its name than the + device you're running on, simply rename the `.xctestrun` so that the version + matches. + + It's convenient to create a shell script to avoid typing that long command + every time. You might want to take a look at Patrol example app's + [run_ios_testlab script][example_ios_script]. + + + + +[native setup]: /documentation +[gcloud]: https://cloud.google.com/sdk/gcloud +[example_android_script]: https://github.com/leancodepl/patrol/blob/master/dev/e2e_app/run_android_testlab +[example_ios_script]: https://github.com/leancodepl/patrol/blob/master/dev/e2e_app/run_ios_testlab +[firebase test lab]: https://firebase.google.com/products/test-lab +[aws_device_farm]: https://aws.amazon.com/device-farm +[install the gcloud tool]: https://cloud.google.com/sdk/docs/install +[gcloud_android]: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +[gcloud_ios]: https://cloud.google.com/sdk/gcloud/reference/firebase/test/ios/run +[alpha version of the gcloud tool]: https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/android/run#--grant-permissions diff --git a/docs/documentation/ci/overview.mdx b/docs/documentation/ci/overview.mdx new file mode 100644 index 000000000..53c02cf40 --- /dev/null +++ b/docs/documentation/ci/overview.mdx @@ -0,0 +1,10 @@ +--- +title: CI - Overview +--- + +This section of the documentation is focused on running Patrol tests as part of +your Continuous Integration workflows. + +Having tests doesn't bring you any benefits if you don't automatically verify +that they pass. We know this too well, and we're putting a lot of work into +making it easy to do so. diff --git a/docs/documentation/ci/platforms.mdx b/docs/documentation/ci/platforms.mdx new file mode 100644 index 000000000..a2d68f310 --- /dev/null +++ b/docs/documentation/ci/platforms.mdx @@ -0,0 +1,153 @@ +--- +title: CI - Platforms +--- + +In this document, we'll outline a few ways to run Patrol UI tests of Flutter +apps. + +Generally, the solutions for running UI tests of mobile apps can be divided into +2 groups: + +- Device labs - platforms that provide access to mobile devices in the cloud. You + upload an app binary with tests to the device lab, which runs the tests and + reports the results back to you. + +- Traditional – containers or VMs, either managed or self-hosted. In this + approach, you get access to the shell, so everything is possible. You manually + script what you want to achieve, which is usually: installing the Android/iOS + SDK tools, creating a virtual device, running tests, and collecting results. + +There are quite a few solutions in each of these groups, and each is unique, but +generally, **device labs trade flexibility for ease of use**. They're a good fit +for most apps but make certain more complicated scenarios impossible. + +# Device labs + +### Firebase Test Lab + +[Firebase Test Lab] is one of the most popular device labs. It is a good choice +for most projects. + +You upload the app main app, the test app, select devices to run on, and after a +while, test results along with a video recording are available. + +Firebase Test Lab has a large pool of physical and virtual devices. + +See also: + +- [Firebase Test Lab pricing] + +### emulator.wtf + +[emulator.wtf] is a fairly new solution created by Madis Pink and Tauno Talimaa. It +claims to provide a 2-5x speedup compared to Firebase Test Lab, and 4-20x +speedup compared to spawning emulators on CI machines. It works similarly to +Firebase Test Lab - you upload your main apk, test apk, select emulators to run +on, and the rest is up to emulator.wtf - it runs the tests and outputs results. + +The emulators are indeed rock stable. Emulator.wtf automatically records videos +from test runs, and it presents the test results nicely. + +It's a solid choice if you can accept that your tests will run only on Android +emulator. + +Reports are available in JUnit. + +See also: + +- [emulator.wtf pricing] + +### Xcode Cloud + +[Xcode Cloud] is a CI/CD platform built into Xcode and designed expressly for +Apple developers. It doesn't support testing on Android. + +Since integration tests written with Patrol are also native `XCTest`s, it should +be possible to run Patrol on Xcode Cloud. We plan to research it soon and share +our findings here. + +### Other + +Another popular device lab is [AWS Device Farm]. + +If your use-case is highly specific, you might want to build an in-house device +farm. A project that helps with this is [Simple Test Farm]. + +### Limitations + +We mentioned above that device labs make certain scenarios impossible to +accomplish. + +An example of such a scenario scanning a QR code. One of the apps we worked on had +this feature, and we wanted to test it because it was a critical part of the user +flow. When you have access to the shell filesystem (which you do have in the +"manual" approach, and don't have in the "device lab" approach), you can easily +[replace the scene that is visible in the camera's viewfinder][so_viewfinder]. + +This is not possible on device labs. + +# Traditional + +### GitHub Actions + +[GitHub Actions] is a very popular CI/CD platform, especially among open-source +projects thanks to unlimited minutes. + +Unfortunately, running Flutter integration tests on GitHub Actions is not a +pleasant experience. + +**Android** + +We used the [ReactiveCircus/android-emulator-runner] GitHub Action to run +Android emulator on GitHub Actions. Our takeaway is this: Running an Android +emulator on the default GitHub Actions runner is a bad idea. It is slow to start and +unstable (apps crash randomly) and very slow. Really, really slow. We tried to +mitigate its instability by using [Test Butler], but it comes with its own +restrictions, most notably, it doesn't allow for Google Play Services. + +**iOS** + +We use the [futureware-tech/simulator-action] GitHub Action to run iOS simulator +on GitHub Actions is stable. But given that the iOS simulator is just that – a +simulator, not an emulator – the range of cases it can be used for is reduced. +For example, there's no easy way to disable an internet connection, which makes it +very hard to test the behavior of an app when offline. + +Bear in mind that to run an iOS simulator on GitHub Actions, you have to use a +macOS runner. 1 minute on macos-latest counts as 10 minutes on ubuntu-latest. +You can also use a custom runner – more on that below. + +Custom Runners Workflows on GitHub Actions can run on external runners, in +addition to default runners such as ubuntu-latest and macos-latest. + +One example of such a custom runner provider is BuildJet. We tried running +Android emulator on it, hoping that the performance benefits it brings would +help with the abysmal stability, but we've found that, even though the emulator +works faster and is more stable, it sometimes just crashes with no actionable +error message. + +### Other + +There are many more CI/CD platforms. Some of the most popular include +[CircleCI], [CirrusCI], and [GitLab CI/CD]. There are also CI providers that are +focused specifically on mobile apps, for example [Bitrise] and [Codemagic]. If +you used these platforms, we (and other Patrol users) will be happy to hear +about your experiences! + +[github actions]: https://github.com/features/actions +[aws device farm]: https://aws.amazon.com/device-farm +[emulator.wtf]: https://emulator.wtf +[emulator.wtf pricing]: https://emulator.wtf/pricing +[firebase test lab]: https://firebase.google.com/docs/test-lab +[firebase test lab pricing]: https://firebase.google.com/docs/test-lab/usage-quotas-pricing +[xcode cloud]: https://developer.apple.com/xcode-cloud +[test butler]: https://github.com/linkedin/test-butler +[reactivecircus/android-emulator-runner]: https://github.com/ReactiveCircus/android-emulator-runner +[futureware-tech/simulator-action]: https://github.com/futureware-tech/simulator-action +[simple test farm]: https://github.com/DeviceFarmer/stf +[so_viewfinder]: https://stackoverflow.com/questions/13818389/android-emulator-camera-custom-image +[circleci]: https://circleci.com +[cirrusci]: https://cirrus-ci.org +[gitlab ci/cd]: https://docs.gitlab.com/ee/ci +[bitrise]: https://bitrise.io +[codemagic]: https://codemagic.io/start diff --git a/docs/documentation/compatibility-table.mdx b/docs/documentation/compatibility-table.mdx new file mode 100644 index 000000000..8786655a4 --- /dev/null +++ b/docs/documentation/compatibility-table.mdx @@ -0,0 +1,25 @@ +--- +title: Compatibility table +--- + +The following table describes which versions of `patrol` +and `patrol_cli` are compatible with each other. +The simplest way to ensure that both packages are compatible +is by always using the latest version. However, +if for some reason that isn't possible, you can refer to +the table below to assess which version you should use. + +| patrol_cli | patrol | Min Flutter Version | +| -------------- | -------------- | ------------------- | +| 3.4.0+ | 3.13.0+ | 3.24.0 | +| 3.3.0 | 3.12.0 | 3.24.0 | +| 3.2.1 | 3.11.2 | 3.24.0 | +| 3.2.0 | 3.11.0 - 3.11.1| 3.22.0 | +| 3.1.0 - 3.1.1 | 3.10.0 | 3.22.0 | +| 2.6.5 - 3.0.1 | 3.6.0 - 3.10.0 | 3.16.0 | +| 2.6.0 - 2.6.4 | 3.4.0 - 3.5.2 | 3.16.0 | +| 2.3.0 - 2.5.0 | 3.0.0 - 3.3.0 | 3.16.0 | +| 2.2.0 - 2.2.2 | 2.3.0 - 2.3.2 | 3.3.0 | +| 2.0.1 - 2.1.5 | 2.0.1 - 2.2.5 | 3.3.0 | +| 2.0.0 | 2.0.0 | 3.3.0 | +| 1.1.4 - 1.1.11 | 1.0.9 - 1.1.11 | 3.3.0 | diff --git a/docs/documentation/debugging-patrol-tests.mdx b/docs/documentation/debugging-patrol-tests.mdx new file mode 100644 index 000000000..8cd321cee --- /dev/null +++ b/docs/documentation/debugging-patrol-tests.mdx @@ -0,0 +1,38 @@ +--- +title: Debugging Patrol tests +--- + +If you want to debug your application during patrol tests, +you can do in Visual Studio Code by attaching a debugger to the running process. +Here is how you can do it: + +1. In your `launch.json` file, add a new configuration for attaching debugger to a process: + +```json +{ + "name": "attach debugger", + "request": "attach", + "type": "dart", + "cwd": "integration_test", + "vmServiceUri": "${command:dart.promptForVmService}" +} +``` + +2. Run your patrol tests using `develop` command with the same arguments as you would normally do. + +3. When the tests will start running, at some point you will see a message with the VM service URI. + Copy the URI from the message. + +`The Dart VM service is listening on http://127.0.0.1:63725/57XmBI_pwSA=/` + +4. From "Run and Debug" tab in Visual Studio Code, select the configuration you have created in step 1. + You will be prompted to enter the VM service URI. Paste the URI you copied in step 3. + +5. Once the debugger is attached, you can set breakpoints and debug your application as you would normally do. + + + Intellij/Android Studio does not support attaching a debugger to a running process via Observatory Uri. + Therefore you cannot achieve the same behavior in those IDEs (See this [issue]). + + +[issue]: https://github.com/flutter/flutter-intellij/issues/2250 diff --git a/docs/documentation/effective-patrol.mdx b/docs/documentation/effective-patrol.mdx new file mode 100644 index 000000000..0ce72279a --- /dev/null +++ b/docs/documentation/effective-patrol.mdx @@ -0,0 +1,190 @@ +--- +title: Effective Patrol +--- + +Over the past months, we've written many Patrol tests and often learned the hard +way what works well and what doesn't. We're sharing our findings hoping that +they'll help you write robust tests. + +This document follows [RFC 2119][rfc2119]. + +### PREFER using keys to find widgets + +Patrol's custom finders are very powerful, and you might often be inclined to +find the widget you want in a variety of ways. While we're encouraging you to +explore and play with Patrol's custom finders, we are quite confident that keys +are the best way to find widgets. + +**Why not strings?** + +At first, strings might seem like a good way to find widgets. + +They'll get increasingly annoying to work with as your app grows and changes, +for example, when the strings in your app change. + +Using strings stops making any sense when you have more than 1 language in your +app. Using strings in such case is asking for trouble. + +**Why not classes?** + +There are 2 problems with using classes. + +First is that they hurt your test's readability. You want to tap on _the_ login +button or enter text into _the_ username field. You don't want to tap on, say, +the third button and enter text into the second text field. + +The second problem is that classes are almost always an implementation detail. +As a tester, you shouldn't care if something is a `TextButton` or an +`OutlineButton`. You care that it is _the_ login button, and you want to tap on +it. In most cases, that login button should have a key. + +Let's consider this simple example: + +```dart +await $(LoginForm).$(Button).at(1).tap(); // taps on the second ("login") button +``` + +This works, but the code is not very self-explanatory. To make it understandable +at glance, you had to add a comment. + +But if you assigned a key to the login button, the above could be simplified to: + +```dart +await $(#loginButton).tap(); +``` + +Much better! + +Let's see another example: + +```dart +await $(Select).tap(); // taps on the first Select +``` + +If the type parameter is changed from `String` to, for example, some specialized +`PersonData` model, that finder won't find anything. You'd have to update it to: + +```dart +await $(Select).tap(); +``` + +You had to change your test, even though nothing changed from the user's +perspective. This is usually a sign that you rely too much on classes to find +widgets. + +This whole section could be summed up to the simple maxim: + +> Have tester's mindset. + +Treat your finders as if they were the tester's eyes. + +### CONSIDER having a file where all keys are defined + +The number of keys will get bigger as your app grows and you write more tests. +To keep track of them, it's a good idea to keep all keys in, say, +`lib/keys.dart` file. + +```dart title="lib/keys.dart" +import 'package:flutter/foundation.dart'; + +typedef K = Keys; + +class Keys { + const Keys(); + + static const usernameTextField = Key('usernameTextField'); + static const passwordTextField = Key('passwordTextField'); + static const loginButton = Key('loginButton'); + static const forgotPasswordButton = Key('forgotPasswordButton'); + static const privacyPolicyLink = Key('privacyPolicyLink'); +} +``` + +Then you can use it in your app's and tests' code: + +```dart title="In app UI code" +@override +Widget build(BuildContext context) { + return Column( + children: [ + /// some widgets + TextField( + key: K.usernameTextField, + // some other TextField properties + ), + // more widgets + ], + ); +} +``` + +```dart title="In app test code" +void main() { + patrolTest('logs in', (PatrolIntegrationTester $) { + // some code + await $(K.usernameTextField).enterText('CoolGuy'); + // more code + }); +} +``` + +This is a good way to make sure that the same keys are used in app and tests. No +more typos! + +### PREFER having one test path + +Good tests test one feature, and test it well (this applies to all tests, not +only Patrol tests). This is often called the "main path". Try to introduce as +little condional logic as possible to help keep the main path straight. In +practice, this usually comes down to having as few `if`s as possible. + +Keeping your test code simple and to the point will also help you in debugging +it. + +### DO add a good test description explaining the test's purpose + +If your app is non-trivial, your Patrol test will become long pretty quickly. +You may be sure now that you'll always remember what the 200 line long test +you've just written does and are (rightfully) very proud of it. + +Believe us, in 3 months you will not remember what your test does. This is why +the first argument to `patrolTest` is the test description. Use it well! + +```dart +// GOOD +import 'package:awesome_app/main.dart'; +import 'package:patrol/patrol.dart'; + +void main() { + patrolTest( + 'signs up for the newsletter and receives a reward', + ($) async { + await $.pumpWidgetAndSettle(AwesomeApp()); + + await $(#phoneNumber).enterText('800-555-0199'); + await $(#loginButton).tap(); + + // more code + }, + ); +} +``` + +```dart +// BAD +void main() { + patrolTest( + 'test', + ($) async { + await $.pumpWidgetAndSettle(AwesomeApp()); + + await $(#phoneNumber).enterText('800-555-0199'); + await $(#loginButton).tap(); + + // more code + }, + ); +} +``` + +[rfc2119]: https://www.ietf.org/rfc/rfc2119.txt diff --git a/docs/documentation/finders/advanced.mdx b/docs/documentation/finders/advanced.mdx new file mode 100644 index 000000000..b99e58a9e --- /dev/null +++ b/docs/documentation/finders/advanced.mdx @@ -0,0 +1,231 @@ +--- +title: Patrol finders - advanced +--- + +We aim to make Patrol as simple as possible, but there are still a few matters +that we feel require some more attention. We'll explain them in this section. + +### How is Patrol's `tap()` different from Flutter's `tap()`? + +Let's consider this test, written without Patrol: + +```dart +await tester.tap(find.byKey(Key('addComment')).first); +await tester.pumpAndSettle(); +``` + +This code: + +1. Immediately atttempts to find the first widget with the `addComment` key +2. After finding the widget, it immediately attempts to tap on it + +This is the default behavior, but in our experience, it's often a source of +flakiness. For example, the widget having `addComment` key might not be visible +at the time when the finder is run. This usually doesn't means that the test +should fail. Probably an HTTP request was made to fetch the post, and when the +fetching is done, the widget having `addComment` key will show up. + +To achieve this behavior, you'd have to do: + +```dart +while (find.byKey(Key('addComment')).first.evaluate().isEmpty) { + await tester.pump(Duration(milliseconds: 100)); +} + +await tester.tap(find.byKey(Key('addComment')).first); +await tester.pumpAndSettle(); +``` + +Our tiny example got really big, but it's still got two problems. + +1. If something goes wrong and `addComment` never shows up, we'll keep waiting + indefinitely. + +2. The widget with `addComment` key might be present in the widget tree, but + still not be visible to the user. By default, Flutter's default + `WidgetTester` doesn't care. This is almost never desirable. + +Fortunately, you don't have to overcome these problems. Patrol already did it! + +Below is the same test, with all the above problems fixed, written with Patrol's +custom finders: + +```dart +await $(#addComment).tap(); +``` + +This code: + +1. Attempts to find the first widget with `addComment` that is visible on + screen. If it's not found immediately, it keeps trying until it finds it, or + throws an exception if timeout. +2. Taps on it. + +The timeout can be configured globally: + +```dart +patrolWidgetTest( + 'logs in successfully', + config: PatrolTestConfig(findTimeout: Duration(seconds: 10)), + ($) async { + // your test code + }, +); +``` + +You can also change the timeout ad-hoc: + +```dart +await $(#addComment).tap(findTimeout: Duration(seconds: 30)); +``` + +### You gotta pump it up! But which one to use? + +In Flutter, "pumping" means rendering frames to the screen. + +If there are no frames to pump, no animations are pending, which usually means +that the next action during the test can be executed. It is an equivalent of +what a human tester would do while testing an app - they would wait until the +app's state stabilizes after they've done something. For example, they tap on a +button and get redirected to another screen, but the data that will be shown +there hasn't been loaded yet. In such a case, a human tester waits until a +loader (or other animation) finishes. Pumping mechanism does exactly this - it +renders consecutive frames on the screen. For how long, exactly? Usually, we +want to pump frames as long as they come. That's what [pumpAndSettle()] does. + +`pumpAndSettle()` method is called by default inside all actions that can be +performed while testing - tapping, scrolling, entering text, and so on. You can +change that by setting the `settlePolicy` argument: + +```dart +await $('Delete account').tap(settlePolicy: SettlePolicy.settle); +await $('Confirm').tap(settlePolicy: SettlePolicy.pump); +``` + +`SettlePolicy` is an `enum` with 3 values. The default is `SettlePolicy.settle` +but you can change it to `pump` or `trySettle`. Those values map to methods like this: + - `noSettle` -> `pump()`, + - `trySettle` -> `pumpAndTrySettle()`, + - `settle` -> `pumpAndSettle()`. + +While `settle` and `pump` simply refer to Flutter's built-in methods, +`trySettle` is available only in Patrol. How is it different from other ones? + +`pumpAndTrySettle()` is pretty much like `pumpAndSettle()`, the only difference +is that `pumpAndSettle()` throws an exception, if there were still new frames to +render after sonme defined timeout, while `pumpAndTrySettle()` does not. That's +why it has "`try`" in it's name. + +When to use this new pumping method? Let us picture a scenario, in which we have +to deal with some animations. Let's say, that your app has some endless +animations, e.g. on a homescreen, to keep user's attention. You'd like to wait +for some things to happen, but using `pumpAndSettle`, you'll keep getting an +exception, because after some time, defined by `timeout`, there will be still +new frames to render. On the other hand, you still want to pump frames for some +time - if you didn't, the screen you want to interact with might not be rendered +yet, or it would have some widgets missing or data not yet loaded. + +So, we decided to add a way to try settle - pump frames for some time (10 +seconds by default), but if after that time there is still something new to +render - do nothing and continue the test. + +We recommend using `pumpAndTrySettle()`, because it works with both kinds of +animations - finite and infinite. This settle policy will be new default in +future Patrol releases. + +### How does `scrollTo()` work? + +The `scrollTo()` method is simple to use, yet very powerful. Here's how you use +it to scroll to and tap on the first widget with the `"Delete account"` text: + +```dart +await $('Delete account').scrollTo().tap(); +``` + +And here's how `scrollTo()` works: + +1. Waits for at least 1 [Scrollable] widget (or whatever you provided in + `view` argument) to become visible +2. Scrolls this widget in its scrolling direction until the target + widget becomes visible +3. If the target widget becomes visible within timeout, it finishes, otherwise + it throws an exception + +Most of the time, you use `scrollTo()` and it just works, but there's 1 +important thing to keep in mind when using `scrollTo()`: + +**`scrollTo()`, by default, scrolls the first `Scrollable` widget** + +This default is reasonable and what you want most of the time. Unfortunately, +this behavior can sometimes cause problems in more complicated UIs, where more +than a single `Scrollable` widget is visible at the same time. In such cases we +strongly recommend explicitly specifying the `view` that `scrollTo()` +should scroll, to avoid the problem of the target widget never becoming visible +because the wrong widget was scrolled. + +To demonstrate this problem, let's consider this very simple app: + +```dart +class App extends StatelessWidget { + App({super.key}); + + @override + Widget build(BuildContext context) { + return MaterialApp( + home: Scaffold( + body: Column( + children: [ + Expanded(child: ListView(key: Key('listView1'))), + Expanded( + child: ListView.builder( + key: Key('listView2'), + itemCount: 101, + itemBuilder: (context, index) => Text('index: $index'), + ), + ), + ], + ), + ), + ); + } +} +``` + +Now let's say that you're writing a test and want to scroll to and tap on the +first widget with the `"index: 100"` text (that is the last `Text` widget built +by the second `ListView` widget): + +There's a high chance that you'd write this: + +```dart +await $('index: 100').scrollTo().tap(); +``` + +Unfortunately, running this test gives the `pumpAndSettle timed out` error. +That's because the `scrollTo()` was trying to scroll the first visible +`Scrollable` widget, which happens to be the first `ListView` (the one with +`listView1` key and no children). + +To fix this problem, you have to explicitly specify which `Scrollable` you want +to use: + +```dart +await $('index: 100').scrollTo(view: $(#listView2).$(Scrollable)).tap(); +``` + +The above snippet will scroll the second `Scrollable` and find the widget with +`"index: 100"` text. + +**Why so verbose?** + +You might be wondering why `scrollTo(view: $(#listView2))` is not enough? +Why is it needed to look for a `Scrollable` widget inside the widget with the +`listView2` key? + +This is because the [ListView] widget doesn't extend [Scrollable] – instead, it +builds a subclass of [Scrollable] itself. [This is a known Flutter +problem](https://github.com/flutter/flutter/issues/88762). + +[listview]: https://api.flutter.dev/flutter/widgets/ListView-class.html +[scrollable]: https://api.flutter.dev/flutter/widgets/Scrollable-class.html +[pumpAndSettle()]: https://api.flutter.dev/flutter/flutter_test/WidgetTester/pumpAndSettle.html diff --git a/docs/documentation/finders/finders-setup.mdx b/docs/documentation/finders/finders-setup.mdx new file mode 100644 index 000000000..760777d3a --- /dev/null +++ b/docs/documentation/finders/finders-setup.mdx @@ -0,0 +1,83 @@ +--- +title: Using Patrol finders in widget tests +--- + +Since `patrol_finders` is a separate package, referenced in `patrol` package, +you can use it in your widget or golden tests, without depending on `patrol`. +Below you can find a short tutorial on how to use `$` in already existing widget tests. + +### Install + +First, add the `patrol_finders` package as a `dev_dependency` in your app's +`pubspec.yaml`. You can do this by executing the following command in the app's +directory: + +```console +flutter pub add patrol_finders --dev +``` + +### Use + + + Our custom finders depend only on Flutter itself. This means you can use them + no matter if you're writing a mobile, web, desktop, or embedded app! + + +To use Patrol finders in your test files, first import it: + +```dart +import 'package:patrol_finders/patrol_finders.dart'; +``` + +Once imported, you can use it in widget tests: + +```dart title="test/widget_test_with_patrol_finders.dart" +void main() { + testWidgets( + 'counter is incremented when plus button is tapped', + (WidgetTester tester) async { + PatrolTester $ = PatrolTester( + tester: tester, + config: PatrolTesterConfig(), + ); + await $.pumpWidget(const MyApp()); + + expect($('0'), findsOneWidget); + expect($('-1'), findsNothing); + + await $(Icons.remove).tap(); + + expect($('0'), findsNothing); + expect($('-1'), findsOneWidget); + }, + ); +} +``` + +Or you can use our wrapper on `testWidgets` method, which initializes +`PatrolTester` object for you. + +```dart title="test/patrol_widget_test.dart" +void main() { + patrolWidgetTest( + 'counter is incremented when plus button is tapped', + (PatrolTester $) async { + await $.pumpWidget(const MyApp()); + + expect($('0'), findsOneWidget); + expect($('-1'), findsNothing); + + await $(Icons.remove).tap(); + + expect($('0'), findsNothing); + expect($('-1'), findsOneWidget); + }, + ); +} +``` + +To run the test, simply execute: + +```console +flutter test +``` diff --git a/docs/documentation/finders/overview.mdx b/docs/documentation/finders/overview.mdx new file mode 100644 index 000000000..7b308cab7 --- /dev/null +++ b/docs/documentation/finders/overview.mdx @@ -0,0 +1,58 @@ +--- +title: Patrol finders - overview +--- + +Flutter's finders are powerful, but not very intuitive to use. + +We took them and made something awesome. + +Thanks to Patrol's custom finders, you'll take your tests from this: + +```dart +testWidgets('signs up', (WidgetTester tester) async { + await tester.pumpWidget(AwesomeApp()); + await tester.pumpAndSettle(); + + await tester.enterText( + find.byKey(Key('emailTextField')), + 'charlie@root.me', + ); + await tester.pumpAndSettle(); + + await tester.enterText( + find.byKey(Key('nameTextField')), + 'Charlie', + ); + await tester.pumpAndSettle(); + + await tester.enterText( + find.byKey(Key('passwordTextField')), + 'ny4ncat', + ); + await tester.pumpAndSettle(); + + await tester.tap(find.byKey(Key('termsCheckbox'))); + await tester.pumpAndSettle(); + + await tester.tap(find.byKey(Key('signUpButton'))); + await tester.pumpAndSettle(); + + expect(find.text('Welcome, Charlie!'), findsOneWidget); +}); +``` + +to this: + +```dart +patrolWidgetTest('signs up', (PatrolTester $) async { + await $.pumpWidgetAndSettle(AwesomeApp()); + + await $(#emailTextField).enterText('charlie@root.me'); + await $(#nameTextField).enterText('Charlie'); + await $(#passwordTextField).enterText('ny4ncat'); + await $(#termsCheckbox).tap(); + await $(#signUpButton).tap(); + + await $('Welcome, Charlie!').waitUntilVisible(); +}); +``` diff --git a/docs/documentation/finders/usage.mdx b/docs/documentation/finders/usage.mdx new file mode 100644 index 000000000..2e38e0288 --- /dev/null +++ b/docs/documentation/finders/usage.mdx @@ -0,0 +1,277 @@ +--- +title: Patrol finders - usage +--- + +This page introduces Patrol finder system. Let's get our hands dirty +and find some widgets! + +### Finding widgets + +Let's say you want to find some `Text` widget – nothing easier than that! + +```dart +find.byType(Text); +``` + +Using Patrol finder, you'd write the above as: + +```dart +$(Text); +``` + +Or let's find a `Text` widget with a specific text: + +```dart +find.text('Subscribe'); +``` + +Using Patrol finder, you'd write the above as: + +```dart +$('Subscribe'); +``` + +Worth mentioning is also `Key`. The below lines are equivalent: + +```dart +find.byKey(Key('loginButton')); +$(Key('loginButton')); +$(#loginButton); +``` + +For those wondering what is that `#` thing – it's a +[Symbol]! Yes, we're Dart +(ab)users. + +All the types that can be passed to `$` are [listed here][doc]. + +### Making assertions + +Creating a finder doesn't do anything – it just _is_. Let's put them to use and +write a few simple assertions. + +Here's how you can make sure that a widget with text `Log in` exists in the +widget tree: + +```dart +expect(find.text('Log in'), findsOneWidget); +``` + +With our Patrol finders, you'd write the above as: + +```dart +expect($('Log in'), findsOneWidget); +``` + +Alternatively, you could also use the `exists` getter, which returns true if the +finder finds at least 1 widget: + +```dart +expect($('Log in').exists, equals(true)); +``` + +We can also make sure that no widget exists, or that a particular number of +widgets exist: + +```dart +expect(find.text("Can't touch this"), findsNothing); +expect(find.byType(Card), findsNWidgets(3)); +``` + +The above expressed with Patrol finders: + +```dart +expect($("Can't touch this"), findsNothing); +expect($(Card), findsNWidgets(3)); +``` + +You could alternatively write the first line as: + +```dart +expect($("Can't touch this").exists, equals(false)); +``` + +[//]: # (not true, TODO: rewrite) +It's important to note that Flutter's default finder functions, such as +[findsNothing] and [findsOneWidget], check if the widget is present in the +widget tree, not if it is visible to the user, which is usually not what we're +interested in. + +To check if the finder finds at least 1 _visible_ widget, use the `visible` +getter: + +```dart +expect($('Log in').visible, equals(true)); +``` + +And to wait for at least 1 widget with the "Log in" text to become visible: + +```dart +await $('Log in').waitUntilVisible(); +``` + +### Performing actions + +Finding widgets alone is cool, but what's even cooler is being able to tap on +them! Let's tap on the first "Subscribe" text: + +```dart +await tester.tap(find.text('Subscribe').first); +``` + +It's usually a good practice to use `first`, because if there were multiple +"Subscribe" texts, `tap()` would throw an exception. + +With Patrol, you get concise code, but you preserve the flexibility: + +```dart +await $('Subscribe').tap(); +``` + +What's very cool about Patrol's `tap()` is that it doesn't immediately fail if +the finder finds no visible widgets – instead, it waits for some time (which you +can specify globally in [PatrolTesterConfig] or as argument to the `tap()` +method) and taps on the first widget as soon as it becomes visible. This lets +you get rid of fixed timeouts and test your app just like a real user would. + +If you wanted to tap on the third "Subscribe" text, you'd do: + +```dart +await $('Subscribe').at(2).tap(); +``` + +And if the "Subscribe" text was in a [Scrollable] widget, such as +[SingleChildScrollView] or [ListView], and you want to make sure that it is +visible (so you can `tap()` on it), you can scroll to it very easily: + +```dart +await $('Subscribe').scrollTo().tap(); +``` + +### Going deeper + +But hey, these were very simple examples. In real apps, unfortunately, finding +widgets is not that easy. + +Often, you'll need to tap on a widget which is in some other widget. + +```dart +await tester.tap( + find.descendant( + of: find.byType(ListView), + matching: find.text('Subscribe'), + ).first +); +``` + +Flutter's finders are starting to grow, while Patrol stays lean: + +```dart +await $(ListView).$('Subscribe').tap(); +``` + +Now, we also make sure that the `Subscribe` text is in a `ListTile`: + +```dart +await tester.tap( + find.descendant( + of: find.byType(ListView), + matching: find.descendant( + of: find.byType(ListTile), + matching: find.text('Subscribe'), + ), + ).first +); +``` + +Hey, this is starting to look complex! Fortunately, you have Patrol: + +```dart +await $(ListView).$(ListTile).$('Subscribe').tap(); +``` + +Sometimes, you might want to perform a lookahead check. Let's say that you want +tap on the first widget with the `Key('learnMore')` that is a descendant of some +`ListTile`, but that `ListTile` must also have the `Text` descendant with the +`Activated` text. + +If you were to express the above as a `Finder`, you'd get: + +```dart +await tester.tap( + find.ancestor( + of: find.text('Activated'), + matching: find.descendant( + of: find.byType(ListTile), + matching: find.byKey(Key('learnMore')), + ), + ).first +); +``` + +With the help of Patrol's custom finders, it's much easier: + +```dart +await $(ListTile).containing('Activated').$(#learnMore).tap(); +``` + +Sometimes, however, the logic required to find a widget cannot be expressed by +the descendant/ancestor relationship like above. In situations like this, +when all ways of finding widgets known to you fail, Patrol has an ace up its +sleeve: the [which()] method. You can use it to find widgets by their +properties. A few examples include: +. + +- entering a text into a text field with no text entered: + + ```dart + await $(#cityTextField) + .which((widget) => widget.controller.text.isNotEmpty) + .enterText('Warsaw, Poland'); + ``` + +- asserting that the icon has the correct color: + + ```dart + await $(Icons.error) + .which((widget) => widget.color == Colors.red) + .waitUntilVisible(); + ``` + +- asserting that the button is disabled and has the correct color + + ```dart + await $('Delete account') + .which((button) => !button.enabled) + .which( + (btn) => btn.style?.backgroundColor?.resolve({}) == Colors.red, + ) + .waitUntilVisible(); + + ``` + +### Falling back + +What's cool about Patrol is that it builds on top of `flutter_test` instead of +replacing it. This means that you can freely mix Patrol's finders with finders +from `flutter_test`, `PatrolTester` with `WidgetTester`, and so on. + +Here's how you can access the default `WidgetTester`: + +```dart +patrolWidgetTest('adds comment', (PatrolTester $) async { + final WidgetTester tester = $.tester; + + await tester.enterText(find.byKey(Key('commentTextField')), 'Very nice!'); +}); +``` + +[doc]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/createFinder.html +[which()]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/PatrolFinder/which.html +[findsNothing]: https://api.flutter.dev/flutter/flutter_test/findsNothing-constant.html +[findsOneWidget]: https://api.flutter.dev/flutter/flutter_test/findsOneWidget-constant.html +[PatrolTesterConfig]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/PatrolTesterConfig-class.html +[singlechildscrollview]: https://api.flutter.dev/flutter/widgets/SingleChildScrollView-class.html +[listview]: https://api.flutter.dev/flutter/widgets/ListView-class.html +[Scrollable]: https://api.flutter.dev/flutter/widgets/Scrollable-class.html +[Symbol]: https://api.dart.dev/dart-core/Symbol-class.html diff --git a/docs/documentation/index.mdx b/docs/documentation/index.mdx new file mode 100644 index 000000000..895f163ca --- /dev/null +++ b/docs/documentation/index.mdx @@ -0,0 +1,688 @@ +--- +title: Install Patrol +--- + +Check out our video version of this tutorial on YouTube! + + + + + If you want to use Patrol finders in your existing widget or golden + tests, go to [Using Patrol finders in widget tests]. + + +## Setup + + + + Install `patrol_cli`: + ``` + flutter pub global activate patrol_cli + ``` + + [Patrol CLI] (command-line interface) is a small program that enables running + Patrol UI tests. It is necessary to run UI tests (`flutter test` won't work! [Here's why]). + + Make sure to add `patrol` to your `PATH` environment variable. + It's explained in the [README]. + + + + + Verify that installation was successful and your environment is set up properly: + + ``` + patrol doctor + ``` + Example output: + ``` + Patrol CLI version: 2.3.1+1 + Android: + • Program adb found in /Users/username/Library/Android/sdk/platform-tools/adb + • Env var $ANDROID_HOME set to /Users/username/Library/Android/sdk + iOS / macOS: + • Program xcodebuild found in /usr/bin/xcodebuild + • Program ideviceinstaller found in /opt/homebrew/bin/ideviceinstaller + ``` + Be sure that for the platform you want to run the test on, all the checks are green. + + + + Patrol CLI invokes the Flutter CLI for certain commands. To override the command used, + pass the `--flutter-command` argument or set the `PATROL_FLUTTER_COMMAND` environment + variable. This supports FVM (by setting the value to `fvm flutter`), puro (`puro flutter`) + and potentially other version managers. + + + + + Add a dependency on the `patrol` package in the + `dev_dependencies` section of `pubspec.yaml`. `patrol` package requires + Android SDK version 21 or higher. + + ``` + flutter pub add patrol --dev + ``` + + + + Create `patrol` section in your `pubspec.yaml`: + + ```yaml title="pubspec.yaml" + dependencies: + # ... + + dev_dependencies: + # ... + + patrol: + app_name: My App + android: + package_name: com.example.myapp + ios: + bundle_id: com.example.MyApp + macos: + bundle_id: com.example.macos.MyApp + ``` + + + In this tutorial, we are using example app, which has package name + `com.example.myapp` on Android, bundle id `com.example.MyApp` on iOS, + `com.example.macos.MyApp` on macOS and `My App` name on all platforms. + Replace any occurences of those names with proper values. + + + + If you don't know where to get `package_name` and `bundle_id` from, see the [FAQ] section. + + + + + + Integrate with native side + + The 3 first steps were common across platforms. The rest is platform-specific. + + Psst... Android is a bit easier to set up, so we recommend starting with it! + + + + + Go to **android/app/src/androidTest/java/com/example/myapp/** in your project + directory. If there are no such folders, create them. **Remember to replace + `/com/example/myapp/` with the path created by your app's package name.** + + + + Create a file named `MainActivityTest.java` and copy there the code below. + + ```java title="MainActivityTest.java" + package com.example.myapp; // replace "com.example.myapp" with your app's package + + import androidx.test.platform.app.InstrumentationRegistry; + import org.junit.Test; + import org.junit.runner.RunWith; + import org.junit.runners.Parameterized; + import org.junit.runners.Parameterized.Parameters; + import pl.leancode.patrol.PatrolJUnitRunner; + + @RunWith(Parameterized.class) + public class MainActivityTest { + @Parameters(name = "{0}") + public static Object[] testCases() { + PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); + // replace "MainActivity.class" with "io.flutter.embedding.android.FlutterActivity.class" + // if in AndroidManifest.xml in manifest/application/activity you have + // android:name="io.flutter.embedding.android.FlutterActivity" + instrumentation.setUp(MainActivity.class); + instrumentation.waitForPatrolAppService(); + return instrumentation.listDartTests(); + } + + public MainActivityTest(String dartTestName) { + this.dartTestName = dartTestName; + } + + private final String dartTestName; + + @Test + public void runDartTest() { + PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); + instrumentation.runDartTest(dartTestName); + } + } + ``` + + + + Go to the **build.gradle** file, located in **android/app** folder in your + project directory. + + + + Add these 2 lines to the `defaultConfig` section: + + ```groovy title="android/app/build.gradle" + testInstrumentationRunner "pl.leancode.patrol.PatrolJUnitRunner" + testInstrumentationRunnerArguments clearPackageData: "true" + ``` + + + + Add this section to the `android` section: + + ```groovy title="android/app/build.gradle" + testOptions { + execution "ANDROIDX_TEST_ORCHESTRATOR" + } + ``` + + + + Add this line to `dependencies` section: + + ```groovy title="android/app/build.gradle" + androidTestUtil "androidx.test:orchestrator:1.4.2" + ``` + + + + + Bear in mind that ProGuard can lead to some problems if not well configured, potentially causing issues such as `ClassNotFoundException`s. + Keep all the Patrol packages or disable ProGuard in `android/app/build.gradle`: + ```groovy title="android/app/build.gradle" + ... + buildTypes { + release { + ... + } + debug { + minifyEnabled false + shrinkResources false + } + } + ``` + + + + + + + + Open `ios/Runner.xcworkspace` in Xcode. + + + + Create a test target if you do not already have one (see the screenshot below + for the reference). Select `File > New > Target...` and select `UI Testing Bundle`. + Change the `Product Name` to `RunnerUITests`. Set the `Organization Identifier` + to be the same as for the `Runner` (no matter if you app has flavors or not). + For our example app, it's `com.example.MyApp` just as in the `pubspec.yaml` file. + Make sure `Target to be Tested` is set to `Runner` and language is set to `Objective-C`. + Select `Finish`. + + ![Xcode iOS test target](/assets/ios_test_target.png) + + + + 2 files are created: `RunnerUITests.m` and `RunnerUITestsLaunchTests.m`. + Delete `RunnerUITestsLaunchTests.m` **through Xcode** by clicking on it and + selecting `Move to Trash`. + + + + Make sure that the **iOS Deployment Target** of `RunnerUITests` within the + **Build Settings** section is the same as `Runner`. + The minimum supported **iOS Deployment Target** is `11.0`. For the + [example app], + we set it to `13.0` because it's required by the app dependencies. + + ![Xcode iOS deployment target](/assets/ios_deployment_target.png) + + ![Xcode iOS deployment target 2](/assets/ios_deployment_target_2.png) + + + + Replace contents of `RunnerUITests.m` file with the following: + + ```objective-c title="ios/RunnerUITests/RunnerUITests.m" + @import XCTest; + @import patrol; + @import ObjectiveC.runtime; + + PATROL_INTEGRATION_TEST_IOS_RUNNER(RunnerUITests) + ``` + + Add the newly created target to `ios/Podfile` by embedding in the existing + `Runner` target. + + ```ruby title="ios/Podfile" + target 'Runner' do + # Do not change existing lines. + ... + + target 'RunnerUITests' do + inherit! :complete + end + end + ``` + + + + Create an empty file `integration_test/example_test.dart` in the root of your Flutter project. From the command line, run + the following command and make sure it completes with no errors: + + ``` + $ flutter build ios --config-only integration_test/example_test.dart + ``` + + + + Go to your `ios` directory and run: + + ``` + $ pod install --repo-update + ``` + + + + Open your Xcode project and Make sure that for each build configuration, + the `RunnerUITests` have the same Configuration Set selected as the `Runner`: + + ![Xcode config setup](/assets/ios_runner_configs.png) + + + + Go to **RunnerUITests** -> **Build Phases** and add 2 new "Run Script Phase" Build Phases. + Name them `xcode_backend build` and `xcode_backend embed_and_thin`. + + ![Xcode config setup](/assets/ios_runner_build_phases_create.png) + + + + Arrange the newly created Build Phases in the order shown in the screenshot below. + + ![Xcode config setup](/assets/ios_runner_build_phases.png) + + + + Paste this code into the `xcode_backend build` Build Phase: + + ``` + /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh" build + ``` + + + + Paste this code into the `xcode_backend embed_and_thin` Build Phase: + + ``` + /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh" embed_and_thin + ``` + + + + Xcode by default also enables a "parallel execution" setting, which + breaks Patrol. Disable it **for all schemes** (if you have more than one): + + + + + + Go to **RunnerUITests** -> **Build Settings**, search for **User Script Sandboxing** + and make sure it's set to **No**. + + + + + + + + + Support for macOS is in alpha stage. Please be aware that some features + may not work as expected. There is also no native automation support + for macOS yet. If you encounter any issues, please report them on + GitHub. + + + + + Open `macos/Runner.xcworkspace` in Xcode. + + + + Create a test target if you do not already have one via `File > New > Target...` + and select `UI Testing Bundle`. Change the `Product Name` to `RunnerUITests`. Make + sure `Target to be Tested` is set to `Runner` and language is set to `Objective-C`. + Select `Finish`. + + + + 2 files are created: `RunnerUITests.m` and `RunnerUITestsLaunchTests.m`. + Delete `RunnerUITestsLaunchTests.m` **through Xcode**. + + + + Make sure that the **macOS Deployment Target** of `RunnerUITests` within the + **Build Settings** section is the same as `Runner`. + The minimum supported **macOS Deployment Target** is `10.14`. + + ![Xcode macOS deployment target](/assets/macos_deployment_target.png) + + ![Xcode macOS deployment target 2](/assets/macos_deployment_target_2.png) + + + + Replace contents of `RunnerUITests.m` file with the following: + + ```objective-c title="macos/RunnerUITests/RunnerUITests.m" + @import XCTest; + @import patrol; + @import ObjectiveC.runtime; + + PATROL_INTEGRATION_TEST_MACOS_RUNNER(RunnerUITests) + ``` + + Add the newly created target to `macos/Podfile` by embedding in the existing + `Runner` target. + + ```ruby title="macos/Podfile" + target 'Runner' do + # Do not change existing lines. + ... + + target 'RunnerUITests' do + inherit! :complete + end + end + ``` + + + + Create an empty file `integration_test/example_test.dart` in the root of your Flutter project. From the command line, run: + + ``` + $ flutter build macos --config-only integration_test/example_test.dart + ``` + + + + Go to your `macos` directory and run: + + ``` + $ pod install --repo-update + ``` + + + + Go to **RunnerUITests** -> **Build Phases** and add 2 new "Run Script Phase" Build Phases. + Rename them to `xcode_backend build` and `xcode_backend embed_and_thin` by double clicking + on their names. + + ![Xcode config setup](/assets/macos_runner_build_phases_create.png) + + + + Arrange the newly created Build Phases in the order shown in the screenshot below. + + ![Xcode config setup](/assets/macos_runner_build_phases.png) + + + + Paste this code into the first `macos_assemble build` Build Phase: + + ``` + /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/macos_assemble.sh" build + ``` + + + + Paste this code into the second `macos_assemble embed` Build Phase: + + ``` + /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/macos_assemble.sh" embed + ``` + + + + Xcode by default also enables a "parallel execution" setting, which + breaks Patrol. Disable it **for all schemes** (if you have more than one): + + + + + + Go to **RunnerUITests** -> **Build Settings**, search for **User Script Sandboxing** + and make sure it's set to **No**. + + + + Go to **Runner** -> **Signing & Capabilities**. Make sure that in all **App Sandbox** + sections, **Incoming Connections (Server)** and **Outgoing Connections (Client)** checkboxes + are checked. + + ![Xcode entitlements setup](/assets/macos_entitlements.png) + + + + **Copy** `DebugProfile.entitlements` and `Release.entitlements` files from `macos/Runner` + to `macos/RunnerUITests` directory. + + + + Go to **RunnerUITests** -> **Build Settings** and set **Code Signing Entitlements** to + `RunnerUITests/DebugProfile.entitlements` for **Debug** and **Profile** configuration and to + `RunnerUITests/Release.entitlements` for **Release** configuration. + + ![Xcode RunnerUITests entitlements setup](/assets/macos_ui_entitlements.png) + + + + + + + Create a simple integration test + + Let's create a dummy Flutter integration test that you'll use to verify + that Patrol is correctly set up. + + Paste the following code into `integration_test/example_test.dart`: + + ```dart title="integration_test/example_test.dart" + import 'dart:io'; + + import 'package:flutter/material.dart'; + import 'package:flutter_test/flutter_test.dart'; + import 'package:patrol/patrol.dart'; + + void main() { + patrolTest( + 'counter state is the same after going to home and switching apps', + ($) async { + // Replace later with your app's main widget + await $.pumpWidgetAndSettle( + MaterialApp( + home: Scaffold( + appBar: AppBar(title: const Text('app')), + backgroundColor: Colors.blue, + ), + ), + ); + + expect($('app'), findsOneWidget); + if (!Platform.isMacOS) { + await $.native.pressHome(); + } + }, + ); + } + ``` + + It does only 2 things: + + - first, it finds a text `app` + - then (on mobile platforms), it exits to home screen + + It's a very simple test, but it's enough to verify that Patrol is correctly set + up. To run `integration_test/example_test.dart` on a connected Android, iOS or macOS device: + + ``` + patrol test -t integration_test/example_test.dart + ``` + + If the setup is successful, you should see a summary like one below. + ``` + Test summary: + 📝 Total: 1 + ✅ Successful: 1 + ❌ Failed: 0 + ⏩ Skipped: 0 + 📊 Report: + ⏱️ Duration: 4s + ``` + If something went wrong, please proceed to the [FAQ] section which might + contain an answer to your issue. + + + + + To prevent issues during Patrol tests, please follow these guidelines: + + 1. Do not call `IntegrationTestWidgetsFlutterBinding.ensureInitialized`. + Patrol automatically initializes its own test binding. + 2. Do not modify the global `FlutterError.onError` callback. Patrol's + internals depend on it. Keep in mind that this callback can also + be modified by popular packages such as Sentry or Crashlytics. + In such cases, you can disable them for Patrol tests. + + + + If you are looking for a working example of a Flutter app with Patrol tests, + check out the [example app] + in the patrol repository. + + + ## Flavors + If your app is using flavors, then you can pass them like so: + + ``` + patrol test --target integration_test/example_test.dart --flavor development + ``` + + or you can specify them in `pubspec.yaml` (recommended): + + ```yaml title="pubspec.yaml" + patrol: + app_name: My App + flavor: development + android: + package_name: com.example.myapp + ios: + bundle_id: com.example.MyApp + app_name: The Awesome App + macos: + bundle_id: com.example.macos.MyApp + ``` + +## FAQ + + + The reason is probably a mismatch of `patrol` and `patrol_cli` versions. Go to [Compatibility table] + and make sure that the versions of `patrol` and `patrol_cli` you are using are compatible. + + + + To run your application within the patrol test, you need to call `$.pumpWidgetAndSettle()`, + and pass your application's main widget to it. Be sure that you registered all the + necessary services before calling `$.pumpWidgetAndSettle()`. + Here's the example of running an app within the patrol test: + ```dart + + void main() { + patrolTest('real app test', ($) async { + + // Do all the necessary setup here (DI, services, etc.) + + await $.pumpWidgetAndSettle(const MyApp()); // Your's app main widget + + // Start testing your app here + + }); + } + + ``` + It's a good practice to create a setup wrapper function for your tests, so you don't have to + repeat the same code in every test. Look at the [example] + of a wrapper function. + + + +### Android + + + Go to `android/app/build.gradle` and look for `applicationId` in `defaultConfig` section. + + + + It's most likely caused by using incompatible JDK version. + Run `javac -version` to check your JDK version. Patrol officially works on JDK 17, + so unexpected errors may occur on other versions. + If you have AndroidStudio or Intellij IDEA installed, you can find the path to JDK by + opening your project's android directory in AS/Intellij and going to + **Settings** -> **Build, Execution, Deployment** -> **Build Tools** -> **Gradle** -> **Gradle JDK**. + [Learn more] + + + +### iOS + + + For iOS go to `ios/Runner.xcodeproj/project.pbxproj` and look for `PRODUCT_BUNDLE_IDENTIFIER`. + For macOS go to `macos/Runner.xcodeproj/project.pbxproj` and look for `PRODUCT_BUNDLE_IDENTIFIER`. + + + + Make sure that you disabled "Paralell execution" for **all schemes** in Xcode. + See [this video] for details. + + + + Search for a `FLUTTER_TARGET` in your project files and remove it (both value and key) + from *.xcconfig and *.pbxproj files. + + + + Search for a `FLUTTER_TARGET` in your project files and remove it (both value and key) + from *.xcconfig and *.pbxproj files. + + + + Check if this line in `Podfile` is present and uncommented. + ``` + platform :ios, '11.0' + ``` + If yes, then check if **iOS deployment version** in Xcode project's **Build Settings** + section for all targets (Runner and RunnerUITests) are set to the same value as in Podfile + (in case presented in snippet above, all should be set to 11.0). + + +If you couldn't find an answer to your question/problem, feel free to ask on +[Patrol Discord Server]. + +## Going from here + +To learn how to write Patrol tests, see [finders] and [native automation] sections. + +[native automation]: /documentation/native/usage +[finders]: /documentation/finders/usage +[Using Patrol finders in widget tests]: /documentation/finders/finders-setup +[Here's why]: /documentation/native/advanced#embrace-the-native-tests +[Patrol CLI]: https://pub.dev/packages/patrol_cli +[FAQ]: /documentation#faq +[Compatibility table]: /documentation/compatibility-table +[README]: https://pub.dev/packages/patrol_cli#installation +[example app]: https://github.com/leancodepl/patrol/tree/master/packages/patrol/example +[example]: https://github.com/leancodepl/patrol/blob/d2c7493f9399a028e39cb94fd204affdb932c5fc/dev/e2e_app/integration_test/common.dart#L17-L33 +[Learn more]: https://developer.android.com/build/jdks +[this video]: https://www.youtube.com/watch?v=9LdEJR59fW4 +[Patrol Discord Server]: https://discord.gg/ukBK5t4EZg diff --git a/docs/documentation/integrations/allure.mdx b/docs/documentation/integrations/allure.mdx new file mode 100644 index 000000000..f5bd90b70 --- /dev/null +++ b/docs/documentation/integrations/allure.mdx @@ -0,0 +1,221 @@ +--- +title: Integrations - allure +--- + +## Overview + +If you're using [Allure] to report your test results, you can use the +alternative test runner to get more detailed test report. + +We decided not to package this alternative runner together with Patrol because +it'd make Patrol depend on Allure, which is not desirable. Instead, you can +easily do it yourself. + +This guide assumes basic familiarity with Allure. To get started, see: + +- [official Allure documentation] +- [allure-framework/allure2 repository] + +This integration is currently Android-only. + + + Before you proceed with the steps listed below, make sure that you've + completed the [native setup] guide. + + +### Add dependencies and change runner + +First, you have to modify the **app-level build.gradle**: + +```groovy title="android/app/build.gradle" +android { + // ... + defaultConfig { + // ... + + // Replace the existing "testInstrumentationRunner" line with: + testInstrumentationRunner "pl.leancode.patrol.example.AllurePatrolJUnitRunner" + } + // ... +} + +dependencies { + androidTestImplementation "io.qameta.allure:allure-kotlin-model:2.4.0" + androidTestImplementation "io.qameta.allure:allure-kotlin-commons:2.4.0" + androidTestImplementation "io.qameta.allure:allure-kotlin-junit4:2.4.0" + androidTestImplementation "io.qameta.allure:allure-kotlin-android:2.4.0" +} +``` + + + Replace `pl.leancode.patrol.example` with your app's package name. + + +See also: + +- [the README of allure-kotlin library][allure_kotlin] + +### Create alternative runner + +Create a new Kotlin file in the same directory as **MainActivityTest.java** and +paste the following code, replacing the package: + +```kotlin title="AllurePatrolJUnitRunner.kt" +package pl.leancode.patrol.example // replace "pl.leancode.patrol.example" with your app's package + +import android.os.Bundle +import io.qameta.allure.android.AllureAndroidLifecycle +import io.qameta.allure.android.listeners.ExternalStoragePermissionsListener +import io.qameta.allure.android.writer.TestStorageResultsWriter +import io.qameta.allure.kotlin.Allure +import io.qameta.allure.kotlin.junit4.AllureJunit4 +import io.qameta.allure.kotlin.util.PropertiesUtils +import pl.leancode.patrol.PatrolJUnitRunner + +class AllurePatrolJUnitRunner : PatrolJUnitRunner() { + override fun onCreate(arguments: Bundle) { + Allure.lifecycle = createAllureAndroidLifecycle() + val listenerArg = listOfNotNull( + arguments.getCharSequence("listener"), + AllureJunit4::class.java.name, + ExternalStoragePermissionsListener::class.java.name.takeIf { useTestStorage } + ).joinToString(separator = ",") + arguments.putCharSequence("listener", listenerArg) + super.onCreate(arguments) + } + + private fun createAllureAndroidLifecycle() : AllureAndroidLifecycle { + return createDefaultAllureAndroidLifecycle() + } + + private fun createDefaultAllureAndroidLifecycle() : AllureAndroidLifecycle { + if (useTestStorage) { + return AllureAndroidLifecycle(TestStorageResultsWriter()) + } + + return AllureAndroidLifecycle() + } + + private val useTestStorage: Boolean + get() = PropertiesUtils.loadAllureProperties() + .getProperty("allure.results.useTestStorage", "true") + .toBoolean() +} +``` + + + In the snippet above, remember to replace the `package + pl.leancode.patrol.example` line at the top of the file with your app's + package name! + + +### Create allure.properties + +This is required if you enabled the `clearPackageData` option for Android Test +Orchestrator. If you enabled that option but don't create the +`allure.properties` file as below, your tests reports will be cleared after each +test. + +```txt title="android/app/src/main/res/allure.properties" +allure.results.useTestStorage=true +``` + +### Add rules to MainActivityTest + +Finally, modify the **MainActivityTest.java**. You'll add a 3 rules, which add +the following features: + +- automatically take a screenshot at the end of each test +- automatically dump the window hierarchy at the end of each test +- automatically embed the logcat into the report + +You can simply copy-paste the following code (remember to replace the package +name): + +```java title="MainActivityTest.java" +package pl.leancode.patrol.example; // replace "pl.leancode.patrol.example" with your app's package + +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import io.qameta.allure.android.rules.LogcatRule; +import io.qameta.allure.android.rules.ScreenshotRule; +import io.qameta.allure.android.rules.WindowHierarchyRule; +import pl.leancode.patrol.PatrolJUnitRunner; + +@RunWith(Parameterized.class) +public class MainActivityTest { + @Rule + public ScreenshotRule screenshotRule = new ScreenshotRule(ScreenshotRule.Mode.END, "ss_end"); + + @Rule + public WindowHierarchyRule windowHierarchyRule = new WindowHierarchyRule(); + + @Rule + public LogcatRule logcatRule = new LogcatRule(); + + @Parameters(name = "{0}") + public static Object[] testCases() { + PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); + instrumentation.setUp(MainActivity.class); + instrumentation.waitForPatrolAppService(); + return instrumentation.listDartTests(); + } + + public MainActivityTest(String dartTestName) { + this.dartTestName = dartTestName; + } + + private final String dartTestName; + + @Test + public void runDartTest() { + PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); + instrumentation.runDartTest(dartTestName); + } +} +``` + + + In the snippet above, remember to replace the `package + pl.leancode.patrol.example` line at the top of the file with your app's + package name! + + +### Retrieve the report + +Run the tests with `patrol test` as usual. + +After the tests are complete, create a directory for them, for example: + +```bash +mkdir -p ./build/reports +``` + +and then retrieve the results from the device: + +```bash +adb exec-out sh -c 'cd /sdcard/googletest/test_outputfiles && tar cf - allure-results' | tar xvf - -C build/reports +``` + +Finally, serve the results with Allure: + +```bash +allure serve ./build/reports/allure-results +``` + + + If you're using Homebrew, `brew install allure` is the quickest way to get + Allure. + + +[native setup]: /documentation +[allure]: https://qameta.io/allure-report +[allure_kotlin]: https://github.com/allure-framework/allure-kotlin/blob/master/README.md +[official Allure documentation]: https://docs.qameta.io/allure-report +[allure-framework/allure2 repository]: https://github.com/allure-framework/allure2 diff --git a/docs/documentation/integrations/browserstack.mdx b/docs/documentation/integrations/browserstack.mdx new file mode 100644 index 000000000..f9c6441c2 --- /dev/null +++ b/docs/documentation/integrations/browserstack.mdx @@ -0,0 +1,90 @@ +--- +title: Integrations - Browserstack +--- + +## Overview + +[Browserstack App Automate] is a popular cloud device farm. + +This integration is currently Android-only. + +### Change runner + +Modify the **app-level build.gradle**: + +```groovy title="android/app/build.gradle" +android { + // ... + defaultConfig { + //... + testInstrumentationRunner "pl.leancode.patrol.BrowserstackPatrolJUnitRunner" + } + // ... +} + +// ... +``` + +That's all! + +### Upload to Browserstack + +To run Android UI tests on BrowserStack: + +1. Build the app under test and the instrumentation app ([see docs][patrol build]) +1. Upload the app under test APK to Browserstack ([see docs][bs_app_docs]) +1. Upload the instrumentation app APK to Browserstack ([see docs][bs_test_docs]) +1. Start test execution on Browserstack ([see docs][bs_execute_docs]) + +You can do it all manually, but we recommend using the [bs_android] script to +speed up this process. `bs_android` is part of LeanCode's [mobile-tools]. If +you're using Homebrew, you can install it with: + +```bash +brew tap leancodepl/tools +brew install mobile-tools +``` + +The `bs_android` script requires the `BROWSERSTACK_CREDS` environment variable +to be set so it can authenticate with Browserstack: + +```bash +export YOUR_USERNAME:YOUR_ACCESS_KEY +``` + +Get your username and access on [Browserstack's account page][bs_account]. + +Now reload your shell (e.g. `exec zsh`) and run `bs_android`: + +``` +$ export BROWSERSTACK_PROJECT=AwesomeApp # optional +$ export BROWSERSTACK_DEVICES="[\"Google Pixel 4-10.0\"]" # optional +$ bs_android +• Building apk with entrypoint test_bundle.dart... +✓ Completed building apk with entrypoint test_bundle.dart (11.0s) + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed +100 87.4M 100 235 100 87.4M 7 2857k 0:00:33 0:00:31 0:00:02 2052k +Uploaded app, url: bs://fb61a714e1a0c60e2578d940dad52b74da244d54 +Uploaded test, url: bs://a715b1231d41ac627bd683f1b16c28476babd72e +{"message":"Success","build_id":"a30440db559fcab65554ab0273437f3bd45d761b"} +Scheduled test execution +``` + +`bs_android` forwards all its options and flags to `patrol build`: + +```bash +bs_android \ + --target integration_test/example_test.dart,integration_test/another_test.dart \ + --release \ + --dart-define 'KEY_EXAMPLE=VALUE_EXAMPLE' +``` + +[Browserstack App Automate]: https://www.browserstack.com/app-automate +[patrol build]: /cli-commands/build +[bs_android]: https://github.com/leancodepl/mobile-tools/blob/master/bin/bs_android +[bs_app_docs]: https://www.browserstack.com/docs/app-automate/api-reference/espresso/apps +[bs_test_docs]: https://www.browserstack.com/docs/app-automate/api-reference/espresso/tests +[bs_execute_docs]: https://www.browserstack.com/docs/app-automate/api-reference/espresso/builds#execute-a-build +[bs_account]: https://www.browserstack.com/accounts/profile +[mobile-tools]: https://github.com/leancodepl/mobile-tools diff --git a/docs/documentation/logs.mdx b/docs/documentation/logs.mdx new file mode 100644 index 000000000..51e135f7b --- /dev/null +++ b/docs/documentation/logs.mdx @@ -0,0 +1,123 @@ +--- +title: Logs and test results +--- + +Once you've written and executed your tests, it's essential to monitor their results. Patrol provides two main methods for reporting test outcomes: **console logs** and **native test reports**. + +## Logging test steps + + +This feature is available starting from version `3.13.0`. + +If you're using this version but don't see logs for test steps, check if you're passing a custom `PatrolTesterConfig` to `patrolTest()`. If so, ensure the `printLogs: true` argument is included in the constructor. + + +During test execution, every test step (e.g., `tap` or `enterText`) is logged to the console along with its status. Additionally, the test name, status, and execution time are displayed. + +**Example console output:** + +``` +... +🧪 denies various permissions + ✅ 1. scrollTo widgets with text "Open permissions screen". + ✅ 2. scrollTo widgets with text "Open permissions screen". + ✅ 3. tap widgets with text "Open permissions screen". + ✅ 4. tap widgets with text "Request camera permission". + ✅ 5. isPermissionDialogVisible (native) + ✅ 6. tap widgets with text "Request camera permission". + ✅ 7. isPermissionDialogVisible (native) + ⏳ 8. denyPermission (native) +❌ denies various permissions (integration_test/permissions/deny_many_permissions_twice_test.dart) (9s) +══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞═════════════════ +The following PlatformException was thrown running a test: +PlatformException(PermissionHandler.PermissionManager, A request +for permissions is already running, please wait for it to finish +before doing another request (note that you can request multiple +permissions at the same time)., null, null) + +When the exception was thrown, this was the stack: +#0 StandardMethodCodec.decodeEnvelope (package:flutter/src/services/message_codecs.dart:648:7) +#1 MethodChannel._invokeMethod (package:flutter/src/services/platform_channel.dart:334:18) + +#2 MethodChannelPermissionHandler.requestPermissions (package:permission_handler_platform_interface/src/method_channel/method_channel_permission_handler.dart:79:9) + +#3 PermissionActions.request (package:permission_handler/permission_handler.dart:52:31) + +#4 _PermissionsScreenState._requestCameraPermission (package:e2e_app/permissions_screen.dart:21:20) + + +The test description was: + denies various permissions +═════════════════════════════════════════════════════════════════ + +✅ taps on notification (integration_test/permissions/notifications_test.dart) (16s) +✅ taps on notification native2 (integration_test/permissions/notifications_test.dart) (14s) +✅ grants various permissions (integration_test/permissions/permissions_many_test.dart) (15s) +... +``` + +## Test summary + +Once the tests are complete, a summary is printed: + + +``` +Test summary: +📝 Total: 8 +✅ Successful: 3 +❌ Failed: 5 + - taps on notification (integration_test/permissions/notifications_test.dart) + - taps on notification native2 (integration_test/permissions/notifications_test.dart) + - accepts location permission (integration_test/permissions/permissions_location_test.dart) + - accepts location permission native2 (integration_test/permissions/permissions_location_test.dart) + - grants various permissions (integration_test/permissions/permissions_many_test.dart) +⏩ Skipped: 0 +📊 Report: file:///Users/user/patrol/dev/e2e_app/build/app/reports/androidTests/connected/index.html +⏱️ Duration: 227s +``` + +## Customizing log behavior + +You can customize which logs are displayed by using the following flags. These can be passed to the `patrol test` or `patrol develop` commands: + +| Flag | Description | Available in | Default value | +| ------------------------- | ------------------------------------------ | -------------------------------------------------- | ------------- | +| --[no-]show-flutter-logs | Show Flutter logs while running the tests. | `patrol test`, in `patrol develop` it's always on | `false` | +| --[no-]hide-test-steps | Hide test steps while running the tests. | `patrol test` and `patrol develop` | `false` | +| --[no-]clear-test-steps | Clear test steps after the test finishes. | `patrol test` | `true` | + +## Native test reports + +In addition to console logs, you can review test results in a **native test report**. The report's file path is provided in the test summary, for example: + +``` +📊 Report: file:///Users/user/patrol/dev/e2e_app/build/app/reports/androidTests/connected/index.html +``` + +## Logs in `patrol_finders` + +By default, enhanced logs are disabled when using `patrol_finders` without the `patrol` package. To enable them, pass the `printLogs: true` argument to the `PatrolTesterConfig` constructor: + +```dart +patrolWidgetTest( + 'throws exception when no widget to tap on is found', + config: const PatrolTesterConfig(printLogs: true), + (tester) async { + // test body + // ... + }, +); +``` +```dart +testWidgets( + 'description', + (widgetTester) async { + final $ = PatrolTester( + tester: widgetTester, + config: PatrolTesterConfig(printLogs: true), + ); + // test body + // ... + }, +); +``` diff --git a/docs/documentation/native/advanced.mdx b/docs/documentation/native/advanced.mdx new file mode 100644 index 000000000..b5a3973de --- /dev/null +++ b/docs/documentation/native/advanced.mdx @@ -0,0 +1,157 @@ +--- +title: Native automation - advanced +--- + +### Patrol section in `pubspec.yaml` + +If your app has different name on iOS and Android, you can specify `app_name` +twice – one in `android` block, and one in `ios` block. + +Though the whole Patrol section in `pubspec.yaml` is optional, we highly recommend +adding this section, because it enables the following features: + +- Patrol will automatically uninstall your app after every test (using + `package_name` and `bundle_id`). This will make the environment which your + tests run in more stable and predictable. +- Patrol will be able to tap on your app's notifications (using `app_name`) + +### Specific version of `patrol_cli` + +You can install a specific version of Patrol CLI. For example: + +``` +dart pub global activate patrol_cli ^1.0.0 +``` + +will install the latest `v1` version. We recommend to install a specific +version on CI systems to avoid unexpected breakages. + +### Isolation of test runs + +To achieve full isolation between test runs, enable the `clearPackageOption`: + +```groovy title="android/app/build.gradle" +defaultConfig { + //... + testInstrumentationRunner "pl.leancode.patrol.PatrolJUnitRunner" + testInstrumentationRunnerArguments clearPackageData: "true" +} +``` + +This will clear the app's data and permissions before each test run. +Unfortunately, no equivalent feature is available on iOS. + +### Embrace the native tests + +If you've diligently followed the steps in [native automation setup] and `patrol test` prints a **TEST +PASSED** message, you might be now thinking: what did I just do? + +The answer is: You've just integrated Flutter testing with native Android/iOS +testing frameworks. This means that your Flutter integration tests can now be +run as _native tests_. + +#### What are native tests good for, anyway? + +iOS and Android have existed for more than 15 years, and during that time many +of awesome testing-related things were built around them – open-source test +runners, device farms, HTML report generators. Developers who create native +mobile apps can easily reap benefits from this huge, mature ecosystem. + +Meanwhile we, Flutter developers, don't have as much at our disposal. Our +framework is much younger and less mature. + +What if we could masquerade our Flutter tests so that from the outside they +would be truly native? This way we leverage many existing tools while +maintaining the convenience of writing the tests in pure Dart. + +> For example, you can run your Patrol tests directly from Xcode. Xcode knows +> nothing about Flutter, Dart and Patrol – it only launches your test app. +> Flutter tests are then run inside the test app and the results are reported +> back to Xcode. This way you get the best of both worlds – the maturity of +> native iOS development and the productivity of Flutter and Dart. + +That's exactly what Patrol does (and what the default [integration_test] package +does at well, but at a bit smaller scale). + +Take a look at this simple Flutter integration tests using Patrol: + +```dart title="integration_test/example_test.dart" +void main() { + patrolTest( + 'counter state is the same after going to Home and switching apps', + nativeAutomatorConfig: NativeAutomatorConfig( + packageName: 'pl.leancode.patrol.example', + bundleId: 'pl.leancode.patrol.Example', + ), + ($) async { + await $.pumpWidget(ExampleApp()); + + await $(FloatingActionButton).tap(); + expect($(#counterText).text, '1'); + + await $.native.pressHome(); + await $.native.openApp(); + + expect($(#counterText).text, '1'); + + await $(FloatingActionButton).tap(); + expect($(#counterText).text, '2'); + }, + ); +} +``` + +You can run this test and view its results in many ways, using all sorts of +different tools, platforms, and IDEs: + + + + When Android test finishes, its test results are automatically generated in + `build/app/outputs/androidTest-results/connected/test-result.pb`. To view + them in Android Studio, use the `Run > Import tests from file` option. + + ![Test results in Android Studio](/assets/android_test_results_as.png) + + + + ![Patrol test results in Xcode](/assets/ios_test_results_xcode.png) + + + It just works ✨ + + ![Test results in Firebase Test Lab](/assets/android_test_results_firebase.png) + + + + You don't have to use the bulky Android Studio to view your test results, + because Gradle 🐘 automatically generates a nice HTML summary! + + You can find it in `build/app/reports/androidTests/connected/index.html`. + + ![Test results generated by Gradle](/assets/android_test_results_gradle.png) + + + + With the help of awesome [fastlane scan] you can prettify the output of + `xcodebuild` to make it easier to understand and generate HTML summary of + your tests. + + ![Test results by xcpretty](/assets/ios_test_results_xcpretty.png) + + + + +This is so awesome! + +[native automation setup]: /documentation +[fastlane scan]: https://docs.fastlane.tools/actions/scan +[integration_test]: https://github.com/flutter/flutter/tree/master/packages/integration_test diff --git a/docs/documentation/native/feature-parity.mdx b/docs/documentation/native/feature-parity.mdx new file mode 100644 index 000000000..629898fe4 --- /dev/null +++ b/docs/documentation/native/feature-parity.mdx @@ -0,0 +1,58 @@ +--- +title: Native feature's parity +--- + +Here you can see what you can already do with Patrol, and what is yet to be +implemented. We hope that it will help you evaluate Patrol. + +We strive for high feature parity across iOS and Android, but in some cases it's +impossible to reach 100%. macOS support is still in alpha, so it has no native features yet. + +| **Feature** | **Android** | **iOS** | **macOS (alpha)** | +| --------------------------- | -------------- | ------------------ | ----------------- | +| [Press home] | ✅ | ✅ | ❌ | +| [Press back] | ✅ | ❌ (no API) | ❌ | +| [Open any app] | ✅ | ✅ | ✅ | +| [Open notifications] | ✅ | ✅ | ❌ | +| [Tap on notification] | ✅ | ✅ | ❌ | +| [Open quick settings] | ✅ | ✅ | ❌ | +| [Open url] | ✅ | ✅ | ✅ | +| [Toggle dark mode] | ✅ | ✅ | ❌ | +| [Toggle airplane mode] | ✅ | ✅ | ❌ | +| [Toggle cellular] | ✅ | ✅ | ❌ | +| [Toggle Wi-Fi] | ✅ | ✅ | ❌ | +| [Toggle Bluetooth] | ✅ | ✅ | ❌ | +| Toggle location | ✅ | ✅ see [#326] | ❌ | +| [Tap] | ✅ | ✅ | ❌ | +| [Double tap] | ✅ | ✅ | ❌ | +| [Tap at coordinate] | ✅ | ✅ | ❌ | +| [Enter text] | ✅ | ✅ | ❌ | +| [Swipe] | ✅ | ✅ | ❌ | +| [Handle permission dialogs] | ✅ | ✅ | ❌ | +| Interact with WebView | ⚠️ see [#244] | ✅ | ❌ | +| [Press volume down] | ✅ | ✅ (simulator ❌) | ❌ | +| [Press volume up] | ✅ | ✅ (simulator ❌) | ❌ | + + +[#244]: https://github.com/leancodepl/patrol/issues/244 +[#326]: https://github.com/leancodepl/patrol/issues/326 +[press home]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressHome.html +[press back]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressBack.html +[open any app]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openApp.html +[open notifications]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openNotifications.html +[tap on notification]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/tapOnNotificationBySelector.html +[open quick settings]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openQuickSettings.html +[open url]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openUrl.html +[toggle dark mode]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableDarkMode.html +[toggle airplane mode]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableAirplaneMode.html +[toggle cellular]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableCellular.html +[toggle wi-fi]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableWifi.html +[toggle bluetooth]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableBluetooth.html +[tap]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/tap.html +[double tap]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/doubleTap.html +[tap at coordinate]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/tapAt.html +[enter text]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enterText.html +[swipe]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/swipe.html +[handle permission dialogs]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/grantPermissionWhenInUse.html +[press volume down]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressVolumeDown.html +[press volume up]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressVolumeUp.html diff --git a/docs/documentation/native/overview.mdx b/docs/documentation/native/overview.mdx new file mode 100644 index 000000000..cede0d872 --- /dev/null +++ b/docs/documentation/native/overview.mdx @@ -0,0 +1,49 @@ +--- +title: Native automation - overview +--- + +Flutter's [integration_test][integration_test] does a good job at providing +basic support for integration testing Flutter apps. What it can't do is +interaction with the OS your Flutter app is running on. This makes it impossible +to test many critical business features: + +- granting runtime permissions +- signing into the app which uses WebView or 0Auth (like Google) as the login + page +- listing and tapping on notifications +- exiting the app, coming back, and verifying that state is preserved +- enabling and disabling features such as Wi-Fi, mobile data, location, or dark + mode + +Patrol's _native automation_ feature finally solves these problems. Here's a +tiny snippet to spice things up: + +```dart title="integration_test/demo_test.dart" +void main() { + patrolTest('demo', (PatrolIntegrationTester $) async { + await $.pumpWidgetAndSettle(AwesomeApp()); + // prepare network conditions + await $.native.enableCellular(); + await $.native.disableWifi(); + + // toggle system theme + await $.native.enableDarkMode(); + + // handle native location permission request dialog + await $.native.selectFineLocation(); + await $.native.grantPermissionWhenInUse(); + + // tap on the first notification + await $.native.openNotifications(); + await $.native.tapOnNotificationByIndex(0); + }); +} +``` + +Native automation is currently available only on Android and iOS. + + + + + +[integration_test]: https://github.com/flutter/flutter/tree/master/packages/integration_test diff --git a/docs/documentation/native/setup.mdx b/docs/documentation/native/setup.mdx new file mode 100644 index 000000000..e47cbfb88 --- /dev/null +++ b/docs/documentation/native/setup.mdx @@ -0,0 +1,3 @@ +--- +redirect: /documentation +--- \ No newline at end of file diff --git a/docs/documentation/native/usage.mdx b/docs/documentation/native/usage.mdx new file mode 100644 index 000000000..babce3b9e --- /dev/null +++ b/docs/documentation/native/usage.mdx @@ -0,0 +1,140 @@ +--- +title: Native automation - usage +--- + +Once set up, interacting with the native UI using Patrol is very easy! + +### Basics + +After you've got your `NativeAutomator` object, you simply call methods on it +and it does the magic. + +To tap on a native view (for example, a button in a WebView): + +```dart +await $.native.tap(Selector(text: 'Sign up for newsletter')); +``` + +To enter text into a native view (for example, a form in a WebView): + +```dart +await $.native.enterText( + Selector(text: 'Enter your email'), + text: 'charlie@root.me', +); +``` + +You can also enter text into n-th currently visible text field (counting from +0): + +```dart +await $.native.enterTextByIndex('charlie_root', index: 0); // enter username +await $.native.enterTextByIndex('ny4ncat', index: 1); // enter password +``` + +The above are the simplest, most common actions, but they already make it +possible to test scenarios that were impossible to test before, such as +WebViews. + + + To tap, enter text, or perform generally any UI interaction with a iOS app + that is not your Flutter app under test, you need to pass its bundle + identifier. For example, to tap on the plus button in the iPhone contacts app, + use: + +```dart +await $.native.tap( + Selector(text: 'Add'), + appId: 'com.apple.MobileAddressBook', +); +``` + + + +### Notifications + +To open the notification shade: + +```dart +await $.native.openNotifications(); +``` + +To tap on the second notification: + +```dart +await $.native.tapOnNotificationByIndex(1); +``` + +You can also tap on notification by its content: + +```dart +await $.native.tapOnNotificationBySelector( + Selector(textContains: 'Someone liked your recent post'), +); +``` + + + +### Permissions + +To handle the native permission request dialog: + +```dart +await $.native.grantPermissionWhenInUse(); +await $.native.grantPermissionOnlyThisTime(); +await $.native.denyPermission(); +``` + + + +If the permission request dialog visible is the location dialog, you can also +select the accuracy: + +```dart +await $.native.selectFineLocation(); +await $.native.selectCoarseLocation(); +``` + + + +The test will fail if the permission request dialog is not visible. You can +check if it is with: + +```dart +if (await $.native.isPermissionDialogVisible()) { + await $.native.grantPermissionWhenInUse(); +} +``` + +By default, `isPermissionDialogVisible()` waits for a short amount of time and +then returns `false` if the dialog is not visible. To increase the timeout: + +```dart +if (await $.native.isPermissionDialogVisible(timeout: Duration(seconds: 5))) { + await $.native.grantPermissionWhenInUse(); +} +``` + + + Patrol can handle permissions on iOS only if the device language is set to + English (preferably US). That's because there's no way to refer to a specific + view in a language-independent way (like resourceId on Android). + +If you want to handle permissions on iOS device with non-English locale, do it +manually: + +```dart +await $.native.tap( + Selector(text: 'Allow'), + appId: 'com.apple.springboard', +); +``` + + + +### More resources + +To see more integration tests demonstrating Patrol's various features, check out +our [example app][example_app]. + +[example_app]: https://github.com/leancodepl/patrol/tree/master/packages/patrol/example \ No newline at end of file diff --git a/docs/documentation/supported-platforms.mdx b/docs/documentation/supported-platforms.mdx new file mode 100644 index 000000000..f17f57d46 --- /dev/null +++ b/docs/documentation/supported-platforms.mdx @@ -0,0 +1,17 @@ +--- +title: Supported platforms +--- + +Patrol works on: + +- Android 5.0 (API 21) and newer, +- iOS 11 and newer, +- macOS 10.14 and newer (alpha support). + +Web, Windows, and Linux are not supported. + +On mobile platforms it works on both physical and virtual devices. + +If you want to check which native features are supported, see [feature parity]. + +[feature parity]: /documentation/native/feature-parity diff --git a/docs/documentation/tips-and-tricks.mdx b/docs/documentation/tips-and-tricks.mdx new file mode 100644 index 000000000..042c20949 --- /dev/null +++ b/docs/documentation/tips-and-tricks.mdx @@ -0,0 +1,169 @@ +--- +title: Tips and tricks +--- + +### Inspecting native view hierarchy + +It's hard to tap on or enter text into a view you don't know how to refer to. In +such situation we recommend doing a native view hierarchy dump and finding the +properties of the view you want to act on. + +**Android** + +First, perform a native view hierarchy dump using `adb`: + +``` +adb shell uiautomator dump +``` + +Then, copy the dump file from the device to your machine: + +``` +adb pull /sdcard/window_dump.xml . +``` + +**iOS** + +The easiest way to perform the native view hierarchy dumb on iOS is to use the +[idb] tool. + +Once you have [idb] installed, perform a dump: + +``` +idb ui describe-all +``` + +### Avoiding hardcoding credentials in tests + +It's a bad practice to hardcode data such as emails, usernames, and passwords in +test code. + +```dart +await $(#nameTextField).enterText('Bartek'); // bad! +await $(#passwordTextField).enterText('ny4ncat'); // bad as well! +``` + +To fix this, we recommend removing the hardcoded credentials from test code and +providing them through the environment: + +```dart +await $(#nameTextField).enterText(const String.fromEnvironment('USERNAME')); +await $(#passwordTextField).enterText(const String.fromEnvironment('PASSWORD')); +``` + +> Make sure that you're using `const` here because of [issue #55870][55870]. + +To set `USERNAME` and `PASSWORD`, use `--dart-define`: + +``` +patrol test --dart-define 'USERNAME=Bartek' --dart-define 'PASSWORD=ny4ncat' +``` + +Alternatively you can create a `.patrol.env` file in your project's root. Here's +an example: + +``` +$ cat .patrol.env +EMAIL=user@example.com +PASSWORD=ny4ncat +DISABLE_ANALYTICS=true +``` + +### Granting sensitive permission through the Settings app + +Some particularly sensitive permissions (such as access to background location +or controlling the Do Not Disturb feature) cannot be requested in the permission +dialog like most of the common permissions. Instead, you have to ask the user to +go to the Settings app and grant your app the permission you need. + +Testing such flows is not as simple as simply granting normal permission, but +it's totally possible with Patrol. + +Below we present you with a snippet that will make the built-in Camera app have +access to the Do Not Disturb feature on Android. Let's assume that the Settings +app on the device we want to run the tests on looks like this: + +![settings_screenshot] + +And here's the code: + +```dart +await $.native.tap(Selector(text: 'Camera')); // tap on the list tile +await $.native.tap(Selector(text: 'ALLOW')); // handle the confirmation dialog +await $.native.pressBack(); // go back to the app under test +``` + +Please note that the UI of the Settings app differs across operating systems, +their versions, and OEM flavors (in case of Android). You'll have to handle all +edge cases yourself. + +### Ignoring exceptions + +If an exception is thrown during a test, it is marked as failed. This is +Flutter's default behavior and it's usually good – after all, it's better to fix +the cause of a problem instead of ignoring it. + +That said, sometimes you do have a legitimate reason to ignore an exception. +This can be accomplished with the help of the +[WidgetTester.takeException()][take_exception] method, which returns the last +exception that occurred and removes it from the internal list of uncaught +exceptions, so that it won't mark the test as failed. To use it, just call it +once: + +```dart +final widgetTester = $.tester; +widgetTester.takeException(); +``` + +If more than a single exception is thrown during the test and you want to ignore +all of them, the below snippet should come in handy: + +```dart +var exceptionCount = 0; +dynamic exception = $.tester.takeException(); +while (exception != null) { + exceptionCount++; + exception = $.tester.takeException(); +} +if (exceptionCount != 0) { + $.log('Warning: $exceptionCount exceptions were ignored'); +} +``` + +### Handling permission dialogs before the main app widget is pumped + +Sometimes you might want to manually request permissions in the test before the +main app widget is pumped. Let's say that you're using the [geolocator] package: + +```dart +final permission = await Geolocator.requestPermission(); +final position = await Geolocator.getCurrentPosition(); +await $.pumpWidgetAndSettle(MyApp(position: position)); +``` + +In such case, first call the `requestPermission()` method, but instead of +awaiting it, assign the `Future` it returns to some `final`. Then, use Patrol to +grant the permissions, and finally, await the `Future` from the first step: + +```dart +// 1. request the permission +final permissionRequestFuture = Geolocator.requestPermission(); +// 2. grant the permission using Patrol +await $.native.grantPermissionWhenInUse(); +// 3. wait for permission being granted +final permissionRequestResult = await permissionRequestFuture; +expect(permissionRequestResult, equals(LocationPermission.whileInUse)); +final position = await Geolocator.getCurrentPosition(); +await $.pumpWidgetAndSettle(MyApp(position: position)); +``` + +See also: + +- [Patrol issue #628] + +[patrol issue #628]: https://github.com/leancodepl/patrol/issues/628 +[geolocator]: https://pub.dev/packages/geolocator +[idb]: https://github.com/facebook/idb +[settings_screenshot]: https://user-images.githubusercontent.com/10289319/194897313-849b8b84-df7a-4bf3-9b06-bb2782876d03.png +[take_exception]: https://api.flutter.dev/flutter/flutter_test/WidgetTester/takeException.html +[55870]: https://github.com/flutter/flutter/issues/55870 diff --git a/docs/documentation/write-your-first-test.mdx b/docs/documentation/write-your-first-test.mdx new file mode 100644 index 000000000..fc8344271 --- /dev/null +++ b/docs/documentation/write-your-first-test.mdx @@ -0,0 +1,473 @@ +--- +title: Write your first test +--- + +Patrol is a powerful, open-source testing framework created by LeanCode that enhances Flutter's testing capabilities by enabling interaction with native platform features directly in Dart. It allows to handle permission dialogs, notifications, WebViews, and device settings—features previously unavailable in standard Flutter tests, making it truly possible to test the whole app. + +This tutorial will take you through writing your first substantial Patrol test, interacting both with the Flutter app itself and also with native permission dialogs and notifications. + +Before writing any tests, make sure you [install the Patrol CLI]. Then just clone the following repository from GitHub to follow along. The app we’re going to be testing is fully functional and ready to be tested, with Patrol already configured. + + +To learn how to set up Patrol for your own project, check out the [Patrol Setup Docs]. + + + +Clone the [STARTER PROJECT] to follow along. + + + + +### App Walkthrough + +Before we can start writing automated Patrol tests, we need to know what the app does and to test it manually. Please, check out the video tutorial for a visual walkthrough. + +The first screen of our app is for signing in. It’s not using any actual sign-in provider but it only validates the email address and password. In order to successfully “sign in” and get to the home screen, we need to input a valid email and a password that’s at least 8 characters long. + + +You can test any real authentication providers that use WebView for signing in with the powerful Patrol native automation. + + +On the second screen, we’re greeted with a notifications permission dialog. Once we allow them, we can tap on the notification button in the app bar to manually trigger a local notification which will be displayed after 3 seconds both when the app is running in the foreground or in the background. + +Once we open the native notification bar and tap on the notification from our app, we’re gonna see a snackbar on the bottom saying _"Notification was tapped!”_ + +### Testing the “Happy Path” + +You’ve just seen the full walkthrough of the app, including errors that can show up if you input an invalid email or password. UI tests (integration tests), like the ones we’re going to write with Patrol, should only be testing the “happy path” of a UI flow. We only want them to fail if the app suddenly stops the user from doing what the app is for - in this case, that’s displaying a notification. Validation error messages are not “what the app is for”, they exist only to allow the user to successfully sign in with a proper email and password. That’s why we won’t be checking for them in the tests. + +### Writing the Test + +We have only one UI flow in this app, that is signing in, showing the notification and then tapping on that notification. This means, we’re going to have only a single test. Let’s create it in `/integration_test/app_test.dart`. + +Like any other test, we need to have a `main()` top-level function. Inside it we’re going to have our single `patrolTest` with a description properly describing what we’re about to test. An optional step is to set the frame policy to “fully live” to make sure all frames are shown, even those which are not pumped by our test code. Without it, we would see that our app stutters and animations are not played properly. + +```dart +import 'package:flutter_test/flutter_test.dart'; +import 'package:patrol/patrol.dart'; + +void main() { + patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + // Test code will go here + }, + ); +} +``` + +We could start writing the test right now and then re-run it from scratch every time we add a new line of test code by calling `patrol test --target integration_test/app_test.dart` but since we’re writing a Patrol test that runs on an Android or iOS device, constantly building the whole Flutter app is not time effective. Thankfully, Patrol offers a different approach - hot restarting the tests! We can run the command `patrol develop --target integration_test/app_test.dart` right now and anytime we add a new line of test code, we can just type “r” in the terminal to re-run the tests without the time-costly app building. Just make sure that you have an emulator running first - Patrol will select it automatically. + +First, we need to perform any initializations that need to happen before the app is run and pump the top-level widget of our app. We’re effectively doing what the `main` function inside `main.dart` does - this time not for just running the app as usual but for running an automated Patrol test. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + }, +); +``` + +Hot-restarting the test by typing “r” into the terminal won’t really do much since we’re not yet performing any user-like actions but you will at least see the sign in page for a brief moment before the test finishes. + +Let’s now perform some action! We know we have to sign in if we want to continue to the home screen. First, we have to type in both email and password. There are multiple ways to find widgets on the screen - by widget type, by text and lastly by key. + +Although it’s not the best practice, we’re first going to find the fields by type. Both are of type `TextFormField` but there are two of them on the screen so the following won’t work. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(TextFormField).enterText('test@email.com'); + await $(TextFormField).enterText('password'); + }, +); +``` + +That’s because finders always find the first matching widget so both the email address and password are entered into the same field - in this case, the email field. + +If multiple widgets on a screen match the finder, we can tell Patrol which one we want by specifying its index in the list of all found widgets from top to bottom like this: + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(TextFormField).enterText('test@email.com'); + await $(TextFormField).at(1).enterText('password'); + }, +); +``` + +We can use a text finder to tap on the “Sign in” button. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(TextFormField).enterText('test@email.com'); + await $(TextFormField).at(1).enterText('password'); + await $('Sign in').tap(); + }, +); +``` + +Hot-restarting the test will now take you all the way to the home page from which we will want to trigger the notification. + +As you can imagine though, using type and text finders in any app that’s just a bit more complex will result in a huge mess. The recommended approach is to always find your widgets by their `Key`. There are currently no keys specified for these widgets so let’s change that. In `sign_in_page.dart` pass in the following into the `TextFormFields` and `ElevatedButton`: + +```dart +class SignInPage extends StatelessWidget { + const SignInPage({super.key}); + + @override + Widget build(BuildContext context) { + return Scaffold( + ... + body: Padding( + padding: const EdgeInsets.all(16), + child: Form( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + TextFormField( + key: const Key('emailTextField'), + decoration: const InputDecoration( + labelText: 'Email', + ), + ... + ), + const SizedBox(height: 16), + TextFormField( + key: const Key('passwordTextField'), + decoration: const InputDecoration( + labelText: 'Password', + ), + ... + ), + const SizedBox(height: 16), + Builder(builder: (context) { + return ElevatedButton( + key: const Key('signInButton'), + ... + child: const Text('Sign in'), + ); + }), + ], + ), + ), + ), + ); + } +} +``` + +With the keys in place, we can now rewrite our test code to use `Key` finders. The simplest approach is to prefix the key’s value with a hash symbol. For this approach to work, your keys mustn’t contain any invalid characters such as spaces. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(#emailTextField).enterText('test@email.com'); + await $(#passwordTextField).enterText('password'); + await $(#signInButton).tap(); + }, +); +``` + +Looking at this test code again, it’s certain we can do better. Why? We’ve just added code duplication to our codebase! The key values in `sign_in_page.dart` and in `app_test.dart` are fully duplicated and if we change one, the other won’t be automatically updated, thus breaking our tests. + +That’s why production-grade apps should have a single source for all the `Keys` exposed as a global final variable inside `integration_test_keys.dart`. That’s going to look as follows if we already take into account the home page which we want to test next. + +```dart +import 'package:flutter/foundation.dart'; + +class SignInPageKeys { + final emailTextField = const Key('emailTextField'); + final passwordTextField = const Key('passwordTextField'); + final signInButton = const Key('signInButton'); +} + +class HomePageKeys { + final notificationIcon = const Key('notificationIcon'); + final successSnackbar = const Key('successSnackbar'); +} + +class Keys { + final signInPage = SignInPageKeys(); + final homePage = HomePageKeys(); +} + +final keys = Keys(); +``` + + +Feel free to put your page-specific key classes (e.g. `SignInPageKeys`) into separate files in more complex apps. + + +The updated `sign_in_page.dart` code will now look like this: + +```dart +class SignInPage extends StatelessWidget { + const SignInPage({super.key}); + + @override + Widget build(BuildContext context) { + return Scaffold( + ... + body: Padding( + padding: const EdgeInsets.all(16), + child: Form( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + TextFormField( + key: keys.signInPage.emailTextField, + decoration: const InputDecoration( + labelText: 'Email', + ), + ... + ), + const SizedBox(height: 16), + TextFormField( + key: keys.signInPage.passwordTextField, + decoration: const InputDecoration( + labelText: 'Password', + ), + ... + ), + const SizedBox(height: 16), + Builder(builder: (context) { + return ElevatedButton( + key: keys.signInPage.signInButton, + ... + child: const Text('Sign in'), + ); + }), + ], + ), + ), + ), + ); + } +} +``` + +The test code will now also use the `keys` global final variable instead of the hash symbol notation: + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(keys.signInPage.emailTextField).enterText('test@email.com'); + await $(keys.signInPage.passwordTextField).enterText('password'); + await $(keys.signInPage.signInButton).tap(); + }, +); +``` + +Hot-restarting the test won’t show any change in its functionality but it sure is more maintainable and easier to work with. + +### Home Page + +First, let’s add the keys we’ve already created to the `IconButton` and the `SnackBar` shown when the notification has been tapped. + +```dart +class HomePage extends StatefulWidget { + ... +} + +class _HomePageState extends State { + ... + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: const Text('Home'), + actions: [ + IconButton( + key: keys.homePage.notificationIcon, + icon: const Icon(Icons.notification_add), + onPressed: () { + triggerLocalNotification( + onPressed: () { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + key: keys.homePage.successSnackbar, + content: const Text('Notification was tapped!'), + ), + ); + }, + onError: () { + ... + }, + ); + }, + ), + ], + ), + ... + ); + } +} +``` + +The first thing the user sees when first navigating to the `HomePage` is a notifications permission dialog. We need to accept it from within the test. Patrol’s native automation makes this as easy as it gets. + + +Native automation allows you to interact with the OS your Flutter app is running on. Patrol currently supports Android, iOS and macOS native interactions. [Learn more from the docs] + + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(keys.signInPage.emailTextField).enterText('test@email.com'); + await $(keys.signInPage.passwordTextField).enterText('password'); + await $(keys.signInPage.signInButton).tap(); + await $.native.grantPermissionWhenInUse(); + }, +); +``` + +Hot-restarting the test will work wonderfully the first time, however, once the permission has already been granted, calling `grantPermissionWhenInUse()` will fail. This is not going to be an issue if you use Patrol as a part of your CI/CD process since everytime you test with Patrol there, the app will be built from scratch and no permission will be granted yet. But when we’re writing the test locally with `patrol develop` command, we need to make sure that the permission dialog is visible before trying to accept it. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(keys.signInPage.emailTextField).enterText('test@email.com'); + await $(keys.signInPage.passwordTextField).enterText('password'); + await $(keys.signInPage.signInButton).tap(); + if (await $.native.isPermissionDialogVisible()) { + await $.native.grantPermissionWhenInUse(); + } + }, +); +``` + + +It’s generally a bad practice to add any branching logic within your tests and you should be 100% certain that it cannot introduce any test flakiness before doing so. Checking if a permission dialog is visible is an example of a proper use of branching logic. + + +Next up, we want to tap on the notification icon button and then go to the device home screen to test the notification while the app is running in the background. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(keys.signInPage.emailTextField).enterText('test@email.com'); + await $(keys.signInPage.passwordTextField).enterText('password'); + await $(keys.signInPage.signInButton).tap(); + if (await $.native.isPermissionDialogVisible()) { + await $.native.grantPermissionWhenInUse(); + } + await $(keys.homePage.notificationIcon).tap(); + await $.native.pressHome(); + }, +); +``` + +Once we’re on the home screen, we want to open the notification shade and tap on the notification we get from our app. You can either tap on a notification by index or by finding a text. We know that the title of our notification is “Patrol says hello!” so let’s do the latter. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(keys.signInPage.emailTextField).enterText('test@email.com'); + await $(keys.signInPage.passwordTextField).enterText('password'); + await $(keys.signInPage.signInButton).tap(); + if (await $.native.isPermissionDialogVisible()) { + await $.native.grantPermissionWhenInUse(); + } + await $(keys.homePage.notificationIcon).tap(); + await $.native.pressHome(); + await $.native.openNotifications(); + await $.native.tapOnNotificationBySelector( + Selector(textContains: 'Patrol says hello!'), + ); + }, +); +``` + +Since the notification is delayed by 3 seconds, we have to provide a timeout that’s at least as long in order to wait for the notification to appear - 5 seconds should do the trick here. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + ... + await $.native.openNotifications(); + await $.native.tapOnNotificationBySelector( + Selector(textContains: 'Patrol says hello!'), + timeout: const Duration(seconds: 5), + ); + }, +); +``` + +Lastly, we want to check if the snackbar has been shown after tapping on a notification. We can call `waitUntilVisible()` after selecting it with its key. + +```dart +patrolTest( + 'signs in, triggers a notification, and taps on it', + framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, + ($) async { + initApp(); + await $.pumpWidgetAndSettle(const MainApp()); + await $(keys.signInPage.emailTextField).enterText('test@email.com'); + await $(keys.signInPage.passwordTextField).enterText('password'); + await $(keys.signInPage.signInButton).tap(); + if (await $.native.isPermissionDialogVisible()) { + await $.native.grantPermissionWhenInUse(); + } + await $(keys.homePage.notificationIcon).tap(); + await $.native.pressHome(); + await $.native.openNotifications(); + await $.native.tapOnNotificationBySelector( + Selector(textContains: 'Patrol says hello!'), + timeout: const Duration(seconds: 5), + ); + $(keys.homePage.successSnackbar).waitUntilVisible(); + }, +); +``` + +And just like that, we have now tested the whole flow of the app with Patrol! If any part of the logic breaks, this test will notify us about that sooner than our real users do and that’s what we’re all after! + +[install the Patrol CLI]: https://pub.dev/packages/patrol_cli#installation +[Patrol Setup Docs]: /documentation +[STARTER PROJECT]: https://github.com/ResoCoder/patrol-basics-tutorial +[Learn more from the docs]: /documentation/native/overview diff --git a/docs/effective-patrol.mdx b/docs/effective-patrol.mdx index ff368e539..7254dbdb5 100644 --- a/docs/effective-patrol.mdx +++ b/docs/effective-patrol.mdx @@ -1,192 +1,3 @@ --- -title: Effective Patrol +redirect: /documentation/effective-patrol --- - -# Effective Patrol - -Over the past months, we've written many Patrol tests and often learned the hard -way what works well and what doesn't. We're sharing our findings hoping that -they'll help you write robust tests. - -This document follows [RFC 2119][rfc2119]. - -### PREFER using keys to find widgets - -Patrol's custom finders are very powerful, and you might often be inclined to -find the widget you want in a variety of ways. While we're encouraging you to -explore and play with Patrol's custom finders, we are quite confident that keys -are the best way to find widgets. - -**Why not strings?** - -At first, strings might seem like a good way to find widgets. - -They'll get increasingly annoying to work with as your app grows and changes, -for example, when the strings in your app change. - -Using strings stops making any sense when you have more than 1 language in your -app. Using strings in such case is asking for trouble. - -**Why not classes?** - -There are 2 problems with using classes. - -First is that they hurt your test's readability. You want to tap on _the_ login -button or enter text into _the_ username field. You don't want to tap on, say, -the third button and enter text into the second text field. - -The second problem is that classes are almost always an implementation detail. -As a tester, you shouldn't care if something is a `TextButton` or an -`OutlineButton`. You care that it is _the_ login button, and you want to tap on -it. In most cases, that login button should have a key. - -Let's consider this simple example: - -```dart -await $(LoginForm).$(Button).at(1).tap(); // taps on the second ("login") button -``` - -This works, but the code is not very self-explanatory. To make it understandable -at glance, you had to add a comment. - -But if you assigned a key to the login button, the above could be simplified to: - -```dart -await $(#loginButton).tap(); -``` - -Much better! - -Let's see another example: - -```dart -await $(Select).tap(); // taps on the first Select -``` - -If the type parameter is changed from `String` to, for example, some specialized -`PersonData` model, that finder won't find anything. You'd have to update it to: - -```dart -await $(Select).tap(); -``` - -You had to change your test, even though nothing changed from the user's -perspective. This is usually a sign that you rely too much on classes to find -widgets. - -This whole section could be summed up to the simple maxim: - -> Have tester's mindset. - -Treat your finders as if they were the tester's eyes. - -### CONSIDER having a file where all keys are defined - -The number of keys will get bigger as your app grows and you write more tests. -To keep track of them, it's a good idea to keep all keys in, say, -`lib/keys.dart` file. - -```dart title="lib/keys.dart" -import 'package:flutter/foundation.dart'; - -typedef K = Keys; - -class Keys { - const Keys(); - - static const usernameTextField = Key('usernameTextField'); - static const passwordTextField = Key('passwordTextField'); - static const loginButton = Key('loginButton'); - static const forgotPasswordButton = Key('forgotPasswordButton'); - static const privacyPolicyLink = Key('privacyPolicyLink'); -} -``` - -Then you can use it in your app's and tests' code: - -```dart title="In app UI code" -@override -Widget build(BuildContext context) { - return Column( - children: [ - /// some widgets - TextField( - key: K.usernameTextField, - // some other TextField properties - ), - // more widgets - ], - ); -} -``` - -```dart title="In app test code" -void main() { - patrolTest('logs in', (PatrolIntegrationTester $) { - // some code - await $(K.usernameTextField).enterText('CoolGuy'); - // more code - }); -} -``` - -This is a good way to make sure that the same keys are used in app and tests. No -more typos! - -### PREFER having one test path - -Good tests test one feature, and test it well (this applies to all tests, not -only Patrol tests). This is often called the "main path". Try to introduce as -little condional logic as possible to help keep the main path straight. In -practice, this usually comes down to having as few `if`s as possible. - -Keeping your test code simple and to the point will also help you in debugging -it. - -### DO add a good test description explaining the test's purpose - -If your app is non-trivial, your Patrol test will become long pretty quickly. -You may be sure now that you'll always remember what the 200 line long test -you've just written does and are (rightfully) very proud of it. - -Believe us, in 3 months you will not remember what your test does. This is why -the first argument to `patrolTest` is the test description. Use it well! - -```dart -// GOOD -import 'package:awesome_app/main.dart'; -import 'package:patrol/patrol.dart'; - -void main() { - patrolTest( - 'signs up for the newsletter and receives a reward', - ($) async { - await $.pumpWidgetAndSettle(AwesomeApp()); - - await $(#phoneNumber).enterText('800-555-0199'); - await $(#loginButton).tap(); - - // more code - }, - ); -} -``` - -```dart -// BAD -void main() { - patrolTest( - 'test', - ($) async { - await $.pumpWidgetAndSettle(AwesomeApp()); - - await $(#phoneNumber).enterText('800-555-0199'); - await $(#loginButton).tap(); - - // more code - }, - ); -} -``` - -[rfc2119]: https://www.ietf.org/rfc/rfc2119.txt diff --git a/docs/finders/advanced.mdx b/docs/finders/advanced.mdx index 49b5a0915..9d73494a8 100644 --- a/docs/finders/advanced.mdx +++ b/docs/finders/advanced.mdx @@ -1,233 +1,3 @@ --- -title: Patrol finders - advanced +redirect: /documentation/finders/advanced --- - -# Advanced - -We aim to make Patrol as simple as possible, but there are still a few matters -that we feel require some more attention. We'll explain them in this section. - -### How is Patrol's `tap()` different from Flutter's `tap()`? - -Let's consider this test, written without Patrol: - -```dart -await tester.tap(find.byKey(Key('addComment')).first); -await tester.pumpAndSettle(); -``` - -This code: - -1. Immediately atttempts to find the first widget with the `addComment` key -2. After finding the widget, it immediately attempts to tap on it - -This is the default behavior, but in our experience, it's often a source of -flakiness. For example, the widget having `addComment` key might not be visible -at the time when the finder is run. This usually doesn't means that the test -should fail. Probably an HTTP request was made to fetch the post, and when the -fetching is done, the widget having `addComment` key will show up. - -To achieve this behavior, you'd have to do: - -```dart -while (find.byKey(Key('addComment')).first.evaluate().isEmpty) { - await tester.pump(Duration(milliseconds: 100)); -} - -await tester.tap(find.byKey(Key('addComment')).first); -await tester.pumpAndSettle(); -``` - -Our tiny example got really big, but it's still got two problems. - -1. If something goes wrong and `addComment` never shows up, we'll keep waiting - indefinitely. - -2. The widget with `addComment` key might be present in the widget tree, but - still not be visible to the user. By default, Flutter's default - `WidgetTester` doesn't care. This is almost never desirable. - -Fortunately, you don't have to overcome these problems. Patrol already did it! - -Below is the same test, with all the above problems fixed, written with Patrol's -custom finders: - -```dart -await $(#addComment).tap(); -``` - -This code: - -1. Attempts to find the first widget with `addComment` that is visible on - screen. If it's not found immediately, it keeps trying until it finds it, or - throws an exception if timeout. -2. Taps on it. - -The timeout can be configured globally: - -```dart -patrolWidgetTest( - 'logs in successfully', - config: PatrolTestConfig(findTimeout: Duration(seconds: 10)), - ($) async { - // your test code - }, -); -``` - -You can also change the timeout ad-hoc: - -```dart -await $(#addComment).tap(findTimeout: Duration(seconds: 30)); -``` - -### You gotta pump it up! But which one to use? - -In Flutter, "pumping" means rendering frames to the screen. - -If there are no frames to pump, no animations are pending, which usually means -that the next action during the test can be executed. It is an equivalent of -what a human tester would do while testing an app - they would wait until the -app's state stabilizes after they've done something. For example, they tap on a -button and get redirected to another screen, but the data that will be shown -there hasn't been loaded yet. In such a case, a human tester waits until a -loader (or other animation) finishes. Pumping mechanism does exactly this - it -renders consecutive frames on the screen. For how long, exactly? Usually, we -want to pump frames as long as they come. That's what [pumpAndSettle()] does. - -`pumpAndSettle()` method is called by default inside all actions that can be -performed while testing - tapping, scrolling, entering text, and so on. You can -change that by setting the `settlePolicy` argument: - -```dart -await $('Delete account').tap(settlePolicy: SettlePolicy.settle); -await $('Confirm').tap(settlePolicy: SettlePolicy.pump); -``` - -`SettlePolicy` is an `enum` with 3 values. The default is `SettlePolicy.settle` -but you can change it to `pump` or `trySettle`. Those values map to methods like this: - - `noSettle` -> `pump()`, - - `trySettle` -> `pumpAndTrySettle()`, - - `settle` -> `pumpAndSettle()`. - -While `settle` and `pump` simply refer to Flutter's built-in methods, -`trySettle` is available only in Patrol. How is it different from other ones? - -`pumpAndTrySettle()` is pretty much like `pumpAndSettle()`, the only difference -is that `pumpAndSettle()` throws an exception, if there were still new frames to -render after sonme defined timeout, while `pumpAndTrySettle()` does not. That's -why it has "`try`" in it's name. - -When to use this new pumping method? Let us picture a scenario, in which we have -to deal with some animations. Let's say, that your app has some endless -animations, e.g. on a homescreen, to keep user's attention. You'd like to wait -for some things to happen, but using `pumpAndSettle`, you'll keep getting an -exception, because after some time, defined by `timeout`, there will be still -new frames to render. On the other hand, you still want to pump frames for some -time - if you didn't, the screen you want to interact with might not be rendered -yet, or it would have some widgets missing or data not yet loaded. - -So, we decided to add a way to try settle - pump frames for some time (10 -seconds by default), but if after that time there is still something new to -render - do nothing and continue the test. - -We recommend using `pumpAndTrySettle()`, because it works with both kinds of -animations - finite and infinite. This settle policy will be new default in -future Patrol releases. - -### How does `scrollTo()` work? - -The `scrollTo()` method is simple to use, yet very powerful. Here's how you use -it to scroll to and tap on the first widget with the `"Delete account"` text: - -```dart -await $('Delete account').scrollTo().tap(); -``` - -And here's how `scrollTo()` works: - -1. Waits for at least 1 [Scrollable] widget (or whatever you provided in - `view` argument) to become visible -2. Scrolls this widget in its scrolling direction until the target - widget becomes visible -3. If the target widget becomes visible within timeout, it finishes, otherwise - it throws an exception - -Most of the time, you use `scrollTo()` and it just works, but there's 1 -important thing to keep in mind when using `scrollTo()`: - -**`scrollTo()`, by default, scrolls the first `Scrollable` widget** - -This default is reasonable and what you want most of the time. Unfortunately, -this behavior can sometimes cause problems in more complicated UIs, where more -than a single `Scrollable` widget is visible at the same time. In such cases we -strongly recommend explicitly specifying the `view` that `scrollTo()` -should scroll, to avoid the problem of the target widget never becoming visible -because the wrong widget was scrolled. - -To demonstrate this problem, let's consider this very simple app: - -```dart -class App extends StatelessWidget { - App({super.key}); - - @override - Widget build(BuildContext context) { - return MaterialApp( - home: Scaffold( - body: Column( - children: [ - Expanded(child: ListView(key: Key('listView1'))), - Expanded( - child: ListView.builder( - key: Key('listView2'), - itemCount: 101, - itemBuilder: (context, index) => Text('index: $index'), - ), - ), - ], - ), - ), - ); - } -} -``` - -Now let's say that you're writing a test and want to scroll to and tap on the -first widget with the `"index: 100"` text (that is the last `Text` widget built -by the second `ListView` widget): - -There's a high chance that you'd write this: - -```dart -await $('index: 100').scrollTo().tap(); -``` - -Unfortunately, running this test gives the `pumpAndSettle timed out` error. -That's because the `scrollTo()` was trying to scroll the first visible -`Scrollable` widget, which happens to be the first `ListView` (the one with -`listView1` key and no children). - -To fix this problem, you have to explicitly specify which `Scrollable` you want -to use: - -```dart -await $('index: 100').scrollTo(view: $(#listView2).$(Scrollable)).tap(); -``` - -The above snippet will scroll the second `Scrollable` and find the widget with -`"index: 100"` text. - -**Why so verbose?** - -You might be wondering why `scrollTo(view: $(#listView2))` is not enough? -Why is it needed to look for a `Scrollable` widget inside the widget with the -`listView2` key? - -This is because the [ListView] widget doesn't extend [Scrollable] – instead, it -builds a subclass of [Scrollable] itself. [This is a known Flutter -problem](https://github.com/flutter/flutter/issues/88762). - -[listview]: https://api.flutter.dev/flutter/widgets/ListView-class.html -[scrollable]: https://api.flutter.dev/flutter/widgets/Scrollable-class.html -[pumpAndSettle()]: https://api.flutter.dev/flutter/flutter_test/WidgetTester/pumpAndSettle.html diff --git a/docs/finders/finders-setup.mdx b/docs/finders/finders-setup.mdx index 412a0067e..683629396 100644 --- a/docs/finders/finders-setup.mdx +++ b/docs/finders/finders-setup.mdx @@ -1,85 +1,3 @@ --- -title: Using Patrol finders in widget tests +redirect: /documentation/finders/finders-setup --- - -# Using Patrol finders in widget tests - -Since `patrol_finders` is a separate package, referenced in `patrol` package, -you can use it in your widget or golden tests, without depending on `patrol`. -Below you can find a short tutorial on how to use `$` in already existing widget tests. - -### Install - -First, add the `patrol_finders` package as a `dev_dependency` in your app's -`pubspec.yaml`. You can do this by executing the following command in the app's -directory: - -```console -flutter pub add patrol_finders --dev -``` - -### Use - - - Our custom finders depend only on Flutter itself. This means you can use them - no matter if you're writing a mobile, web, desktop, or embedded app! - - -To use Patrol finders in your test files, first import it: - -```dart -import 'package:patrol_finders/patrol_finders.dart'; -``` - -Once imported, you can use it in widget tests: - -```dart title="test/widget_test_with_patrol_finders.dart" -void main() { - testWidgets( - 'counter is incremented when plus button is tapped', - (WidgetTester tester) async { - PatrolTester $ = PatrolTester( - tester: tester, - config: PatrolTesterConfig(), - ); - await $.pumpWidget(const MyApp()); - - expect($('0'), findsOneWidget); - expect($('-1'), findsNothing); - - await $(Icons.remove).tap(); - - expect($('0'), findsNothing); - expect($('-1'), findsOneWidget); - }, - ); -} -``` - -Or you can use our wrapper on `testWidgets` method, which initializes -`PatrolTester` object for you. - -```dart title="test/patrol_widget_test.dart" -void main() { - patrolWidgetTest( - 'counter is incremented when plus button is tapped', - (PatrolTester $) async { - await $.pumpWidget(const MyApp()); - - expect($('0'), findsOneWidget); - expect($('-1'), findsNothing); - - await $(Icons.remove).tap(); - - expect($('0'), findsNothing); - expect($('-1'), findsOneWidget); - }, - ); -} -``` - -To run the test, simply execute: - -```console -flutter test -``` diff --git a/docs/finders/overview.mdx b/docs/finders/overview.mdx index 1b138c91d..facb9db4f 100644 --- a/docs/finders/overview.mdx +++ b/docs/finders/overview.mdx @@ -1,60 +1,3 @@ --- -title: Patrol finders - overview +redirect: /documentation/finders/overview --- - -# Custom finders overview - -Flutter's finders are powerful, but not very intuitive to use. - -We took them and made something awesome. - -Thanks to Patrol's custom finders, you'll take your tests from this: - -```dart -testWidgets('signs up', (WidgetTester tester) async { - await tester.pumpWidget(AwesomeApp()); - await tester.pumpAndSettle(); - - await tester.enterText( - find.byKey(Key('emailTextField')), - 'charlie@root.me', - ); - await tester.pumpAndSettle(); - - await tester.enterText( - find.byKey(Key('nameTextField')), - 'Charlie', - ); - await tester.pumpAndSettle(); - - await tester.enterText( - find.byKey(Key('passwordTextField')), - 'ny4ncat', - ); - await tester.pumpAndSettle(); - - await tester.tap(find.byKey(Key('termsCheckbox'))); - await tester.pumpAndSettle(); - - await tester.tap(find.byKey(Key('signUpButton'))); - await tester.pumpAndSettle(); - - expect(find.text('Welcome, Charlie!'), findsOneWidget); -}); -``` - -to this: - -```dart -patrolWidgetTest('signs up', (PatrolTester $) async { - await $.pumpWidgetAndSettle(AwesomeApp()); - - await $(#emailTextField).enterText('charlie@root.me'); - await $(#nameTextField).enterText('Charlie'); - await $(#passwordTextField).enterText('ny4ncat'); - await $(#termsCheckbox).tap(); - await $(#signUpButton).tap(); - - await $('Welcome, Charlie!').waitUntilVisible(); -}); -``` diff --git a/docs/finders/usage.mdx b/docs/finders/usage.mdx index 32005fd4c..e2654cab2 100644 --- a/docs/finders/usage.mdx +++ b/docs/finders/usage.mdx @@ -1,278 +1,3 @@ --- -title: Patrol finders - usage +redirect: /documentation/finders/usage --- - -# Usage - -This page introduces Patrol finder system. Let's get our hands dirty -and find some widgets! - -### Finding widgets - -Let's say you want to find some `Text` widget – nothing easier than that! - -```dart -find.byType(Text); -``` - -Using Patrol finder, you'd write the above as: - -```dart -$(Text); -``` - -Or let's find a `Text` widget with a specific text: - -```dart -find.text('Subscribe'); -``` - -Using Patrol finder, you'd write the above as: - -```dart -$('Subscribe'); -``` - -Worth mentioning is also `Key`. The below lines are equivalent: - -```dart -find.byKey(Key('loginButton')); -$(Key('loginButton')); -$(#loginButton); -``` - -For those wondering what is that `#` thing – it's a -[Symbol](https://api.dart.dev/dart-core/Symbol-class.html)! Yes, we're Dart -(ab)users. - -All the types that can be passed to `$` are [listed here][doc]. - -### Making assertions - -Creating a finder doesn't do anything – it just _is_. Let's put them to use and -write a few simple assertions. - -Here's how you can make sure that a widget with text `Log in` exists in the -widget tree: - -```dart -expect(find.text('Log in'), findsOneWidget); -``` - -With our Patrol finders, you'd write the above as: - -```dart -expect($('Log in'), findsOneWidget); -``` - -Alternatively, you could also use the `exists` getter, which returns true if the -finder finds at least 1 widget: - -```dart -expect($('Log in').exists, equals(true)); -``` - -We can also make sure that no widget exists, or that a particular number of -widgets exist: - -```dart -expect(find.text("Can't touch this"), findsNothing); -expect(find.byType(Card), findsNWidgets(3)); -``` - -The above expressed with Patrol finders: - -```dart -expect($("Can't touch this"), findsNothing); -expect($(Card), findsNWidgets(3)); -``` - -You could alternatively write the first line as: - -```dart -expect($("Can't touch this").exists, equals(false)); -``` - -[//]: # (not true, TODO: rewrite) -It's important to note that Flutter's default finder functions, such as -[findsNothing] and [findsOneWidget], check if the widget is present in the -widget tree, not if it is visible to the user, which is usually not what we're -interested in. - -To check if the finder finds at least 1 _visible_ widget, use the `visible` -getter: - -```dart -expect($('Log in').visible, equals(true)); -``` - -And to wait for at least 1 widget with the "Log in" text to become visible: - -```dart -await $('Log in').waitUntilVisible(); -``` - -### Performing actions - -Finding widgets alone is cool, but what's even cooler is being able to tap on -them! Let's tap on the first "Subscribe" text: - -```dart -await tester.tap(find.text('Subscribe').first); -``` - -It's usually a good practice to use `first`, because if there were multiple -"Subscribe" texts, `tap()` would throw an exception. - -With Patrol, you get concise code, but you preserve the flexibility: - -```dart -await $('Subscribe').tap(); -``` - -What's very cool about Patrol's `tap()` is that it doesn't immediately fail if -the finder finds no visible widgets – instead, it waits for some time (which you -can specify globally in [PatrolTesterConfig] or as argument to the `tap()` -method) and taps on the first widget as soon as it becomes visible. This lets -you get rid of fixed timeouts and test your app just like a real user would. - -If you wanted to tap on the third "Subscribe" text, you'd do: - -```dart -await $('Subscribe').at(2).tap(); -``` - -And if the "Subscribe" text was in a [Scrollable] widget, such as -[SingleChildScrollView] or [ListView], and you want to make sure that it is -visible (so you can `tap()` on it), you can scroll to it very easily: - -```dart -await $('Subscribe').scrollTo().tap(); -``` - -### Going deeper - -But hey, these were very simple examples. In real apps, unfortunately, finding -widgets is not that easy. - -Often, you'll need to tap on a widget which is in some other widget. - -```dart -await tester.tap( - find.descendant( - of: find.byType(ListView), - matching: find.text('Subscribe'), - ).first -); -``` - -Flutter's finders are starting to grow, while Patrol stays lean: - -```dart -await $(ListView).$('Subscribe').tap(); -``` - -Now, we also make sure that the `Subscribe` text is in a `ListTile`: - -```dart -await tester.tap( - find.descendant( - of: find.byType(ListView), - matching: find.descendant( - of: find.byType(ListTile), - matching: find.text('Subscribe'), - ), - ).first -); -``` - -Hey, this is starting to look complex! Fortunately, you have Patrol: - -```dart -await $(ListView).$(ListTile).$('Subscribe').tap(); -``` - -Sometimes, you might want to perform a lookahead check. Let's say that you want -tap on the first widget with the `Key('learnMore')` that is a descendant of some -`ListTile`, but that `ListTile` must also have the `Text` descendant with the -`Activated` text. - -If you were to express the above as a `Finder`, you'd get: - -```dart -await tester.tap( - find.ancestor( - of: find.text('Activated'), - matching: find.descendant( - of: find.byType(ListTile), - matching: find.byKey(Key('learnMore')), - ), - ).first -); -``` - -With the help of Patrol's custom finders, it's much easier: - -```dart -await $(ListTile).containing('Activated').$(#learnMore).tap(); -``` - -Sometimes, however, the logic required to find a widget cannot be expressed by -the descendant/ancestor relationship like above. In situations like this, -when all ways of finding widgets known to you fail, Patrol has an ace up its -sleeve: the [which()] method. You can use it to find widgets by their -properties. A few examples include: -. - -- entering a text into a text field with no text entered: - - ```dart - await $(#cityTextField) - .which((widget) => widget.controller.text.isNotEmpty) - .enterText('Warsaw, Poland'); - ``` - -- asserting that the icon has the correct color: - - ```dart - await $(Icons.error) - .which((widget) => widget.color == Colors.red) - .waitUntilVisible(); - ``` - -- asserting that the button is disabled and has the correct color - - ```dart - await $('Delete account') - .which((button) => !button.enabled) - .which( - (btn) => btn.style?.backgroundColor?.resolve({}) == Colors.red, - ) - .waitUntilVisible(); - - ``` - -### Falling back - -What's cool about Patrol is that it builds on top of `flutter_test` instead of -replacing it. This means that you can freely mix Patrol's finders with finders -from `flutter_test`, `PatrolTester` with `WidgetTester`, and so on. - -Here's how you can access the default `WidgetTester`: - -```dart -patrolWidgetTest('adds comment', (PatrolTester $) async { - final WidgetTester tester = $.tester; - - await tester.enterText(find.byKey(Key('commentTextField')), 'Very nice!'); -}); -``` - -[doc]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/createFinder.html -[which()]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/PatrolFinder/which.html -[findsNothing]: https://api.flutter.dev/flutter/flutter_test/findsNothing-constant.html -[findsOneWidget]: https://api.flutter.dev/flutter/flutter_test/findsOneWidget-constant.html -[PatrolTesterConfig]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/PatrolTesterConfig-class.html -[singlechildscrollview]: https://api.flutter.dev/flutter/widgets/SingleChildScrollView-class.html -[listview]: https://api.flutter.dev/flutter/widgets/ListView-class.html -[Scrollable]: https://api.flutter.dev/flutter/widgets/Scrollable-class.html diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index 3c2f45dc1..222a85eb0 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -1,588 +1,3 @@ --- -title: Getting started +redirect: /documentation/install --- - -# Getting started - -Check out our video version of this tutorial on YouTube! - - - - - If you want to use Patrol finders in your existing widget or golden - tests, go to [Using Patrol finders in widget tests]. - - - - In this tutorial, we are using example app, which has package name - `com.example.myapp` on Android, bundle id `com.example.MyApp` on iOS, - `com.example.macos.MyApp` on macOS and `My App` name on all platforms. - Replace any occurences of those names with proper values. - - - - Support for macOS is in alpha stage. Please be aware that some features - may not work as expected. There is also no native automation support - for macOS yet. If you encounter any issues, please report them on - GitHub. - - -### Add dependency on `patrol` - -If you haven't already, add a dependency on the `patrol` package in the -`dev_dependencies` section of `pubspec.yaml`. `patrol` package requires -Android SDK version 21 or higher. - -``` -flutter pub add patrol --dev -``` - -### Configure Patrol in `pubspec.yaml` - -Create `patrol` section in your `pubspec.yaml`: - -```yaml title="pubspec.yaml" -dependencies: - # ... - -dev_dependencies: - # ... - -patrol: - app_name: My App - android: - package_name: com.example.myapp - ios: - bundle_id: com.example.MyApp - macos: - bundle_id: com.example.macos.MyApp -``` - -If you don't know where to get `package_name` and `bundle_id` from, see the [FAQ] section. - -### Install `patrol_cli` - -[Patrol CLI] (command-line interface) is a small program that enables running -Patrol UI tests. It is necessary to run UI tests (`flutter test` won't work! [Here's why]). - -1. Install `patrol_cli` executable: - - ``` - dart pub global activate patrol_cli - ``` - - - Make sure to add `patrol` to your `PATH` environment variable. - It's explained how to do it in the [README](https://pub.dev/packages/patrol_cli#installation). - - -2. Verify that installation was successful and your environment is set up properly: - - ``` - patrol doctor - ``` - Example output: - ``` - Patrol CLI version: 2.3.1+1 - Android: - • Program adb found in /Users/username/Library/Android/sdk/platform-tools/adb - • Env var $ANDROID_HOME set to /Users/username/Library/Android/sdk - iOS / macOS: - • Program xcodebuild found in /usr/bin/xcodebuild - • Program ideviceinstaller found in /opt/homebrew/bin/ideviceinstaller - ``` - Be sure that for the platform you want to run the test on, all the checks are green. - -Patrol CLI invokes the Flutter CLI for certain commands. To override the command used, -pass the `--flutter-command` argument or set the `PATROL_FLUTTER_COMMAND` environment -variable. This supports FVM (by setting the value to `fvm flutter`), puro (`puro flutter`) -and potentially other version managers. - -### Integrate with native side - -The 3 first steps were common across platforms. The rest is platform-specific. - -Psst... Android is a bit easier to set up, so we recommend starting with it! - - - - 1. Go to **android/app/src/androidTest/java/com/example/myapp/** in your project - directory. If there are no such folders, create them. **Remember to replace - `/com/example/myapp/` with the path created by your app's package name.** - - 2. Create a file named `MainActivityTest.java` and copy there the code below. - - ```java title="MainActivityTest.java" - package com.example.myapp; // replace "com.example.myapp" with your app's package - - import androidx.test.platform.app.InstrumentationRegistry; - import org.junit.Test; - import org.junit.runner.RunWith; - import org.junit.runners.Parameterized; - import org.junit.runners.Parameterized.Parameters; - import pl.leancode.patrol.PatrolJUnitRunner; - - @RunWith(Parameterized.class) - public class MainActivityTest { - @Parameters(name = "{0}") - public static Object[] testCases() { - PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); - // replace "MainActivity.class" with "io.flutter.embedding.android.FlutterActivity.class" - // if in AndroidManifest.xml in manifest/application/activity you have - // android:name="io.flutter.embedding.android.FlutterActivity" - instrumentation.setUp(MainActivity.class); - instrumentation.waitForPatrolAppService(); - return instrumentation.listDartTests(); - } - - public MainActivityTest(String dartTestName) { - this.dartTestName = dartTestName; - } - - private final String dartTestName; - - @Test - public void runDartTest() { - PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); - instrumentation.runDartTest(dartTestName); - } - } - ``` - - 3. Go to the **build.gradle** file, located in **android/app** folder in your - project directory. - - 4. Add these 2 lines to the `defaultConfig` section: - - ```groovy - testInstrumentationRunner "pl.leancode.patrol.PatrolJUnitRunner" - testInstrumentationRunnerArguments clearPackageData: "true" - ``` - - 5. Add this section to the `android` section: - - ```groovy - testOptions { - execution "ANDROIDX_TEST_ORCHESTRATOR" - } - ``` - - 6. Add this line to `dependencies` section: - - ```groovy - androidTestUtil "androidx.test:orchestrator:1.4.2" - ``` - - - Bear in mind that ProGuard can lead to some problems if not well configured, potentially causing issues such as `ClassNotFoundException`s. - Keep all the Patrol packages or disable ProGuard in `android/app/build.gradle`: - ```groovy - ... - buildTypes { - release { - ... - } - debug { - minifyEnabled false - shrinkResources false - } - } - ``` - - - - - - - 1. Open `ios/Runner.xcworkspace` in Xcode. - - 2. Create a test target if you do not already have one (see the screenshot below - for the reference). Select `File > New > Target...` and select `UI Testing Bundle`. - Change the `Product Name` to `RunnerUITests`. Set the `Organization Identifier` - to be the same as for the `Runner` (no matter if you app has flavors or not). - For our example app, it's `com.example.MyApp` just as in the `pubspec.yaml` file. - Make sure `Target to be Tested` is set to `Runner` and language is set to `Objective-C`. - Select `Finish`. - - ![Xcode iOS test target](/assets/ios_test_target.png) - - 3. 2 files are created: `RunnerUITests.m` and `RunnerUITestsLaunchTests.m`. - Delete `RunnerUITestsLaunchTests.m` **through Xcode** by clicking on it and - selecting `Move to Trash`. - - 4. Make sure that the **iOS Deployment Target** of `RunnerUITests` within the - **Build Settings** section is the same as `Runner`. - The minimum supported **iOS Deployment Target** is `11.0`. For the - [example app](https://github.com/leancodepl/patrol/tree/master/packages/patrol/example), - we set it to `13.0` because it's required by the app dependencies. - - ![Xcode iOS deployment target](/assets/ios_deployment_target.png) - - ![Xcode iOS deployment target 2](/assets/ios_deployment_target_2.png) - - 5. Replace contents of `RunnerUITests.m` file with the following: - - ```objective-c title="ios/RunnerUITests/RunnerUITests.m" - @import XCTest; - @import patrol; - @import ObjectiveC.runtime; - - PATROL_INTEGRATION_TEST_IOS_RUNNER(RunnerUITests) - ``` - - Add the newly created target to `ios/Podfile` by embedding in the existing - `Runner` target. - - ```ruby title="ios/Podfile" - target 'Runner' do - # Do not change existing lines. - ... - - target 'RunnerUITests' do - inherit! :complete - end - end - ``` - - 6. Create an empty file `integration_test/example_test.dart` in the root of your Flutter project. From the command line, run - the following command and make sure it completes with no errors: - - ``` - $ flutter build ios --config-only integration_test/example_test.dart - ``` - - 7. Go to your `ios` directory and run: - - ``` - $ pod install --repo-update - ``` - - 8. Open your Xcode project and Make sure that for each build configuration, - the `RunnerUITests` have the same Configuration Set selected as the `Runner`: - - ![Xcode config setup](/assets/ios_runner_configs.png) - - 9. Go to **RunnerUITests** -> **Build Phases** and add 2 new "Run Script Phase" Build Phases. - Name them `xcode_backend build` and `xcode_backend embed_and_thin`. - - ![Xcode config setup](/assets/ios_runner_build_phases_create.png) - - 10. Arrange the newly created Build Phases in the order shown in the screenshot below. - - ![Xcode config setup](/assets/ios_runner_build_phases.png) - - 11. Paste this code into the `xcode_backend build` Build Phase: - - ``` - /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh" build - ``` - - 12. Paste this code into the `xcode_backend embed_and_thin` Build Phase: - - ``` - /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh" embed_and_thin - ``` - - 13. Xcode by default also enables a "parallel execution" setting, which - breaks Patrol. Disable it **for all schemes** (if you have more than one): - - - - 14. Go to **RunnerUITests** -> **Build Settings**, search for **User Script Sandboxing** - and make sure it's set to **No**. - - - - - - 1. Open `macos/Runner.xcworkspace` in Xcode. - - 2. Create a test target if you do not already have one via `File > New > Target...` - and select `UI Testing Bundle`. Change the `Product Name` to `RunnerUITests`. Make - sure `Target to be Tested` is set to `Runner` and language is set to `Objective-C`. - Select `Finish`. - - 3. 2 files are created: `RunnerUITests.m` and `RunnerUITestsLaunchTests.m`. - Delete `RunnerUITestsLaunchTests.m` **through Xcode**. - - 4. Make sure that the **macOS Deployment Target** of `RunnerUITests` within the - **Build Settings** section is the same as `Runner`. - The minimum supported **macOS Deployment Target** is `10.14`. - - ![Xcode macOS deployment target](/assets/macos_deployment_target.png) - - ![Xcode macOS deployment target 2](/assets/macos_deployment_target_2.png) - - 5. Replace contents of `RunnerUITests.m` file with the following: - - ```objective-c title="macos/RunnerUITests/RunnerUITests.m" - @import XCTest; - @import patrol; - @import ObjectiveC.runtime; - - PATROL_INTEGRATION_TEST_MACOS_RUNNER(RunnerUITests) - ``` - - Add the newly created target to `macos/Podfile` by embedding in the existing - `Runner` target. - - ```ruby title="macos/Podfile" - target 'Runner' do - # Do not change existing lines. - ... - - target 'RunnerUITests' do - inherit! :complete - end - end - ``` - - 6. Create an empty file `integration_test/example_test.dart` in the root of your Flutter project. From the command line, run: - - ``` - $ flutter build macos --config-only integration_test/example_test.dart - ``` - - 7. Go to your `macos` directory and run: - - ``` - $ pod install --repo-update - ``` - - 8. Go to **RunnerUITests** -> **Build Phases** and add 2 new "Run Script Phase" Build Phases. - Rename them to `xcode_backend build` and `xcode_backend embed_and_thin` by double clicking - on their names. - - ![Xcode config setup](/assets/macos_runner_build_phases_create.png) - - 9. Arrange the newly created Build Phases in the order shown in the screenshot below. - - ![Xcode config setup](/assets/macos_runner_build_phases.png) - - 10. Paste this code into the first `macos_assemble build` Build Phase: - - ``` - /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/macos_assemble.sh" build - ``` - - 11. Paste this code into the second `macos_assemble embed` Build Phase: - - ``` - /bin/sh "$FLUTTER_ROOT/packages/flutter_tools/bin/macos_assemble.sh" embed - ``` - - 12. Xcode by default also enables a "parallel execution" setting, which - breaks Patrol. Disable it **for all schemes** (if you have more than one): - - - - 13. Go to **RunnerUITests** -> **Build Settings**, search for **User Script Sandboxing** - and make sure it's set to **No**. - - 14. Go to **Runner** -> **Signing & Capabilities**. Make sure that in all **App Sandbox** - sections, **Incoming Connections (Server)** and **Outgoing Connections (Client)** checkboxes - are checked. - - ![Xcode entitlements setup](/assets/macos_entitlements.png) - - 15. **Copy** `DebugProfile.entitlements` and `Release.entitlements` files from `macos/Runner` - to `macos/RunnerUITests` directory. - - 16. Go to **RunnerUITests** -> **Build Settings** and set **Code Signing Entitlements** to - `RunnerUITests/DebugProfile.entitlements` for **Debug** and **Profile** configuration and to - `RunnerUITests/Release.entitlements` for **Release** configuration. - - ![Xcode RunnerUITests entitlements setup](/assets/macos_ui_entitlements.png) - - - -### Create a simple integration test - -Let's create a dummy Flutter integration test that you'll use to verify -that Patrol is correctly set up. - -Paste the following code into `integration_test/example_test.dart`: - -```dart title="integration_test/example_test.dart" -import 'dart:io'; - -import 'package:flutter/material.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:patrol/patrol.dart'; - -void main() { - patrolTest( - 'counter state is the same after going to home and switching apps', - ($) async { - // Replace later with your app's main widget - await $.pumpWidgetAndSettle( - MaterialApp( - home: Scaffold( - appBar: AppBar(title: const Text('app')), - backgroundColor: Colors.blue, - ), - ), - ); - - expect($('app'), findsOneWidget); - if (!Platform.isMacOS) { - await $.native.pressHome(); - } - }, - ); -} -``` - -It does only 2 things: - -- first, it finds a text `app` -- then (on mobile platforms), it exits to home screen - -It's a very simple test, but it's enough to verify that Patrol is correctly set -up. To run `integration_test/example_test.dart` on a connected Android, iOS or macOS device: - - ``` - patrol test -t integration_test/example_test.dart - ``` - - If the setup is successful, you should see a **TEST PASSED** message. - If something went wrong, please proceed to the [FAQ] section which might - contain an answer to your issue. - - - If your app is using flavors, then you can pass them like so: - - ``` - patrol test --target integration_test/example_test.dart --flavor development - ``` - - or you can specify them in `pubspec.yaml` (recommended): - - ```yaml title="pubspec.yaml" - patrol: - app_name: My App - flavor: development - android: - package_name: com.example.myapp - ios: - bundle_id: com.example.MyApp - app_name: The Awesome App - macos: - bundle_id: com.example.macos.MyApp - ``` - - - - To prevent issues during Patrol tests, please follow these guidelines: - - 1. Do not call `IntegrationTestWidgetsFlutterBinding.ensureInitialized`. - Patrol automatically initializes its own test binding. - 2. Do not modify the global `FlutterError.onError` callback. Patrol's - internals depend on it. Keep in mind that this callback can also - be modified by popular packages such as Sentry or Crashlytics. - In such cases, you can disable them for Patrol tests. - - - - If you are looking for a working example of a Flutter app with Patrol tests, - check out the [example app](https://github.com/leancodepl/patrol/tree/master/packages/patrol/example) - in the patrol repository. - - -### FAQ - - - The reason is probably a mismatch of `patrol` and `patrol_cli` versions. Go to [Compatibility table] - and make sure that the versions of `patrol` and `patrol_cli` you are using are compatible. - - - - To run your application within the patrol test, you need to call `$.pumpWidgetAndSettle()`, - and pass your application's main widget to it. Be sure that you registered all the - necessary services before calling `$.pumpWidgetAndSettle()`. - Here's the example of running an app within the patrol test: - ```dart - - void main() { - patrolTest('real app test', ($) async { - - // Do all the necessary setup here (DI, services, etc.) - - await $.pumpWidgetAndSettle(const MyApp()); // Your's app main widget - - // Start testing your app here - - }); - } - - ``` - It's a good practice to create a setup wrapper function for your tests, so you don't have to - repeat the same code in every test. Look at the [example](https://github.com/leancodepl/patrol/blob/d2c7493f9399a028e39cb94fd204affdb932c5fc/dev/e2e_app/integration_test/common.dart#L17-L33) - of a wrapper function. - - - -#### Android - - - Go to `android/app/build.gradle` and look for `applicationId` in `defaultConfig` section. - - - - It's most likely caused by using incompatible JDK version. - Run `javac -version` to check your JDK version. Patrol officially works on JDK 17, - so unexpected errors may occur on other versions. - If you have AndroidStudio or Intellij IDEA installed, you can find the path to JDK by - opening your project's android directory in AS/Intellij and going to - **Settings** -> **Build, Execution, Deployment** -> **Build Tools** -> **Gradle** -> **Gradle JDK**. - [Learn more](https://developer.android.com/build/jdks) - - - -#### iOS - - - For iOS go to `ios/Runner.xcodeproj/project.pbxproj` and look for `PRODUCT_BUNDLE_IDENTIFIER`. - For macOS go to `macos/Runner.xcodeproj/project.pbxproj` and look for `PRODUCT_BUNDLE_IDENTIFIER`. - - - - Make sure that you disabled "Paralell execution" for **all schemes** in Xcode. - See [this video](https://www.youtube.com/watch?v=9LdEJR59fW4) for details. - - - - Search for a `FLUTTER_TARGET` in your project files and remove it (both value and key) - from *.xcconfig and *.pbxproj files. - - - - Search for a `FLUTTER_TARGET` in your project files and remove it (both value and key) - from *.xcconfig and *.pbxproj files. - - - - Check if this line in `Podfile` is present and uncommented. - ``` - platform :ios, '11.0' - ``` - If yes, then check if **iOS deployment version** in Xcode project's **Build Settings** - section for all targets (Runner and RunnerUITests) are set to the same value as in Podfile - (in case presented in snippet above, all should be set to 11.0). - - -If you couldn't find an answer to your question/problem, feel free to ask on -[Patrol Discord Server](https://discord.gg/ukBK5t4EZg). - -### Going from here - -To learn how to write Patrol tests, see [finders] and [native automation] sections. - -[native automation]: /native/usage -[finders]: /patrol/finders/usage -[Using Patrol finders in widget tests]: /patrol/finders/finders-setup -[Here's why]: /native/advanced#embrace-the-native-tests -[Patrol CLI]: https://pub.dev/packages/patrol_cli -[FAQ]: /getting-started#faq -[Compatibility table]: /compatibility-table diff --git a/docs/index.mdx b/docs/index.mdx index dd9803e51..99d0a3c7d 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -1,8 +1,3 @@ ---- -title: Patrol -description: The powerful Flutter-first UI testing framework. Ready for action! ---- - # Patrol ![Patrol promotional graphics][promo_graphics] @@ -69,8 +64,8 @@ we use Patrol to test production-grade apps for clients across industries, and n [leancode]: https://leancode.co -[native]: /native/overview -[finders]: /finders/overview +[native]: /documentation/native/overview +[finders]: /documentation/finders/overview [hot restart]: /cli-commands/develop [promo_graphics]: /assets/patrol_lncd.png [patrol_on_pubdev]: https://pub.dev/packages/patrol diff --git a/docs/integrations/allure.mdx b/docs/integrations/allure.mdx index bd727610f..2c3863afe 100644 --- a/docs/integrations/allure.mdx +++ b/docs/integrations/allure.mdx @@ -1,219 +1,3 @@ --- -title: Integrations - allure +redirect: /documentation/integrations/allure --- - -# Allure overview - -If you're using [Allure] to report your test results, you can use the -alternative test runner to get more detailed test report. - -We decided not to package this alternative runner together with Patrol because -it'd make Patrol depend on Allure, which is not desirable. Instead, you can -easily do it yourself. - -This guide assumes basic familiarity with Allure. To get started, see: - -- [official Allure documentation](https://docs.qameta.io/allure-report) -- [allure-framework/allure2 repository](https://github.com/allure-framework/allure2) - -This integration is currently Android-only. - - - Before you proceed with the steps listed below, make sure that you've - completed the [native setup] guide. - - -### Add dependencies and change runner - -First, you have to modify the **app-level build.gradle**: - -```groovy title="android/app/build.gradle" -android { - // ... - defaultConfig { - // ... - - // Replace the existing "testInstrumentationRunner" line with: - testInstrumentationRunner "pl.leancode.patrol.example.AllurePatrolJUnitRunner" - } - // ... -} - -dependencies { - androidTestImplementation "io.qameta.allure:allure-kotlin-model:2.4.0" - androidTestImplementation "io.qameta.allure:allure-kotlin-commons:2.4.0" - androidTestImplementation "io.qameta.allure:allure-kotlin-junit4:2.4.0" - androidTestImplementation "io.qameta.allure:allure-kotlin-android:2.4.0" -} -``` - - - Replace `pl.leancode.patrol.example` with your app's package name. - - -See also: - -- [the README of allure-kotlin library][allure_kotlin] - -### Create alternative runner - -Create a new Kotlin file in the same directory as **MainActivityTest.java** and -paste the following code, replacing the package: - -```kotlin title="AllurePatrolJUnitRunner.kt" -package pl.leancode.patrol.example // replace "pl.leancode.patrol.example" with your app's package - -import android.os.Bundle -import io.qameta.allure.android.AllureAndroidLifecycle -import io.qameta.allure.android.listeners.ExternalStoragePermissionsListener -import io.qameta.allure.android.writer.TestStorageResultsWriter -import io.qameta.allure.kotlin.Allure -import io.qameta.allure.kotlin.junit4.AllureJunit4 -import io.qameta.allure.kotlin.util.PropertiesUtils -import pl.leancode.patrol.PatrolJUnitRunner - -class AllurePatrolJUnitRunner : PatrolJUnitRunner() { - override fun onCreate(arguments: Bundle) { - Allure.lifecycle = createAllureAndroidLifecycle() - val listenerArg = listOfNotNull( - arguments.getCharSequence("listener"), - AllureJunit4::class.java.name, - ExternalStoragePermissionsListener::class.java.name.takeIf { useTestStorage } - ).joinToString(separator = ",") - arguments.putCharSequence("listener", listenerArg) - super.onCreate(arguments) - } - - private fun createAllureAndroidLifecycle() : AllureAndroidLifecycle { - return createDefaultAllureAndroidLifecycle() - } - - private fun createDefaultAllureAndroidLifecycle() : AllureAndroidLifecycle { - if (useTestStorage) { - return AllureAndroidLifecycle(TestStorageResultsWriter()) - } - - return AllureAndroidLifecycle() - } - - private val useTestStorage: Boolean - get() = PropertiesUtils.loadAllureProperties() - .getProperty("allure.results.useTestStorage", "true") - .toBoolean() -} -``` - - - In the snippet above, remember to replace the `package - pl.leancode.patrol.example` line at the top of the file with your app's - package name! - - -### Create allure.properties - -This is required if you enabled the `clearPackageData` option for Android Test -Orchestrator. If you enabled that option but don't create the -`allure.properties` file as below, your tests reports will be cleared after each -test. - -```txt title="android/app/src/main/res/allure.properties" -allure.results.useTestStorage=true -``` - -### Add rules to MainActivityTest - -Finally, modify the **MainActivityTest.java**. You'll add a 3 rules, which add -the following features: - -- automatically take a screenshot at the end of each test -- automatically dump the window hierarchy at the end of each test -- automatically embed the logcat into the report - -You can simply copy-paste the following code (remember to replace the package -name): - -```java title="MainActivityTest.java" -package pl.leancode.patrol.example; // replace "pl.leancode.patrol.example" with your app's package - -import androidx.test.platform.app.InstrumentationRegistry; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -import io.qameta.allure.android.rules.LogcatRule; -import io.qameta.allure.android.rules.ScreenshotRule; -import io.qameta.allure.android.rules.WindowHierarchyRule; -import pl.leancode.patrol.PatrolJUnitRunner; - -@RunWith(Parameterized.class) -public class MainActivityTest { - @Rule - public ScreenshotRule screenshotRule = new ScreenshotRule(ScreenshotRule.Mode.END, "ss_end"); - - @Rule - public WindowHierarchyRule windowHierarchyRule = new WindowHierarchyRule(); - - @Rule - public LogcatRule logcatRule = new LogcatRule(); - - @Parameters(name = "{0}") - public static Object[] testCases() { - PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); - instrumentation.setUp(MainActivity.class); - instrumentation.waitForPatrolAppService(); - return instrumentation.listDartTests(); - } - - public MainActivityTest(String dartTestName) { - this.dartTestName = dartTestName; - } - - private final String dartTestName; - - @Test - public void runDartTest() { - PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation(); - instrumentation.runDartTest(dartTestName); - } -} -``` - - - In the snippet above, remember to replace the `package - pl.leancode.patrol.example` line at the top of the file with your app's - package name! - - -### Retrieve the report - -Run the tests with `patrol test` as usual. - -After the tests are complete, create a directory for them, for example: - -```bash -mkdir -p ./build/reports -``` - -and then retrieve the results from the device: - -```bash -adb exec-out sh -c 'cd /sdcard/googletest/test_outputfiles && tar cf - allure-results' | tar xvf - -C build/reports -``` - -Finally, serve the results with Allure: - -```bash -allure serve ./build/reports/allure-results -``` - - - If you're using Homebrew, `brew install allure` is the quickest way to get - Allure. - - -[native setup]: /getting-started -[allure]: https://qameta.io/allure-report -[allure_kotlin]: https://github.com/allure-framework/allure-kotlin/blob/master/README.md diff --git a/docs/integrations/browserstack.mdx b/docs/integrations/browserstack.mdx index 9ed08063f..5b6e06ef7 100644 --- a/docs/integrations/browserstack.mdx +++ b/docs/integrations/browserstack.mdx @@ -1,90 +1,3 @@ --- -title: Integrations - Browserstack +redirect: /documentation/integrations/browserstack --- - -# Browserstack overview - -[Browserstack App Automate] is a popular cloud device farm. - -This integration is currently Android-only. - -### Change runner - -Modify the **app-level build.gradle**: - -```groovy title="android/app/build.gradle" -android { - // ... - defaultConfig { - //... - testInstrumentationRunner "pl.leancode.patrol.BrowserstackPatrolJUnitRunner" - } - // ... -} - -// ... -``` - -That's all! - -### Upload to Browserstack - -To run Android UI tests on BrowserStack: - -1. Build the app under test and the instrumentation app ([see docs][patrol build]) -1. Upload the app under test APK to Browserstack ([see docs][bs_app_docs]) -1. Upload the instrumentation app APK to Browserstack ([see docs][bs_test_docs]) -1. Start test execution on Browserstack ([see docs][bs_execute_docs]) - -You can do it all manually, but we recommend using the [bs_android] script to -speed up this process. `bs_android` is part of LeanCode's [mobile-tools]. If -you're using Homebrew, you can install it with: - -```bash -brew tap leancodepl/tools -brew install mobile-tools -``` - -The `bs_android` script requires the `BROWSERSTACK_CREDS` environment variable -to be set so it can authenticate with Browserstack: - -```bash -export YOUR_USERNAME:YOUR_ACCESS_KEY -``` - -Get your username and access on [Browserstack's account page][bs_account]. - -Now reload your shell (e.g. `exec zsh`) and run `bs_android`: - -``` -$ export BROWSERSTACK_PROJECT=AwesomeApp # optional -$ export BROWSERSTACK_DEVICES="[\"Google Pixel 4-10.0\"]" # optional -$ bs_android -• Building apk with entrypoint test_bundle.dart... -✓ Completed building apk with entrypoint test_bundle.dart (11.0s) - % Total % Received % Xferd Average Speed Time Time Time Current - Dload Upload Total Spent Left Speed -100 87.4M 100 235 100 87.4M 7 2857k 0:00:33 0:00:31 0:00:02 2052k -Uploaded app, url: bs://fb61a714e1a0c60e2578d940dad52b74da244d54 -Uploaded test, url: bs://a715b1231d41ac627bd683f1b16c28476babd72e -{"message":"Success","build_id":"a30440db559fcab65554ab0273437f3bd45d761b"} -Scheduled test execution -``` - -`bs_android` forwards all its options and flags to `patrol build`: - -```bash -bs_android \ - --target integration_test/example_test.dart,integration_test/another_test.dart \ - --release \ - --dart-define 'KEY_EXAMPLE=VALUE_EXAMPLE' -``` - -[Browserstack App Automate]: https://www.browserstack.com/app-automate -[patrol build]: /cli-commands/build -[bs_android]: https://github.com/leancodepl/mobile-tools/blob/master/bin/bs_android -[bs_app_docs]: https://www.browserstack.com/docs/app-automate/api-reference/espresso/apps -[bs_test_docs]: https://www.browserstack.com/docs/app-automate/api-reference/espresso/tests -[bs_execute_docs]: https://www.browserstack.com/docs/app-automate/api-reference/espresso/builds#execute-a-build -[bs_account]: https://www.browserstack.com/accounts/profile -[mobile-tools]: https://github.com/leancodepl/mobile-tools diff --git a/docs/logs-announcement.mdx b/docs/logs-announcement.mdx index 3ec5f9b58..4cf39e9b9 100644 --- a/docs/logs-announcement.mdx +++ b/docs/logs-announcement.mdx @@ -1,16 +1,3 @@ --- -title: Improved logging and reporting is here! +redirect: /announcements/logs-announcement --- - -# Big Update: Enhanced Logging and Test Reporting! - -We’ve made some major improvements to how you can monitor and analyze your tests! With Patrol 3.13.0 and later, you’ll get: - - - Verbose logging: Test names are now displayed in real time as they’re executed! - - Detailed step reporting: See every action Patrol takes during your test execution, giving you deeper insights into the process. - - Flutter logs in console: Now you can access Flutter logs directly within the patrol test output, streamlining debugging and analysis. -These enhancements will make it easier than ever to understand what's happening behind the code. - -For a full breakdown of these updates, check out the [Logs and test results][logs] page! - -[logs]: /logs \ No newline at end of file diff --git a/docs/logs.mdx b/docs/logs.mdx index 6fe3acc1a..64cffbe38 100644 --- a/docs/logs.mdx +++ b/docs/logs.mdx @@ -1,125 +1,3 @@ --- -title: Logs and test results +redirect: /documentation/logs --- - -# Logs and test results - -Once you've written and executed your tests, it's essential to monitor their results. Patrol provides two main methods for reporting test outcomes: **console logs** and **native test reports**. - -## Logging test steps - - -This feature is available starting from version `3.13.0`. - -If you're using this version but don't see logs for test steps, check if you're passing a custom `PatrolTesterConfig` to `patrolTest()`. If so, ensure the `printLogs: true` argument is included in the constructor. - - -During test execution, every test step (e.g., `tap` or `enterText`) is logged to the console along with its status. Additionally, the test name, status, and execution time are displayed. - -**Example console output:** - -``` -... -🧪 denies various permissions - ✅ 1. scrollTo widgets with text "Open permissions screen". - ✅ 2. scrollTo widgets with text "Open permissions screen". - ✅ 3. tap widgets with text "Open permissions screen". - ✅ 4. tap widgets with text "Request camera permission". - ✅ 5. isPermissionDialogVisible (native) - ✅ 6. tap widgets with text "Request camera permission". - ✅ 7. isPermissionDialogVisible (native) - ⏳ 8. denyPermission (native) -❌ denies various permissions (integration_test/permissions/deny_many_permissions_twice_test.dart) (9s) -══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞═════════════════ -The following PlatformException was thrown running a test: -PlatformException(PermissionHandler.PermissionManager, A request -for permissions is already running, please wait for it to finish -before doing another request (note that you can request multiple -permissions at the same time)., null, null) - -When the exception was thrown, this was the stack: -#0 StandardMethodCodec.decodeEnvelope (package:flutter/src/services/message_codecs.dart:648:7) -#1 MethodChannel._invokeMethod (package:flutter/src/services/platform_channel.dart:334:18) - -#2 MethodChannelPermissionHandler.requestPermissions (package:permission_handler_platform_interface/src/method_channel/method_channel_permission_handler.dart:79:9) - -#3 PermissionActions.request (package:permission_handler/permission_handler.dart:52:31) - -#4 _PermissionsScreenState._requestCameraPermission (package:e2e_app/permissions_screen.dart:21:20) - - -The test description was: - denies various permissions -═════════════════════════════════════════════════════════════════ - -✅ taps on notification (integration_test/permissions/notifications_test.dart) (16s) -✅ taps on notification native2 (integration_test/permissions/notifications_test.dart) (14s) -✅ grants various permissions (integration_test/permissions/permissions_many_test.dart) (15s) -... -``` - -## Test summary - -Once the tests are complete, a summary is printed: - - -``` -Test summary: -📝 Total: 8 -✅ Successful: 3 -❌ Failed: 5 - - taps on notification (integration_test/permissions/notifications_test.dart) - - taps on notification native2 (integration_test/permissions/notifications_test.dart) - - accepts location permission (integration_test/permissions/permissions_location_test.dart) - - accepts location permission native2 (integration_test/permissions/permissions_location_test.dart) - - grants various permissions (integration_test/permissions/permissions_many_test.dart) -⏩ Skipped: 0 -📊 Report: file:///Users/user/patrol/dev/e2e_app/build/app/reports/androidTests/connected/index.html -⏱️ Duration: 227s -``` - -## Customizing log behavior - -You can customize which logs are displayed by using the following flags. These can be passed to the `patrol test` or `patrol develop` commands: - -| Flag | Description | Available in | Default value | -| ------------------------- | ------------------------------------------ | -------------------------------------------------- | ------------- | -| --[no-]show-flutter-logs | Show Flutter logs while running the tests. | `patrol test`, in `patrol develop` it's always on | `false` | -| --[no-]hide-test-steps | Hide test steps while running the tests. | `patrol test` and `patrol develop` | `false` | -| --[no-]clear-test-steps | Clear test steps after the test finishes. | `patrol test` | `true` | - -## Native test reports - -In addition to console logs, you can review test results in a **native test report**. The report's file path is provided in the test summary, for example: - -``` -📊 Report: file:///Users/user/patrol/dev/e2e_app/build/app/reports/androidTests/connected/index.html -``` - -## Logs in `patrol_finders` - -By default, enhanced logs are disabled when using `patrol_finders` without the `patrol` package. To enable them, pass the `printLogs: true` argument to the `PatrolTesterConfig` constructor: - -```dart -patrolWidgetTest( - 'throws exception when no widget to tap on is found', - config: const PatrolTesterConfig(printLogs: true), - (tester) async { - // test body - // ... - }, -); -``` -```dart -testWidgets( - 'description', - (widgetTester) async { - final $ = PatrolTester( - tester: widgetTester, - config: PatrolTesterConfig(printLogs: true), - ); - // test body - // ... - }, -); -``` diff --git a/docs/native/advanced.mdx b/docs/native/advanced.mdx index 7870a0a50..eab251400 100644 --- a/docs/native/advanced.mdx +++ b/docs/native/advanced.mdx @@ -1,159 +1,3 @@ --- -title: Native automation - advanced +redirect: /documentation/native/advanced --- - -# Advanced - -### Patrol section in `pubspec.yaml` - -If your app has different name on iOS and Android, you can specify `app_name` -twice – one in `android` block, and one in `ios` block. - -Though the whole Patrol section in `pubspec.yaml` is optional, we highly recommend -adding this section, because it enables the following features: - -- Patrol will automatically uninstall your app after every test (using - `package_name` and `bundle_id`). This will make the environment which your - tests run in more stable and predictable. -- Patrol will be able to tap on your app's notifications (using `app_name`) - -### Specific version of `patrol_cli` - -You can install a specific version of Patrol CLI. For example: - -``` -dart pub global activate patrol_cli ^1.0.0 -``` - -will install the latest `v1` version. We recommend to install a specific -version on CI systems to avoid unexpected breakages. - -### Isolation of test runs - -To achieve full isolation between test runs, enable the `clearPackageOption`: - -```groovy title="android/app/build.gradle" -defaultConfig { - //... - testInstrumentationRunner "pl.leancode.patrol.PatrolJUnitRunner" - testInstrumentationRunnerArguments clearPackageData: "true" -} -``` - -This will clear the app's data and permissions before each test run. -Unfortunately, no equivalent feature is available on iOS. - -### Embrace the native tests - -If you've diligently followed the steps in [native automation setup] and `patrol test` prints a **TEST -PASSED** message, you might be now thinking: what did I just do? - -The answer is: You've just integrated Flutter testing with native Android/iOS -testing frameworks. This means that your Flutter integration tests can now be -run as _native tests_. - -#### What are native tests good for, anyway? - -iOS and Android have existed for more than 15 years, and during that time many -of awesome testing-related things were built around them – open-source test -runners, device farms, HTML report generators. Developers who create native -mobile apps can easily reap benefits from this huge, mature ecosystem. - -Meanwhile we, Flutter developers, don't have as much at our disposal. Our -framework is much younger and less mature. - -What if we could masquerade our Flutter tests so that from the outside they -would be truly native? This way we leverage many existing tools while -maintaining the convenience of writing the tests in pure Dart. - -> For example, you can run your Patrol tests directly from Xcode. Xcode knows -> nothing about Flutter, Dart and Patrol – it only launches your test app. -> Flutter tests are then run inside the test app and the results are reported -> back to Xcode. This way you get the best of both worlds – the maturity of -> native iOS development and the productivity of Flutter and Dart. - -That's exactly what Patrol does (and what the default [integration_test] package -does at well, but at a bit smaller scale). - -Take a look at this simple Flutter integration tests using Patrol: - -```dart title="integration_test/example_test.dart" -void main() { - patrolTest( - 'counter state is the same after going to Home and switching apps', - nativeAutomatorConfig: NativeAutomatorConfig( - packageName: 'pl.leancode.patrol.example', - bundleId: 'pl.leancode.patrol.Example', - ), - ($) async { - await $.pumpWidget(ExampleApp()); - - await $(FloatingActionButton).tap(); - expect($(#counterText).text, '1'); - - await $.native.pressHome(); - await $.native.openApp(); - - expect($(#counterText).text, '1'); - - await $(FloatingActionButton).tap(); - expect($(#counterText).text, '2'); - }, - ); -} -``` - -You can run this test and view its results in many ways, using all sorts of -different tools, platforms, and IDEs: - - - - When Android test finishes, its test results are automatically generated in - `build/app/outputs/androidTest-results/connected/test-result.pb`. To view - them in Android Studio, use the `Run > Import tests from file` option. - - ![Test results in Android Studio](/assets/android_test_results_as.png) - - - - ![Patrol test results in Xcode](/assets/ios_test_results_xcode.png) - - - It just works ✨ - - ![Test results in Firebase Test Lab](/assets/android_test_results_firebase.png) - - - - You don't have to use the bulky Android Studio to view your test results, - because Gradle 🐘 automatically generates a nice HTML summary! - - You can find it in `build/app/reports/androidTests/connected/index.html`. - - ![Test results generated by Gradle](/assets/android_test_results_gradle.png) - - - - With the help of awesome [fastlane scan] you can prettify the output of - `xcodebuild` to make it easier to understand and generate HTML summary of - your tests. - - ![Test results by xcpretty](/assets/ios_test_results_xcpretty.png) - - - - -This is so awesome! - -[native automation setup]: /getting-started -[fastlane scan]: https://docs.fastlane.tools/actions/scan -[integration_test]: https://github.com/flutter/flutter/tree/master/packages/integration_test diff --git a/docs/native/feature-parity.mdx b/docs/native/feature-parity.mdx index f694a05ae..54ef9fd3f 100644 --- a/docs/native/feature-parity.mdx +++ b/docs/native/feature-parity.mdx @@ -1,61 +1,3 @@ --- -title: Native feature's parity +redirect: /documentation/native/feature-parity --- - -# Feature parity - -Here you can see what you can already do with Patrol, and what is yet to be -implemented. We hope that it will help you evaluate Patrol. - -We strive for high feature parity across iOS and Android, but in some cases it's -impossible to reach 100%. macOS support is still in alpha, so it has no native features yet. - -| **Feature** | **Android** | **iOS** | **macOS (alpha)** | -| --------------------------- | -------------- | ------------------ | ----------------- | -| [Press home] | ✅ | ✅ | ❌ | -| [Press back] | ✅ | ❌ (no API) | ❌ | -| [Open any app] | ✅ | ✅ | ✅ | -| [Open notifications] | ✅ | ✅ | ❌ | -| [Tap on notification] | ✅ | ✅ | ❌ | -| [Open quick settings] | ✅ | ✅ | ❌ | -| [Open url] | ✅ | ✅ | ✅ | -| [Toggle dark mode] | ✅ | ✅ | ❌ | -| [Toggle airplane mode] | ✅ | ✅ | ❌ | -| [Toggle cellular] | ✅ | ✅ | ❌ | -| [Toggle Wi-Fi] | ✅ | ✅ | ❌ | -| [Toggle Bluetooth] | ✅ | ✅ | ❌ | -| Toggle location | ✅ | ✅ see [#326] | ❌ | -| [Tap] | ✅ | ✅ | ❌ | -| [Double tap] | ✅ | ✅ | ❌ | -| [Tap at coordinate] | ✅ | ✅ | ❌ | -| [Enter text] | ✅ | ✅ | ❌ | -| [Swipe] | ✅ | ✅ | ❌ | -| [Handle permission dialogs] | ✅ | ✅ | ❌ | -| Interact with WebView | ⚠️ see [#244] | ✅ | ❌ | -| [Press volume down] | ✅ | ✅ (simulator ❌) | ❌ | -| [Press volume up] | ✅ | ✅ (simulator ❌) | ❌ | - - -[#244]: https://github.com/leancodepl/patrol/issues/244 -[#283]: https://github.com/leancodepl/patrol/issues/283 -[#326]: https://github.com/leancodepl/patrol/issues/326 -[press home]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressHome.html -[press back]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressBack.html -[open any app]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openApp.html -[open notifications]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openNotifications.html -[tap on notification]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/tapOnNotificationBySelector.html -[open quick settings]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openQuickSettings.html -[open url]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/openUrl.html -[toggle dark mode]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableDarkMode.html -[toggle airplane mode]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableAirplaneMode.html -[toggle cellular]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableCellular.html -[toggle wi-fi]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableWifi.html -[toggle bluetooth]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enableBluetooth.html -[tap]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/tap.html -[double tap]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/doubleTap.html -[tap at coordinate]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/tapAt.html -[enter text]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/enterText.html -[swipe]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/swipe.html -[handle permission dialogs]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/grantPermissionWhenInUse.html -[press volume down]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressVolumeDown.html -[press volume up]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator/pressVolumeUp.html diff --git a/docs/native/overview.mdx b/docs/native/overview.mdx index 34f94708e..d64191bc3 100644 --- a/docs/native/overview.mdx +++ b/docs/native/overview.mdx @@ -1,51 +1,3 @@ --- -title: Native automation - overview +redirect: /documentation/native/overview --- - -# Native automation overview - -Flutter's [integration_test][integration_test] does a good job at providing -basic support for integration testing Flutter apps. What it can't do is -interaction with the OS your Flutter app is running on. This makes it impossible -to test many critical business features: - -- granting runtime permissions -- signing into the app which uses WebView or 0Auth (like Google) as the login - page -- listing and tapping on notifications -- exiting the app, coming back, and verifying that state is preserved -- enabling and disabling features such as Wi-Fi, mobile data, location, or dark - mode - -Patrol's _native automation_ feature finally solves these problems. Here's a -tiny snippet to spice things up: - -```dart title="integration_test/demo_test.dart" -void main() { - patrolTest('demo', (PatrolIntegrationTester $) async { - await $.pumpWidgetAndSettle(AwesomeApp()); - // prepare network conditions - await $.native.enableCellular(); - await $.native.disableWifi(); - - // toggle system theme - await $.native.enableDarkMode(); - - // handle native location permission request dialog - await $.native.selectFineLocation(); - await $.native.grantPermissionWhenInUse(); - - // tap on the first notification - await $.native.openNotifications(); - await $.native.tapOnNotificationByIndex(0); - }); -} -``` - -Native automation is currently available only on Android and iOS. - - - - - -[integration_test]: https://github.com/flutter/flutter/tree/master/packages/integration_test diff --git a/docs/native/setup.mdx b/docs/native/setup.mdx index cd2ea9ae4..e47cbfb88 100644 --- a/docs/native/setup.mdx +++ b/docs/native/setup.mdx @@ -1,3 +1,3 @@ --- -redirect: /getting-started +redirect: /documentation --- \ No newline at end of file diff --git a/docs/native/usage.mdx b/docs/native/usage.mdx index 0b6078626..4354117da 100644 --- a/docs/native/usage.mdx +++ b/docs/native/usage.mdx @@ -1,146 +1,3 @@ --- -title: Native automation - usage +redirect: /documentation/native/usage --- - -# Usage - -Once set up, interacting with the native UI using Patrol is very easy! - -### Basics - -After you've got your `NativeAutomator` object, you simply call methods on it -and it does the magic. - -To tap on a native view (for example, a button in a WebView): - -```dart -await $.native.tap(Selector(text: 'Sign up for newsletter')); -``` - -To enter text into a native view (for example, a form in a WebView): - -```dart -await $.native.enterText( - Selector(text: 'Enter your email'), - text: 'charlie@root.me', -); -``` - -You can also enter text into n-th currently visible text field (counting from -0): - -```dart -await $.native.enterTextByIndex('charlie_root', index: 0); // enter username -await $.native.enterTextByIndex('ny4ncat', index: 1); // enter password -``` - -The above are the simplest, most common actions, but they already make it -possible to test scenarios that were impossible to test before, such as -WebViews. - - - To tap, enter text, or perform generally any UI interaction with a iOS app - that is not your Flutter app under test, you need to pass its bundle - identifier. For example, to tap on the plus button in the iPhone contacts app, - use: - -```dart -await $.native.tap( - Selector(text: 'Add'), - appId: 'com.apple.MobileAddressBook', -); -``` - - - -### Notifications - -To open the notification shade: - -```dart -await $.native.openNotifications(); -``` - -To tap on the second notification: - -```dart -await $.native.tapOnNotificationByIndex(1); -``` - -You can also tap on notification by its content: - -```dart -await $.native.tapOnNotificationBySelector( - Selector(textContains: 'Someone liked your recent post'), -); -``` - - - -### Permissions - -To handle the native permission request dialog: - -```dart -await $.native.grantPermissionWhenInUse(); -await $.native.grantPermissionOnlyThisTime(); -await $.native.denyPermission(); -``` - - - -If the permission request dialog visible is the location dialog, you can also -select the accuracy: - -```dart -await $.native.selectFineLocation(); -await $.native.selectCoarseLocation(); -``` - - - -The test will fail if the permission request dialog is not visible. You can -check if it is with: - -```dart -if (await $.native.isPermissionDialogVisible()) { - await $.native.grantPermissionWhenInUse(); -} -``` - -By default, `isPermissionDialogVisible()` waits for a short amount of time and -then returns `false` if the dialog is not visible. To increase the timeout: - -```dart -if (await $.native.isPermissionDialogVisible(timeout: Duration(seconds: 5))) { - await $.native.grantPermissionWhenInUse(); -} -``` - - - Patrol can handle permissions on iOS only if the device language is set to - English (preferably US). That's because there's no way to refer to a specific - view in a language-independent way (like resourceId on Android). - -If you want to handle permissions on iOS device with non-English locale, do it -manually: - -```dart -await $.native.tap( - Selector(text: 'Allow'), - appId: 'com.apple.springboard', -); -``` - - - -### More resources - -To see more integration tests demonstrating Patrol's various features, check out -our [example app][example_app]. - -[nativeautomator]: https://pub.dev/documentation/patrol/latest/patrol/NativeAutomator-class.html -[example_app]: https://github.com/leancodepl/patrol/tree/master/packages/patrol/example -[permissions_many_test]: https://user-images.githubusercontent.com/40357511/219790614-dddcfc2b-c68c-44a6-be4b-c161151c9fe6.mp4 -[permissions_location_test]: https://user-images.githubusercontent.com/40357511/219790600-0ec5f42c-424e-4dc7-a36f-eea5f9a3e49f.mp4 -[notifications_test]: https://user-images.githubusercontent.com/40357511/219792140-09fa967a-8c81-4096-a533-df107741e5f0.mp4 diff --git a/docs/overview/getting-started.mdx b/docs/overview/getting-started.mdx new file mode 100644 index 000000000..c32413c3b --- /dev/null +++ b/docs/overview/getting-started.mdx @@ -0,0 +1,44 @@ +--- +title: Getting started +next: /documentation +nextTitle: Install Patrol +--- + +Try out Patrol's capabilities with few clicks! Follow this tutorial and get to know our framework. + +## Try it out in IDX + + + + + +export const IDXButton = props => { + return + Try in IDX +; +}; + +Create new project in IDX with Patrol demo project by clicking button below. + + + + + + +Check **`This is a Flutter app`** checkbox when creating a project. + + + +Click **Import**. + + +Android emulator will open as preview and a test will start in develop mode. It takes a while, you can see logs in terminal +(it's called `onStart`, may be hidden by default). After test is finished, you can change test's or app's code and type `r` +in terminal to rerun it. + + + + diff --git a/docs/patrol-finders-release.mdx b/docs/patrol-finders-release.mdx index 1f334189f..965f2e05a 100644 --- a/docs/patrol-finders-release.mdx +++ b/docs/patrol-finders-release.mdx @@ -1,29 +1,3 @@ --- -title: New package - patrol_finders +redirect: /announcements/patrol-finders-release --- - -# New package - `patrol_finders` - -We're introducing [`patrol_finders`] - a new package in Patrol framework! It was -created to make it easier to use Patrol finders in widget tests. - -We decided to separate out our finders mechanism to another package, so -developers who would like to use Patrol's awesome finders in their widget tests -don't need to depend on whole Patrol package. This way you can conveniently use -Patrol finders in widget or golden tests, whichever platforms you need to -support in your project! - -### How to use it? - -We made a short tutorial on how to use `patrol_finders` package separately in -widget tests, you can find it in [Using Patrol finders in widget tests] section. - -### Does this change affect my Patrol tests? - -If you have already some Patrol tests in your project, there are no breaking -changes in this release - everything works the same as before. Though you may -see some deprecation warnings in your code - you can get your code aligned with -them to prepare for future changes. - -[Using Patrol finders in widget tests]: /patrol/finders/finders-setup -[`patrol_finders`]: https://pub.dev/packages/patrol_finders diff --git a/docs/supported-platforms.mdx b/docs/supported-platforms.mdx index 29c6b8594..c1ace1b82 100644 --- a/docs/supported-platforms.mdx +++ b/docs/supported-platforms.mdx @@ -1,19 +1,3 @@ --- -title: Supported platforms +redirect: /documentation/supported-platforms --- - -# Supported platforms - -Patrol works on: - -- Android 5.0 (API 21) and newer, -- iOS 11 and newer, -- macOS 10.14 and newer (alpha support). - -Web, Windows, and Linux are not supported. - -On mobile platforms it works on both physical and virtual devices. - -If you want to check which native features are supported, see [feature parity]. - -[feature parity]: /native/feature-parity diff --git a/docs/tips-and-tricks.mdx b/docs/tips-and-tricks.mdx index ea69e20b6..4db26b463 100644 --- a/docs/tips-and-tricks.mdx +++ b/docs/tips-and-tricks.mdx @@ -1,171 +1,3 @@ --- -title: Tips and tricks +redirect: /documentation/tips-and-tricks --- - -# Tips and tricks - -### Inspecting native view hierarchy - -It's hard to tap on or enter text into a view you don't know how to refer to. In -such situation we recommend doing a native view hierarchy dump and finding the -properties of the view you want to act on. - -**Android** - -First, perform a native view hierarchy dump using `adb`: - -``` -adb shell uiautomator dump -``` - -Then, copy the dump file from the device to your machine: - -``` -adb pull /sdcard/window_dump.xml . -``` - -**iOS** - -The easiest way to perform the native view hierarchy dumb on iOS is to use the -[idb] tool. - -Once you have [idb] installed, perform a dump: - -``` -idb ui describe-all -``` - -### Avoiding hardcoding credentials in tests - -It's a bad practice to hardcode data such as emails, usernames, and passwords in -test code. - -```dart -await $(#nameTextField).enterText('Bartek'); // bad! -await $(#passwordTextField).enterText('ny4ncat'); // bad as well! -``` - -To fix this, we recommend removing the hardcoded credentials from test code and -providing them through the environment: - -```dart -await $(#nameTextField).enterText(const String.fromEnvironment('USERNAME')); -await $(#passwordTextField).enterText(const String.fromEnvironment('PASSWORD')); -``` - -> Make sure that you're using `const` here because of [issue #55870][55870]. - -To set `USERNAME` and `PASSWORD`, use `--dart-define`: - -``` -patrol test --dart-define 'USERNAME=Bartek' --dart-define 'PASSWORD=ny4ncat' -``` - -Alternatively you can create a `.patrol.env` file in your project's root. Here's -an example: - -``` -$ cat .patrol.env -EMAIL=user@example.com -PASSWORD=ny4ncat -DISABLE_ANALYTICS=true -``` - -### Granting sensitive permission through the Settings app - -Some particularly sensitive permissions (such as access to background location -or controlling the Do Not Disturb feature) cannot be requested in the permission -dialog like most of the common permissions. Instead, you have to ask the user to -go to the Settings app and grant your app the permission you need. - -Testing such flows is not as simple as simply granting normal permission, but -it's totally possible with Patrol. - -Below we present you with a snippet that will make the built-in Camera app have -access to the Do Not Disturb feature on Android. Let's assume that the Settings -app on the device we want to run the tests on looks like this: - -![settings_screenshot] - -And here's the code: - -```dart -await $.native.tap(Selector(text: 'Camera')); // tap on the list tile -await $.native.tap(Selector(text: 'ALLOW')); // handle the confirmation dialog -await $.native.pressBack(); // go back to the app under test -``` - -Please note that the UI of the Settings app differs across operating systems, -their versions, and OEM flavors (in case of Android). You'll have to handle all -edge cases yourself. - -### Ignoring exceptions - -If an exception is thrown during a test, it is marked as failed. This is -Flutter's default behavior and it's usually good – after all, it's better to fix -the cause of a problem instead of ignoring it. - -That said, sometimes you do have a legitimate reason to ignore an exception. -This can be accomplished with the help of the -[WidgetTester.takeException()][take_exception] method, which returns the last -exception that occurred and removes it from the internal list of uncaught -exceptions, so that it won't mark the test as failed. To use it, just call it -once: - -```dart -final widgetTester = $.tester; -widgetTester.takeException(); -``` - -If more than a single exception is thrown during the test and you want to ignore -all of them, the below snippet should come in handy: - -```dart -var exceptionCount = 0; -dynamic exception = $.tester.takeException(); -while (exception != null) { - exceptionCount++; - exception = $.tester.takeException(); -} -if (exceptionCount != 0) { - $.log('Warning: $exceptionCount exceptions were ignored'); -} -``` - -### Handling permission dialogs before the main app widget is pumped - -Sometimes you might want to manually request permissions in the test before the -main app widget is pumped. Let's say that you're using the [geolocator] package: - -```dart -final permission = await Geolocator.requestPermission(); -final position = await Geolocator.getCurrentPosition(); -await $.pumpWidgetAndSettle(MyApp(position: position)); -``` - -In such case, first call the `requestPermission()` method, but instead of -awaiting it, assign the `Future` it returns to some `final`. Then, use Patrol to -grant the permissions, and finally, await the `Future` from the first step: - -```dart -// 1. request the permission -final permissionRequestFuture = Geolocator.requestPermission(); -// 2. grant the permission using Patrol -await $.native.grantPermissionWhenInUse(); -// 3. wait for permission being granted -final permissionRequestResult = await permissionRequestFuture; -expect(permissionRequestResult, equals(LocationPermission.whileInUse)); -final position = await Geolocator.getCurrentPosition(); -await $.pumpWidgetAndSettle(MyApp(position: position)); -``` - -See also: - -- [Patrol issue #628] - -[patrol issue #628]: https://github.com/leancodepl/patrol/issues/628 -[geolocator]: https://pub.dev/packages/geolocator -[idb]: https://github.com/facebook/idb -[settings_screenshot]: https://user-images.githubusercontent.com/10289319/194897313-849b8b84-df7a-4bf3-9b06-bb2782876d03.png -[take_exception]: https://api.flutter.dev/flutter/flutter_test/WidgetTester/takeException.html -[55870]: https://github.com/flutter/flutter/issues/55870 diff --git a/docs/v3.mdx b/docs/v3.mdx index fc8036c68..d2077151f 100644 --- a/docs/v3.mdx +++ b/docs/v3.mdx @@ -1,82 +1,3 @@ --- -title: Patrol 3.0 is here +redirect: /announcements/v3 --- - -# Patrol 3.0 is here! - -Patrol 3.0 is the new major version of Patrol. - -## `patrol v3` and DevTools extension - -The highlight of this release is the **Patrol DevTools Extension**. We created -it to enhance your UI test development experience with `patrol develop` by -making it much easier to explore the native view hierarchy. With Patrol's new -DevTools extension, you can effortlessly inspect the currently visible -Android/iOS views and discover their properties. This information can be then -used in native selectors like `$.native.tap()`, eliminating the need for -external tools. - -Patrol is one of the first packages in the whole Flutter ecosystem to have a -DevTools extension. We have started working on it as soon as the Flutter team -has announced that they're working on making DevTools extensible. We immediately -realized how powerful this feature is and how it can enable us to deliver better -UI testing experience. - -This is, of course, just the beginning, and we have plans to introduce more -features in future updates of our DevTools extension. - -### Changes in `patrol` v3: - -The DevTools extension is not the only new feature in this release. Other -changes include: - -- **Minimum Flutter version**: The minimum supported Flutter version has been - bumped to 3.16 to make it compatible with a few breaking changes that were - introduced to the `flutter_test` package that `patrol` and `patrol_finders` - depend on. We hope you'll have an easy time upgrading to 3.16, but if not, you - can always use Patrol v2 until you're ready to upgrade. - -- **A few breaking changes**: - - The `bindingType` parameter has been removed from the `patrolTest()` - function. Now, only `PatrolBinding` is used and it's also automatically - initialized. - - The `nativeAutomation` parameter has also been removed from the - `patrolTest()` function. Now `patrolTest()` implies native automation and - you can use `patrolWidgetTest()` if you don't need it. - - `PatrolTester` class has been renamed to `PatrolIntegrationTester`. Now - `PatrolTester` is used with `patrolWidgetTest()` *without* native automation - and `PatrolIntegrationTester` is used with `patrolTest()` *with* native - automation. -- **Patrol CLI version requirement**: Patrol v3 requires Patrol CLI v2.3.0 or - newer, so make sure to `patrol update`! - -## `patrol_finders` v2 - -Along with `patrol` v3, we are releasing the v2 of [patrol_finders]. In case you -missed it, we split `patrol_finders` from `patrol` a few months ago in response -to our community members who loved Patrol's lean finders syntax, but weren't -interested in developing integration tests. [Here's the docs -page][patrol_finders_docs] about `patrol_finders` in case you missed it. - -### Changes in `patrol_finders` v2: - -- **Minimum Flutter version**: The minimum supported Flutter version of - `patrol_finders` has been bumped to 3.16, just like in `patrol`'s case. -- The deprecated `andSettle` method has been removed from all `PatrolTester` and - `PatrolFinder` methods like `tap()`, `enterText()`, and so on. Developers - should now use `settlePolicy` as a replacement, which has been available since - June. -- The default `settlePolicy` has been changed to [SettlePolicy.trySettle]. - -## Wrapping up - -As you can see, these updates have a little bit of everything - a large new -feature, support for the latest Flutter version, and a clean-up of a few -deprecations. We encourage you to explore our new DevTools extension and look -forward to your feedback and ideas for new features as we continue to evolve the -Patrol ecosystem. Meanwhile, we're getting back to work on Patrol, with a single -goal in mind – to make it the go-to UI testing framework for Flutter apps. - -[patrol_finders]: https://pub.dev/packages/patrol_finders -[patrol_finders_docs]: https://patrol.leancode.co/patrol-finders-release -[SettlePolicy.trySettle]: https://pub.dev/documentation/patrol_finders/latest/patrol_finders/SettlePolicy.html#trySettle diff --git a/docs/write-your-first-test.mdx b/docs/write-your-first-test.mdx index 01153945e..066630d69 100644 --- a/docs/write-your-first-test.mdx +++ b/docs/write-your-first-test.mdx @@ -1,470 +1,3 @@ --- -title: Write your first test +redirect: /documentation/write-your-first-test --- - -# Write your first test - -Patrol is a powerful, open-source testing framework created by LeanCode that enhances Flutter's testing capabilities by enabling interaction with native platform features directly in Dart. It allows to handle permission dialogs, notifications, WebViews, and device settings—features previously unavailable in standard Flutter tests, making it truly possible to test the whole app. - -This tutorial will take you through writing your first substantial Patrol test, interacting both with the Flutter app itself and also with native permission dialogs and notifications. - -Before writing any tests, make sure you [install the Patrol CLI](https://patrol.leancode.co/getting-started#install-patrol_cli). Then just clone the following repository from GitHub to follow along. The app we’re going to be testing is fully functional and ready to be tested, with Patrol already configured. - - -To learn how to set up Patrol for your own project, check out the [Patrol Setup Docs](https://patrol.leancode.co/getting-started). - - - -Clone the [STARTER PROJECT](https://github.com/ResoCoder/patrol-basics-tutorial) to follow along. - - - - -### App Walkthrough - -Before we can start writing automated Patrol tests, we need to know what the app does and to test it manually. Please, check out the video tutorial for a visual walkthrough. - -The first screen of our app is for signing in. It’s not using any actual sign-in provider but it only validates the email address and password. In order to successfully “sign in” and get to the home screen, we need to input a valid email and a password that’s at least 8 characters long. - - -You can test any real authentication providers that use WebView for signing in with the powerful Patrol native automation. - - -On the second screen, we’re greeted with a notifications permission dialog. Once we allow them, we can tap on the notification button in the app bar to manually trigger a local notification which will be displayed after 3 seconds both when the app is running in the foreground or in the background. - -Once we open the native notification bar and tap on the notification from our app, we’re gonna see a snackbar on the bottom saying _"Notification was tapped!”_ - -### Testing the “Happy Path” - -You’ve just seen the full walkthrough of the app, including errors that can show up if you input an invalid email or password. UI tests (integration tests), like the ones we’re going to write with Patrol, should only be testing the “happy path” of a UI flow. We only want them to fail if the app suddenly stops the user from doing what the app is for - in this case, that’s displaying a notification. Validation error messages are not “what the app is for”, they exist only to allow the user to successfully sign in with a proper email and password. That’s why we won’t be checking for them in the tests. - -### Writing the Test - -We have only one UI flow in this app, that is signing in, showing the notification and then tapping on that notification. This means, we’re going to have only a single test. Let’s create it in `/integration_test/app_test.dart`. - -Like any other test, we need to have a `main()` top-level function. Inside it we’re going to have our single `patrolTest` with a description properly describing what we’re about to test. An optional step is to set the frame policy to “fully live” to make sure all frames are shown, even those which are not pumped by our test code. Without it, we would see that our app stutters and animations are not played properly. - -```dart -import 'package:flutter_test/flutter_test.dart'; -import 'package:patrol/patrol.dart'; - -void main() { - patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - // Test code will go here - }, - ); -} -``` - -We could start writing the test right now and then re-run it from scratch every time we add a new line of test code by calling `patrol test --target integration_test/app_test.dart` but since we’re writing a Patrol test that runs on an Android or iOS device, constantly building the whole Flutter app is not time effective. Thankfully, Patrol offers a different approach - hot restarting the tests! We can run the command `patrol develop --target integration_test/app_test.dart` right now and anytime we add a new line of test code, we can just type “r” in the terminal to re-run the tests without the time-costly app building. Just make sure that you have an emulator running first - Patrol will select it automatically. - -First, we need to perform any initializations that need to happen before the app is run and pump the top-level widget of our app. We’re effectively doing what the `main` function inside `main.dart` does - this time not for just running the app as usual but for running an automated Patrol test. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - }, -); -``` - -Hot-restarting the test by typing “r” into the terminal won’t really do much since we’re not yet performing any user-like actions but you will at least see the sign in page for a brief moment before the test finishes. - -Let’s now perform some action! We know we have to sign in if we want to continue to the home screen. First, we have to type in both email and password. There are multiple ways to find widgets on the screen - by widget type, by text and lastly by key. - -Although it’s not the best practice, we’re first going to find the fields by type. Both are of type `TextFormField` but there are two of them on the screen so the following won’t work. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(TextFormField).enterText('test@email.com'); - await $(TextFormField).enterText('password'); - }, -); -``` - -That’s because finders always find the first matching widget so both the email address and password are entered into the same field - in this case, the email field. - -If multiple widgets on a screen match the finder, we can tell Patrol which one we want by specifying its index in the list of all found widgets from top to bottom like this: - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(TextFormField).enterText('test@email.com'); - await $(TextFormField).at(1).enterText('password'); - }, -); -``` - -We can use a text finder to tap on the “Sign in” button. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(TextFormField).enterText('test@email.com'); - await $(TextFormField).at(1).enterText('password'); - await $('Sign in').tap(); - }, -); -``` - -Hot-restarting the test will now take you all the way to the home page from which we will want to trigger the notification. - -As you can imagine though, using type and text finders in any app that’s just a bit more complex will result in a huge mess. The recommended approach is to always find your widgets by their `Key`. There are currently no keys specified for these widgets so let’s change that. In `sign_in_page.dart` pass in the following into the `TextFormFields` and `ElevatedButton`: - -```dart -class SignInPage extends StatelessWidget { - const SignInPage({super.key}); - - @override - Widget build(BuildContext context) { - return Scaffold( - ... - body: Padding( - padding: const EdgeInsets.all(16), - child: Form( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - TextFormField( - key: const Key('emailTextField'), - decoration: const InputDecoration( - labelText: 'Email', - ), - ... - ), - const SizedBox(height: 16), - TextFormField( - key: const Key('passwordTextField'), - decoration: const InputDecoration( - labelText: 'Password', - ), - ... - ), - const SizedBox(height: 16), - Builder(builder: (context) { - return ElevatedButton( - key: const Key('signInButton'), - ... - child: const Text('Sign in'), - ); - }), - ], - ), - ), - ), - ); - } -} -``` - -With the keys in place, we can now rewrite our test code to use `Key` finders. The simplest approach is to prefix the key’s value with a hash symbol. For this approach to work, your keys mustn’t contain any invalid characters such as spaces. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(#emailTextField).enterText('test@email.com'); - await $(#passwordTextField).enterText('password'); - await $(#signInButton).tap(); - }, -); -``` - -Looking at this test code again, it’s certain we can do better. Why? We’ve just added code duplication to our codebase! The key values in `sign_in_page.dart` and in `app_test.dart` are fully duplicated and if we change one, the other won’t be automatically updated, thus breaking our tests. - -That’s why production-grade apps should have a single source for all the `Keys` exposed as a global final variable inside `integration_test_keys.dart`. That’s going to look as follows if we already take into account the home page which we want to test next. - -```dart -import 'package:flutter/foundation.dart'; - -class SignInPageKeys { - final emailTextField = const Key('emailTextField'); - final passwordTextField = const Key('passwordTextField'); - final signInButton = const Key('signInButton'); -} - -class HomePageKeys { - final notificationIcon = const Key('notificationIcon'); - final successSnackbar = const Key('successSnackbar'); -} - -class Keys { - final signInPage = SignInPageKeys(); - final homePage = HomePageKeys(); -} - -final keys = Keys(); -``` - - -Feel free to put your page-specific key classes (e.g. `SignInPageKeys`) into separate files in more complex apps. - - -The updated `sign_in_page.dart` code will now look like this: - -```dart -class SignInPage extends StatelessWidget { - const SignInPage({super.key}); - - @override - Widget build(BuildContext context) { - return Scaffold( - ... - body: Padding( - padding: const EdgeInsets.all(16), - child: Form( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - TextFormField( - key: keys.signInPage.emailTextField, - decoration: const InputDecoration( - labelText: 'Email', - ), - ... - ), - const SizedBox(height: 16), - TextFormField( - key: keys.signInPage.passwordTextField, - decoration: const InputDecoration( - labelText: 'Password', - ), - ... - ), - const SizedBox(height: 16), - Builder(builder: (context) { - return ElevatedButton( - key: keys.signInPage.signInButton, - ... - child: const Text('Sign in'), - ); - }), - ], - ), - ), - ), - ); - } -} -``` - -The test code will now also use the `keys` global final variable instead of the hash symbol notation: - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(keys.signInPage.emailTextField).enterText('test@email.com'); - await $(keys.signInPage.passwordTextField).enterText('password'); - await $(keys.signInPage.signInButton).tap(); - }, -); -``` - -Hot-restarting the test won’t show any change in its functionality but it sure is more maintainable and easier to work with. - -### Home Page - -First, let’s add the keys we’ve already created to the `IconButton` and the `SnackBar` shown when the notification has been tapped. - -```dart -class HomePage extends StatefulWidget { - ... -} - -class _HomePageState extends State { - ... - - @override - Widget build(BuildContext context) { - return Scaffold( - appBar: AppBar( - title: const Text('Home'), - actions: [ - IconButton( - key: keys.homePage.notificationIcon, - icon: const Icon(Icons.notification_add), - onPressed: () { - triggerLocalNotification( - onPressed: () { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar( - key: keys.homePage.successSnackbar, - content: const Text('Notification was tapped!'), - ), - ); - }, - onError: () { - ... - }, - ); - }, - ), - ], - ), - ... - ); - } -} -``` - -The first thing the user sees when first navigating to the `HomePage` is a notifications permission dialog. We need to accept it from within the test. Patrol’s native automation makes this as easy as it gets. - - -Native automation allows you to interact with the OS your Flutter app is running on. Patrol currently supports Android, iOS and macOS native interactions. [Learn more from the docs](https://patrol.leancode.co/native/overview) - - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(keys.signInPage.emailTextField).enterText('test@email.com'); - await $(keys.signInPage.passwordTextField).enterText('password'); - await $(keys.signInPage.signInButton).tap(); - await $.native.grantPermissionWhenInUse(); - }, -); -``` - -Hot-restarting the test will work wonderfully the first time, however, once the permission has already been granted, calling `grantPermissionWhenInUse()` will fail. This is not going to be an issue if you use Patrol as a part of your CI/CD process since everytime you test with Patrol there, the app will be built from scratch and no permission will be granted yet. But when we’re writing the test locally with `patrol develop` command, we need to make sure that the permission dialog is visible before trying to accept it. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(keys.signInPage.emailTextField).enterText('test@email.com'); - await $(keys.signInPage.passwordTextField).enterText('password'); - await $(keys.signInPage.signInButton).tap(); - if (await $.native.isPermissionDialogVisible()) { - await $.native.grantPermissionWhenInUse(); - } - }, -); -``` - - -It’s generally a bad practice to add any branching logic within your tests and you should be 100% certain that it cannot introduce any test flakiness before doing so. Checking if a permission dialog is visible is an example of a proper use of branching logic. - - -Next up, we want to tap on the notification icon button and then go to the device home screen to test the notification while the app is running in the background. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(keys.signInPage.emailTextField).enterText('test@email.com'); - await $(keys.signInPage.passwordTextField).enterText('password'); - await $(keys.signInPage.signInButton).tap(); - if (await $.native.isPermissionDialogVisible()) { - await $.native.grantPermissionWhenInUse(); - } - await $(keys.homePage.notificationIcon).tap(); - await $.native.pressHome(); - }, -); -``` - -Once we’re on the home screen, we want to open the notification shade and tap on the notification we get from our app. You can either tap on a notification by index or by finding a text. We know that the title of our notification is “Patrol says hello!” so let’s do the latter. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(keys.signInPage.emailTextField).enterText('test@email.com'); - await $(keys.signInPage.passwordTextField).enterText('password'); - await $(keys.signInPage.signInButton).tap(); - if (await $.native.isPermissionDialogVisible()) { - await $.native.grantPermissionWhenInUse(); - } - await $(keys.homePage.notificationIcon).tap(); - await $.native.pressHome(); - await $.native.openNotifications(); - await $.native.tapOnNotificationBySelector( - Selector(textContains: 'Patrol says hello!'), - ); - }, -); -``` - -Since the notification is delayed by 3 seconds, we have to provide a timeout that’s at least as long in order to wait for the notification to appear - 5 seconds should do the trick here. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - ... - await $.native.openNotifications(); - await $.native.tapOnNotificationBySelector( - Selector(textContains: 'Patrol says hello!'), - timeout: const Duration(seconds: 5), - ); - }, -); -``` - -Lastly, we want to check if the snackbar has been shown after tapping on a notification. We can call `waitUntilVisible()` after selecting it with its key. - -```dart -patrolTest( - 'signs in, triggers a notification, and taps on it', - framePolicy: LiveTestWidgetsFlutterBindingFramePolicy.fullyLive, - ($) async { - initApp(); - await $.pumpWidgetAndSettle(const MainApp()); - await $(keys.signInPage.emailTextField).enterText('test@email.com'); - await $(keys.signInPage.passwordTextField).enterText('password'); - await $(keys.signInPage.signInButton).tap(); - if (await $.native.isPermissionDialogVisible()) { - await $.native.grantPermissionWhenInUse(); - } - await $(keys.homePage.notificationIcon).tap(); - await $.native.pressHome(); - await $.native.openNotifications(); - await $.native.tapOnNotificationBySelector( - Selector(textContains: 'Patrol says hello!'), - timeout: const Duration(seconds: 5), - ); - $(keys.homePage.successSnackbar).waitUntilVisible(); - }, -); -``` - -And just like that, we have now tested the whole flow of the app with Patrol! If any part of the logic breaks, this test will notify us about that sooner than our real users do and that’s what we’re all after!