test(project): add e2e test support with playwright (carbon-design-sy…

…stem#11302)

* test(project): add e2e test support with playwright

* ci(workflows): add vrt-next job to ci workflow

* ci(workflows): update percy to be parallel

* test(playwright): update config

* chore(project): update percy cli

* chore(project): update @percy/storybook

* chore(project): remove cypress from react

* ci(workflows): move vrt-next to vrt

* test(e2e): add snapshotStory utility

* test(e2e): update vrt job to run only in chromium and on vrt tests

* test(breadcrumb): add playwright tests, percy snapshots

* refactor(e2e): update test-utils for snapshots and stories

* test(e2e): add percy config

* docs(testing): add docs for e2e vrt

* refactor(snapshot): update id for percy snapshots

* refactor(snapshot): update id for percy snapshots

* chore(ci): add vrt task to summarize matrix jobs

Co-authored-by: Taylor Jones <[email protected]>

.github/workflows/ci.yml

@@ -111,7 +111,10 @@ jobs:
         if: ${{ steps.filter.outputs.e2e == 'true' }}
         run: yarn test:e2e
 
-  vrt:
+  vrt-runner:
+    strategy:
+      matrix:
+        shard: [1, 2]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -128,11 +131,39 @@ jobs:
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
       - name: Install dependencies
         run: yarn install --immutable --immutable-cache
+      - name: Install browsers
+        run: yarn playwright install --with-deps
       - name: Build project
         run: yarn build --ignore '@carbon/sketch'
+      - name: Build storybook
+        run: yarn workspace @carbon/react build-storybook
+      - name: Run storybook
+        id: storybook
+        run: |
+          npx serve -l 3000 packages/react/storybook-static &
+          pid=$!
+          echo ::set-output name=pid::"$pid"
       - name: Run VRT
         if: github.repository == 'carbon-design-system/carbon'
         env:
           PERCY_TOKEN: c9a21a3fde4fda0a0f822d633426ab26e2ab2c1cba55221d342d4047744c8c24
+          PERCY_PARALLEL_TOTAL: 2
         run: |
-          yarn run percy exec -- yarn workspace @carbon/react test:e2e
+          yarn percy exec --parallel -- yarn playwright test --project chromium --grep @vrt --shard="${{ matrix.shard }}/2"
+      - name: Stop storybook
+        run: kill ${{ steps.storybook.outputs.pid }}
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: playwright-report
+          path: .playwright
+
+  vrt:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: vrt-runner
+    steps:
+      - name: Check VRT Runner job status
+        if: ${{ needs.vrt-runner.result != 'success' }}
+        run: exit 1

.gitignore

@@ -54,3 +54,6 @@ package-lock.json
 
 # Accessibility Verification Testing
 .avt/
+
+# Playwright
+.playwright

.percy.yml

@@ -0,0 +1,7 @@
+version: 2
+snapshot:
+  widths:
+    - 360
+    - 672
+    - 1366
+  minHeight: 1024

docs/testing.md

@@ -0,0 +1,134 @@
+# Testing
+
+<!-- prettier-ignore-start -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+## Table of Contents
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update --> <!-- prettier-ignore-end -->
+
+## Overview
+
+| Task                                                  | Command                                            |
+| :---------------------------------------------------- | :------------------------------------------------- |
+| Run package test                                      | `yarn test:e2e`                                    |
+| Run playwright tests                                  | `yarn playwright test`                             |
+| Run a specific playwright test                        | `yarn playwright test path/to/test-e2e.js`         |
+| Run playwright tests in a specific browser            | `yarn playwright test --browser=chromium`          |
+| Run playwright tests in a specific project            | `yarn playwright test --project=chromium`          |
+| Debug playwright tests                                | `yarn playwright test --debug`                     |
+| Run playwright with browser visible                   | `yarn playwright test --project=chromium --headed` |
+| Run playwright tests that match a specific tag        | `yarn playwright test --grep @tag-name`            |
+| Run playwright tests that do not match a specific tag | `yarn playwright test --grep-invert @tag-name`     |
+
+## End-to-end
+
+We use our end-to-end tests to verify the packaging of libraries along with the
+look and behavior of components in our design system. End-to-end tests for
+packages are run using Jest, components are run using Playwright.
+
+### Playwright
+
+We use Playwright to run end-to-end tests against our components in our Design
+System. We also use it in other situations to prevent visual regressions, like
+in our elements site.
+
+These tests are authored within the `e2e` directory and match the file pattern:
+`*-test.e2e.js`.
+
+#### Developing
+
+When working with Playwright locally, it's important to start up the service
+that you're testing against. For components in React, this will mean starting up
+the React storybook locally by doing the following from the root of the project:
+
+```bash
+cd packages/react
+yarn storybook
+```
+
+One the storybook is loaded, you can run tests against is using the storybook
+test-utils found in `e2e/test-utils/storybook`. A common use-case for testing a
+component is to use Percy to take a snapshot of a component in a particular
+theme from a specific story in storybook.
+
+You can do this by writing the following:
+
+```js
+// e2e/components/component/component-test.e2e.js
+
+'use strict';
+
+const { test } = require('@playwright/test');
+const { themes } = require('../../test-utils/env');
+const { snapshotStory } = require('../../test-utils/storybook');
+
+test('component-name @vrt', ({ page }) => {
+  await snapshotStory(page, {
+    component: 'component',
+    story: 'story-name',
+    theme: 'white',
+  });
+});
+```
+
+You can test this component in multiple themes by writing the following:
+
+```js
+// e2e/components/component/component-test.e2e.js
+
+'use strict';
+
+const { test } = require('@playwright/test');
+const { themes } = require('../../test-utils/env');
+const { snapshotStory } = require('../../test-utils/storybook');
+
+test.describe('component-name @vrt', () => {
+  themes.forEach((theme) => {
+    test(theme, async ({ page }) => {
+      await snapshotStory(page, {
+        component: 'component',
+        story: 'story-name',
+        theme,
+      });
+    });
+  });
+});
+```
+
+If you would like to debug or interact with the test suite, you can use
+Playwright's VS Code integration or run `yarn playwright test` with the
+`--debug` flag to open up the inspector. This will allow you to go through the
+test step-by-step to debug what's going on. It will also allow you to interact
+with the page to quickly find selectors you can use to find items to run tests
+against.
+
+#### Working with snapshots locally
+
+Sometimes you'll want to debug snapshots locally instead of relying on an
+externaly service to get feedback. To do so, you can use the
+`ENABLE_LOCAL_SNAPSHOTS` environment variable to store snapshots locally. Almost
+any playwright command you run can be prefixed with this value in order to store
+screenshots locally.
+
+```bash
+ENABLE_LOCAL_SNAPSHOTS=1 yarn playwright test --project chromium --grep @vrt component-test.e2e.js
+```
+
+**Note: it's important to narrow down tests in order to not generate a lot of
+screenshots locally**
+
+The first time you'll run this command, it will need to generate the baseline
+snapshots for this component. The second time you run it, it will compare the
+snapshots for the current page with what is stored in the screenshot. If the two
+do not match, playwright will report a failure and will provide a link to the
+diff image on your machine.
+
+## FAQ
+
+### Why am I seeing `browserType.launch: Executable doesn't exist at ../path`?
+
+The browser executables need to be installed so that playwright can run tests
+inside chromium, firefox, etc. They can be installed by running
+`yarn playwright install`

e2e/components/accordion/accordion-test.e2e.js

@@ -0,0 +1,24 @@
+/**
+ * Copyright IBM Corp. 2016, 2018
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const { test } = require('@playwright/test');
+const { themes } = require('../../test-utils/env');
+const { snapshotStory } = require('../../test-utils/storybook');
+
+test.describe('accordion @vrt', () => {
+  themes.forEach((theme) => {
+    test(theme, async ({ page }) => {
+      await snapshotStory(page, {
+        component: 'accordion',
+        story: 'accordion-story',
+        theme,
+      });
+    });
+  });
+});

e2e/components/breadcrumb/breadcrumb-test.e2e.js

@@ -0,0 +1,43 @@
+/**
+ * Copyright IBM Corp. 2016, 2018
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const { test } = require('@playwright/test');
+const { themes } = require('../../test-utils/env');
+const { snapshot } = require('../../test-utils/snapshot');
+const { snapshotStory, visitStory } = require('../../test-utils/storybook');
+
+test.describe('breadcrumb @vrt', () => {
+  themes.forEach((theme) => {
+    test.describe(theme, () => {
+      test('breadcrumb', async ({ page }) => {
+        await snapshotStory(page, {
+          component: 'breadcrumb',
+          story: 'breadcrumb-story',
+          theme,
+        });
+      });
+
+      test('breadcrumb with overflow menu', async ({ page }) => {
+        await visitStory(page, {
+          component: 'breadcrumb',
+          story: 'breadcrumb-with-overflow-menu',
+          theme,
+        });
+        await page
+          .locator('button[aria-haspopup="true"][aria-expanded="false"]')
+          .click();
+        await snapshot(page, {
+          component: 'breadcrumb',
+          story: 'breadcrumb-with-overflow-menu',
+          theme,
+        });
+      });
+    });
+  });
+});

e2e/test-utils/env.js

@@ -0,0 +1,14 @@
+/**
+ * Copyright IBM Corp. 2016, 2018
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const themes = ['white', 'g10', 'g90', 'g100'];
+
+module.exports = {
+  themes,
+};

e2e/test-utils/snapshot.js

@@ -0,0 +1,69 @@
+/**
+ * Copyright IBM Corp. 2016, 2018
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const percySnapshot = require('@percy/playwright');
+const { expect } = require('@playwright/test');
+
+/**
+ * Snapshot a page and report it either to Percy or record these snapshots
+ * locally to compare manually
+ */
+async function snapshot(page, context) {
+  const id = getSnapshotId(context);
+
+  if (process.env.ENABLE_LOCAL_SNAPSHOTS) {
+    expect(await page.screenshot()).toMatchSnapshot(`${id}.png`);
+  } else {
+    await percySnapshot(page, id);
+  }
+}
+
+/**
+ * Get a unique identifier for the given context. If the context is a string, it
+ * already represents a valid identifier and will be returned
+ */
+function getSnapshotId(context) {
+  if (typeof context === 'string') {
+    return context;
+  }
+
+  const { component, story, theme } = context;
+  return serialize({
+    theme,
+    component,
+    story,
+  });
+}
+
+/**
+ * Serialize a flat object into a string
+ */
+function serialize(object) {
+  // If local snapshots are enabled, encode this object into a valid filename
+  if (process.env.ENABLE_LOCAL_SNAPSHOTS) {
+    return Object.keys(object).reduce((acc, key, index) => {
+      if (index === 0) {
+        return `${key}=${object[key]}`;
+      }
+      return `${acc}-${key}=${object[key]}`;
+    }, '');
+  }
+
+  return Object.keys(object).reduce((acc, key, index) => {
+    if (index === 0) {
+      return `${key}: ${object[key]}`;
+    }
+    return `${acc}, ${key}: ${object[key]}`;
+  }, '');
+}
+
+module.exports = {
+  snapshot,
+  getSnapshotId,
+};