From ec4fdc8759c3def8446e4a555dfcf9079b855e59 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Wed, 15 May 2024 11:46:16 +0200 Subject: [PATCH 01/28] build: implement visual regression testing with @web/test-runner (#2624) This PR sets up the basic configuration for visual regression testing with `@web/test-runner`. `web-test-runner.config.js` contains a new test group, which is only activated when explicitly called with `yarn test:visual-regression`. This test group introduces a new file pattern `*.snapshot.spec.ts`, which contains the visual regression tests. Additionally, there is a new GitHub Actions workflow, which runs the visual regression tests and saves failed screenshots in an artifact. This artifact is then downloaded in the secure workflow, which adds an `in_progress` check, if any failed screenshots exist. We do not want to save the snapshot png in the repository, as this would bloat it considerably. Due to this, we use containers as an external storage of the png files. The process works as follows: Each continuous integration run on our main branch creates/updates a container `baseline` (https://github.com/lyne-design-system/lyne-components/pkgs/container/lyne-components%2Fvisual-regression). For any other CI run (in the secure context) this container is used as a service and the png files are downloaded when needed. Additionally, the baseline image is hosted at `https://lyne-visual-regression-baseline.app.sbb.ch/` in order to enable local visual regression testing. As for local visual regression testing; To avoid diffs between operating system browser differences, visual regression testing is only directly run in a Linux environment. If it is executed in any other OS, it will try to run it in a container to prevent OS differences showing up in the snapshots. --------- Co-authored-by: Jeremias Peier --- .github/default.conf | 3 + .github/workflows/container-image-cleanup.yml | 68 ++++++ .../continuous-integration-secure.yml | 6 +- .github/workflows/continuous-integration.yml | 72 +++++++ .github/workflows/preview-image-cleanup.yml | 61 ------ .github/workflows/release-please.yml | 18 +- Dockerfile | 4 +- eslint.config.js | 7 + package.json | 8 +- src/components/accordion/accordion.e2e.ts | 4 +- .../button/button/button.snapshot.spec.ts | 130 ++++++++++++ .../button/common/button-common.scss | 2 +- .../button/common/common-stories.ts | 23 +- src/components/core/testing.ts | 1 - src/components/core/testing/private.ts | 4 + .../testing/private/a11y-tree-snapshot.ts | 3 +- .../core/testing/private/describe-each.ts | 40 ++++ .../testing/private/describe-viewports.ts | 53 +++++ .../core/testing/private/fixture.ts | 3 +- .../core/testing/{ => private}/platform.ts | 7 + .../private/visual-regression-snapshot.ts | 94 +++++++++ src/components/core/testing/test-setup-ssr.ts | 40 ++-- src/components/core/testing/test-setup.ts | 112 +++++++++- src/visual-regression-app/index.html | 32 +++ .../src/components/overview/overview.scss | 9 + .../src/components/overview/overview.ts | 70 ++++++ .../fullscreen-diff/fullscreen-diff.scss | 21 ++ .../fullscreen-diff/fullscreen-diff.ts | 58 +++++ .../test-case/image-diff/image-diff.scss | 73 +++++++ .../test-case/image-diff/image-diff.ts | 144 +++++++++++++ .../test-case-filter/test-case-filter.scss | 14 ++ .../test-case-filter/test-case-filter.ts | 93 ++++++++ .../src/components/test-case/test-case.scss | 49 +++++ .../src/components/test-case/test-case.ts | 150 +++++++++++++ src/visual-regression-app/src/interfaces.ts | 11 + src/visual-regression-app/src/main.ts | 44 ++++ src/visual-regression-app/src/screenshots.ts | 142 +++++++++++++ src/visual-regression-app/tsconfig.json | 13 ++ src/visual-regression-app/vite.config.ts | 199 ++++++++++++++++++ src/vite-env.d.ts | 4 + .../baseline.Dockerfile | 13 ++ .../baseline.nginx.conf | 50 +++++ .../diff-app.Dockerfile | 11 + .../etag-map-generation.sh | 10 + tools/visual-regression-testing/exec.ts | 104 +++++++++ .../testing.Dockerfile | 7 + .../testing.Dockerfile.dockerignore | 3 + tools/web-test-runner/index.js | 1 + .../visual-regression-plugin-config.js | 63 ++++++ web-test-runner.config.js | 98 ++++++--- yarn.lock | 59 ++++++ 51 files changed, 2163 insertions(+), 145 deletions(-) create mode 100644 .github/workflows/container-image-cleanup.yml delete mode 100644 .github/workflows/preview-image-cleanup.yml create mode 100644 src/components/button/button/button.snapshot.spec.ts create mode 100644 src/components/core/testing/private/describe-each.ts create mode 100644 src/components/core/testing/private/describe-viewports.ts rename src/components/core/testing/{ => private}/platform.ts (82%) create mode 100644 src/components/core/testing/private/visual-regression-snapshot.ts create mode 100644 src/visual-regression-app/index.html create mode 100644 src/visual-regression-app/src/components/overview/overview.scss create mode 100644 src/visual-regression-app/src/components/overview/overview.ts create mode 100644 src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss create mode 100644 src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts create mode 100644 src/visual-regression-app/src/components/test-case/image-diff/image-diff.scss create mode 100644 src/visual-regression-app/src/components/test-case/image-diff/image-diff.ts create mode 100644 src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.scss create mode 100644 src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.ts create mode 100644 src/visual-regression-app/src/components/test-case/test-case.scss create mode 100644 src/visual-regression-app/src/components/test-case/test-case.ts create mode 100644 src/visual-regression-app/src/interfaces.ts create mode 100644 src/visual-regression-app/src/main.ts create mode 100644 src/visual-regression-app/src/screenshots.ts create mode 100644 src/visual-regression-app/tsconfig.json create mode 100644 src/visual-regression-app/vite.config.ts create mode 100644 tools/visual-regression-testing/baseline.Dockerfile create mode 100644 tools/visual-regression-testing/baseline.nginx.conf create mode 100644 tools/visual-regression-testing/diff-app.Dockerfile create mode 100755 tools/visual-regression-testing/etag-map-generation.sh create mode 100644 tools/visual-regression-testing/exec.ts create mode 100644 tools/visual-regression-testing/testing.Dockerfile create mode 100644 tools/visual-regression-testing/testing.Dockerfile.dockerignore create mode 100644 tools/web-test-runner/visual-regression-plugin-config.js diff --git a/.github/default.conf b/.github/default.conf index dee95097b5..bdb9834ab9 100644 --- a/.github/default.conf +++ b/.github/default.conf @@ -2,6 +2,9 @@ tcp_nopush on; tcp_nodelay on; types_hash_max_size 2048; +# Suppresses the nginx version in the Server header. +server_tokens off; + # Determine if it's a valid origin and set it in the $cors variable. map "$http_origin" $cors { default ''; diff --git a/.github/workflows/container-image-cleanup.yml b/.github/workflows/container-image-cleanup.yml new file mode 100644 index 0000000000..0788b6f553 --- /dev/null +++ b/.github/workflows/container-image-cleanup.yml @@ -0,0 +1,68 @@ +name: Container Image Cleanup + +on: + workflow_dispatch: {} + schedule: + - cron: '0 3 * * *' + +permissions: + packages: write + +jobs: + container-image-cleanup: + runs-on: ubuntu-latest + env: + CLOSED_PR_RETENTION_DAYS: 5 + PACKAGE_NAMES: storybook-preview,visual-regression + steps: + - uses: actions/github-script@v7 + with: + script: | + const { owner, repo } = context.repo; + const pullRequests = await github.paginate( + github.rest.pulls.list.endpoint.merge({ owner, repo, state: 'all' }) + ); + const retentionPivot = + new Date(Date.now() - (+process.env.CLOSED_PR_RETENTION_DAYS * 24 * 60 * 60 * 1000)); + const olderThanTwoWeeks = (date) => new Date(date) < retentionPivot; + const isExpiredPrTag = (version) => { + const prNumber = +version.metadata?.container?.tags + ?.find((t) => t.match(/(preview-pr|pr)(\d+)/))?.match(/(preview-pr|pr)(\d+)/)[2]; + const pr = pullRequests.find((p) => p.number === prNumber); + return !!prNumber && pr?.state === 'closed' && olderThanTwoWeeks(pr.closed_at); + }; + + const packageNames = process.env.PACKAGE_NAME.split(',').map((n) => n.trim()); + let packageDeletionFailed = false; + for (const packageName of packageNames) { + const params = { + package_type: 'container', + package_name: `${repo}/${packageNames}`, + username: owner + }; + const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + for (const version of versions.filter(isExpiredPrTag)) { + try { + await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); + console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); + } catch(e) { + console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); + console.error(e); + packageDeletionFailed = true; + } + } + } + + if (packageDeletionFailed) { + throw new Error('A package deletion failed, please check the log.'); + } + - uses: actions/delete-package-versions@v4 + with: + package-name: lyne-components/storybook-preview + package-type: container + delete-only-untagged-versions: 'true' + - uses: actions/delete-package-versions@v4 + with: + package-name: lyne-components/visual-regression + package-type: container + delete-only-untagged-versions: 'true' diff --git a/.github/workflows/continuous-integration-secure.yml b/.github/workflows/continuous-integration-secure.yml index 93b3f8fb07..c33a9a2a46 100644 --- a/.github/workflows/continuous-integration-secure.yml +++ b/.github/workflows/continuous-integration-secure.yml @@ -46,7 +46,7 @@ jobs: uses: actions/github-script@v7 with: script: | - const environment = process.env.PR_NUMBER ? `preview-pr${process.env.PR_NUMBER}` : 'main'; + const environment = process.env.PR_NUMBER ? `pr${process.env.PR_NUMBER}` : 'main'; const payload = { owner: context.repo.owner, repo: context.repo.repo, environment }; const { data: deployment } = await github.rest.repos.createDeployment({ ...payload, @@ -127,8 +127,8 @@ jobs: run-id: ${{ github.event.workflow_run.id }} github-token: ${{ secrets.GH_ACTIONS_ARTIFACT_DOWNLOAD }} - - name: Build diff-app - run: yarn build:diff-app + - name: Build visual-regression-app + run: yarn build:visual-regression-app - name: Create check if changed uses: actions/github-script@v7 diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 2beaf55455..9050f779fc 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -7,6 +7,9 @@ concurrency: permissions: read-all +env: + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + jobs: lint: runs-on: ubuntu-latest @@ -23,6 +26,7 @@ jobs: integrity: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -38,6 +42,7 @@ jobs: test: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -63,6 +68,7 @@ jobs: build: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -151,3 +157,69 @@ jobs: zip: true onlyChanged: true externals: '**/components/core/styles/**/*.scss' + + visual-regression: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + needs: test + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8050:8080 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run visual regression baseline generation + run: yarn test:visual-regression + env: + NODE_ENV: production + - name: Store visual regression output + uses: actions/upload-artifact@v4 + with: + name: visual-regression-screenshots + path: dist/screenshots-artifact/ + + visual-regression-baseline: + runs-on: ubuntu-latest + permissions: + packages: write + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: test + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8080:8050 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run visual regression baseline generation + run: yarn test:visual-regression --update-visual-baseline + env: + NODE_ENV: production + + - name: Build and push visual regression baseline + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + docker build \ + --file tools/visual-regression-testing/baseline.Dockerfile \ + --tag $IMAGE_REPO_VISUAL_REGRESSION:baseline \ + . + docker push $IMAGE_REPO_VISUAL_REGRESSION:baseline + env: + DOCKER_BUILDKIT: 1 diff --git a/.github/workflows/preview-image-cleanup.yml b/.github/workflows/preview-image-cleanup.yml deleted file mode 100644 index 3a9a8df931..0000000000 --- a/.github/workflows/preview-image-cleanup.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Preview Image Cleanup - -on: - workflow_dispatch: {} - schedule: - - cron: '0 5 * * *' - -permissions: - packages: write - -jobs: - preview-image: - runs-on: ubuntu-latest - env: - CLOSED_PR_RETENTION_DAYS: 14 - PACKAGE_NAME: storybook-preview - PR_TAG_PREFIX: preview-pr - steps: - - uses: actions/github-script@v7 - with: - script: | - const { owner, repo } = context.repo; - const pullRequests = await github.paginate( - github.rest.pulls.list.endpoint.merge({ owner, repo, state: 'all' }) - ); - const twoWeeksAgo = - new Date(Date.now() - (+process.env.CLOSED_PR_RETENTION_DAYS * 24 * 60 * 60 * 1000)); - const olderThanTwoWeeks = (date) => new Date(date) < twoWeeksAgo; - const isExpiredPrTag = (version) => { - const prNumber = +version.metadata?.container?.tags - ?.find((t) => t.startsWith(process.env.PR_TAG_PREFIX))?.split(process.env.PR_TAG_PREFIX)[1]; - const pr = pullRequests.find((p) => p.number === prNumber); - return !!prNumber && pr?.state === 'closed' && olderThanTwoWeeks(pr.closed_at); - }; - - const params = { - package_type: 'container', - package_name: `${repo}/${process.env.PACKAGE_NAME}`, - username: owner - }; - const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); - let packageDeletionFailed = false; - for (const version of versions.filter(isExpiredPrTag)) { - try { - await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); - console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); - } catch(e) { - console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); - console.error(e); - packageDeletionFailed = true; - } - } - - if (packageDeletionFailed) { - throw new Error('A package deletion failed, please check the log.'); - } - - uses: actions/delete-package-versions@v4 - with: - package-name: lyne-components/storybook-preview - package-type: container - delete-only-untagged-versions: 'true' diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 21720da636..9d800563ab 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -35,7 +35,7 @@ jobs: if: needs.release-please.outputs.releases_created runs-on: ubuntu-latest env: - IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook + IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook VERSION: ${{ needs.release-please.outputs.version }} steps: - uses: actions/checkout@v4 @@ -66,15 +66,15 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:$VERSION -t $IMAGE_REPO:latest . + run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest . env: DOCKER_BUILDKIT: 1 - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:$VERSION + run: docker push $IMAGE_REPO_STORYBOOK:$VERSION - name: 'Container: Publish image as latest' - run: docker push $IMAGE_REPO:latest + run: docker push $IMAGE_REPO_STORYBOOK:latest - name: Generate chromatic stories run: yarn generate:chromatic-stories @@ -98,7 +98,7 @@ jobs: if: needs.release-please.outputs.releases_created != true runs-on: ubuntu-latest env: - IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook + IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook VERSION: ${{ needs.release-please.outputs.version }} steps: - uses: actions/checkout@v4 @@ -116,13 +116,13 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:dev . + run: docker build --tag $IMAGE_REPO_STORYBOOK:dev . env: DOCKER_BUILDKIT: 1 - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:dev + run: docker push $IMAGE_REPO_STORYBOOK:dev - name: Generate chromatic stories run: yarn generate:chromatic-stories diff --git a/Dockerfile b/Dockerfile index 57210bfa3f..bd0cd09679 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM nginxinc/nginx-unprivileged:stable +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components # Copy nginx configuration COPY ./.github/default.conf /etc/nginx/conf.d/default.conf diff --git a/eslint.config.js b/eslint.config.js index 673978b295..998116ce58 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -47,6 +47,13 @@ export default [ 'plugin:import-x/typescript', ), eslintPluginLyne.default.configs.recommended, + { + files: ['src/visual-regression-app/**/*.ts'], + rules: { + 'lyne/custom-element-class-name-rule': 'off', + 'import-x/namespace': 'off', + }, + }, { files: ['**/*.ts'], rules: { diff --git a/package.json b/package.json index e8939fbd54..682841a444 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "build:react:production": "vite build --config src/react/vite.config.ts", "build:react:development": "NODE_ENV=development vite build --mode development --config src/react/vite.config.ts", "build:storybook": "storybook build --quiet --output-dir dist/storybook --stats-json", + "build:visual-regression-app": "vite build --config src/visual-regression-app/vite.config.ts", "build": "npm-run-all --sequential build:components build:react build:storybook", "docs": "npm-run-all --sequential docs:manifest docs:to-md", "docs:manifest": "custom-elements-manifest analyze --config tools/manifest/custom-elements-manifest.config.js", @@ -43,16 +44,19 @@ "lint:yml": "eslint \"**/*.{yml,yaml}\"", "lint:styles": "stylelint \"**/*.scss\"", "lint:lit": "yarn lit-analyzer \"src/**/*.ts\"", - "lint:circular-imports": "madge --circular --extensions ts ./src", + "lint:circular-imports": "madge --circular --ts-config ./tsconfig.json --extensions ts ./src", "lint:tsc": "npm-run-all --sequential lint:tsc:*", "lint:tsc:components": "tsc --noEmit --project src/components/tsconfig.json", "lint:tsc:components-spec": "tsc --noEmit --project src/components/tsconfig.spec.json", + "lint:tsc:visual-regression-app": "tsc --noEmit --project src/visual-regression-app/tsconfig.json", "start": "storybook dev -p 6006", + "start:visual-regression-app": "vite --config src/visual-regression-app/vite.config.ts", "test": "wtr --coverage", "test:snapshot": "yarn test:csr --ci --update-snapshots", "test:csr": "wtr --group default", "test:ssr:hydrated": "wtr --group e2e-ssr-hydrated", "test:ssr:non-hydrated": "wtr --group e2e-ssr-non-hydrated", + "test:visual-regression": "tsx tools/visual-regression-testing/exec.ts --group=visual-regression --all-browsers", "prepare": "husky" }, "dependencies": { @@ -65,6 +69,7 @@ "@custom-elements-manifest/to-markdown": "0.1.0", "@eslint/eslintrc": "3.0.2", "@eslint/js": "9.2.0", + "@lit-labs/router": "^0.1.3", "@lit-labs/testing": "0.2.3", "@lit/react": "1.0.5", "@open-wc/lit-helpers": "0.7.0", @@ -90,6 +95,7 @@ "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", "@web/test-runner-puppeteer": "0.16.0", + "@web/test-runner-visual-regression": "0.9.0", "chromatic": "11.3.2", "custom-elements-manifest": "2.1.0", "date-fns": "3.6.0", diff --git a/src/components/accordion/accordion.e2e.ts b/src/components/accordion/accordion.e2e.ts index a01e2ff39c..be20fbb172 100644 --- a/src/components/accordion/accordion.e2e.ts +++ b/src/components/accordion/accordion.e2e.ts @@ -2,8 +2,8 @@ import { assert, expect } from '@open-wc/testing'; import { nothing } from 'lit'; import { html } from 'lit/static-html.js'; -import { fixture } from '../core/testing/private.js'; -import { waitForCondition, waitForLitRender, EventSpy, isSsr } from '../core/testing.js'; +import { fixture, isSsr } from '../core/testing/private.js'; +import { waitForCondition, waitForLitRender, EventSpy } from '../core/testing.js'; import { SbbExpansionPanelElement, type SbbExpansionPanelHeaderElement, diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts new file mode 100644 index 0000000000..3c99e0e7f3 --- /dev/null +++ b/src/components/button/button/button.snapshot.spec.ts @@ -0,0 +1,130 @@ +import { html } from 'lit'; + +import { + describeEach, + describeViewports, + fixture, + isVisualRegressionRun, + testVisualDiff, + testVisualDiffHover, + visualRegressionSnapshot, + visualRegressionWrapperStyles, +} from '../../core/testing/private.js'; + +import './button.js'; + +describe(`sbb-button`, () => { + if (isVisualRegressionRun()) { + describe('visual-regression', () => { + let root: HTMLElement; + + const cases = { + disabled: [false, true], + negative: [false, true], + state: [ + { icon: undefined, text: 'Button' }, + { icon: 'arrow-right-small', text: 'Button' }, + { icon: 'arrow-right-small', text: '' }, + ], + }; + + // 'l' as default is covered by other cases. + const sizeCases = { size: ['s', 'm'] }; + + describeViewports({ viewports: ['zero', 'medium'] }, () => { + describeEach(cases, ({ disabled, negative, state }) => { + beforeEach(async () => { + root = await fixture( + html`
+ + ${state.text} + +
`, + ); + }); + + visualRegressionSnapshot(() => root); + }); + + describeEach(sizeCases, ({ size }) => { + beforeEach(async () => { + root = await fixture( + html`
+ Button +
`, + ); + }); + + testVisualDiff(() => root); + }); + + describe('with ellipsis', () => { + beforeEach(async () => { + root = await fixture( + html`
+ + Button with long text + +
`, + ); + }); + + testVisualDiff(() => root); + }); + + describe('wide width', () => { + beforeEach(async () => { + root = await fixture( + html`
+ + Wide Button + +
`, + ); + }); + + testVisualDiff(() => root); + }); + + describe('slotted icon', () => { + beforeEach(async () => { + root = await fixture( + html`
+ + Button + + +
`, + ); + }); + + testVisualDiff(() => root); + testVisualDiffHover(() => root); + }); + + describe('with hidden slot', () => { + beforeEach(async () => { + root = await fixture( + html`
+ + Button + + +
`, + ); + }); + + testVisualDiff(() => root); + }); + }); + }); + } +}); diff --git a/src/components/button/common/button-common.scss b/src/components/button/common/button-common.scss index 6f04586e62..22af5479ff 100644 --- a/src/components/button/common/button-common.scss +++ b/src/components/button/common/button-common.scss @@ -26,7 +26,7 @@ $icon-only: ':where([data-slot-names~=icon], [icon-name]):not([data-slot-names~= --sbb-button-border-radius: var(--sbb-border-radius-infinity); --sbb-button-min-height: var(--sbb-size-element-m); --sbb-button-transition-duration: var( - --sbb-disable-animation-time, + --sbb-disable-animation-zero-time, var(--sbb-animation-duration-2x) ); --sbb-button-transition-easing-function: var(--sbb-animation-easing); diff --git a/src/components/button/common/common-stories.ts b/src/components/button/common/common-stories.ts index 45bf52f00d..277d997b29 100644 --- a/src/components/button/common/common-stories.ts +++ b/src/components/button/common/common-stories.ts @@ -9,7 +9,6 @@ import type { WebComponentsRenderer, } from '@storybook/web-components'; import type { TemplateResult } from 'lit'; -import { styleMap } from 'lit/directives/style-map.js'; import { html, unsafeStatic } from 'lit/static-html.js'; import { sbbSpread } from '../../../storybook/helpers/spread.js'; @@ -17,11 +16,6 @@ import { sbbSpread } from '../../../storybook/helpers/spread.js'; import '../../icon.js'; import '../../loading-indicator.js'; -const focusStyle = (context: StoryContext): Record => - context.args.negative - ? { '--sbb-focus-outline-color': 'var(--sbb-focus-outline-color-dark)' } - : {}; - /* eslint-disable lit/binding-positions, @typescript-eslint/naming-convention */ const Template = ({ tag, text, active, focusVisible, ...args }: Args): TemplateResult => html` <${unsafeStatic(tag)} ${sbbSpread(args)} ?data-active=${active} ?data-focus-visible=${focusVisible}> @@ -229,14 +223,13 @@ export const withHiddenSlottedIcon: StoryObj = { }; export const commonDecorators = [ - (story: () => WebComponentsRenderer['storyResult'], context: StoryContext) => html` -
- ${story()} -
- `, + (story: () => WebComponentsRenderer['storyResult'], context: StoryContext) => + context.args.negative + ? html` +
+ ${story()} +
+ ` + : story(), withActions as Decorator, ]; diff --git a/src/components/core/testing.ts b/src/components/core/testing.ts index 032a69bed9..428ae3f973 100644 --- a/src/components/core/testing.ts +++ b/src/components/core/testing.ts @@ -1,6 +1,5 @@ export * from './testing/event-spy.js'; export * from './testing/mocha-extensions.js'; -export * from './testing/platform.js'; export * from './testing/scroll.js'; export * from './testing/wait-for-condition.js'; export * from './testing/wait-for-render.js'; diff --git a/src/components/core/testing/private.ts b/src/components/core/testing/private.ts index 57b13775f6..ba1753f948 100644 --- a/src/components/core/testing/private.ts +++ b/src/components/core/testing/private.ts @@ -1,5 +1,9 @@ export * from './private/a11y-tree-snapshot.js'; +export * from './private/describe-each.js'; +export * from './private/describe-viewports.js'; export * from './private/dispatch-events.js'; export * from './private/event-objects.js'; export * from './private/fixture.js'; +export * from './private/platform.js'; export * from './private/type-in-element.js'; +export * from './private/visual-regression-snapshot.js'; diff --git a/src/components/core/testing/private/a11y-tree-snapshot.ts b/src/components/core/testing/private/a11y-tree-snapshot.ts index db975d0875..57e4c576ba 100644 --- a/src/components/core/testing/private/a11y-tree-snapshot.ts +++ b/src/components/core/testing/private/a11y-tree-snapshot.ts @@ -1,4 +1,4 @@ -import { aTimeout, expect, fixture } from '@open-wc/testing'; +import { aTimeout, expect, fixture, fixtureCleanup } from '@open-wc/testing'; import { a11ySnapshot } from '@web/test-runner-commands'; import type { TemplateResult } from 'lit'; import { html } from 'lit/static-html.js'; @@ -18,6 +18,7 @@ async function a11yTreeEqualSnapshot(): Promise { const htmlWrapper = await fixture(html`

${JSON.stringify(snapshot, null, 2)}

`); await expect(htmlWrapper).to.be.equalSnapshot(); + fixtureCleanup(); } /** diff --git a/src/components/core/testing/private/describe-each.ts b/src/components/core/testing/private/describe-each.ts new file mode 100644 index 0000000000..c9889bc6dd --- /dev/null +++ b/src/components/core/testing/private/describe-each.ts @@ -0,0 +1,40 @@ +function generateDescribeName(payload: Record): string { + return Object.entries(payload) + .map( + ([key, value]) => + `${key}=${typeof value === 'object' && value ? `(${generateDescribeName(value as Record)})` : value}`, + ) + .join(', '); +} + +function partialDescribeEach>( + cases: T, + payload: Record, + suiteRun: (params: { [K in keyof T]: T[K][number] }) => void, +): void { + const [key, ...keys] = Object.keys(cases); + const values = cases[key]; + if (keys.length) { + const partialCases = keys.reduce( + (current, next) => Object.assign(current, { [next]: cases[next] }), + {} as T, + ); + for (const value of values) { + partialDescribeEach(partialCases, { ...payload, [key]: value }, suiteRun); + } + } else { + for (const value of values) { + const finalPayload = { ...payload, [key]: value }; + describe(generateDescribeName(finalPayload), function () { + suiteRun.call(this, finalPayload); + }); + } + } +} + +export function describeEach>( + cases: T, + suiteRun: (params: { [K in keyof T]: T[K][number] }) => void, +): void { + partialDescribeEach(cases, {} as Record, suiteRun); +} diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts new file mode 100644 index 0000000000..4e9d33e868 --- /dev/null +++ b/src/components/core/testing/private/describe-viewports.ts @@ -0,0 +1,53 @@ +import { + SbbBreakpointLargeMin, + SbbBreakpointMediumMin, + SbbBreakpointMicroMin, + SbbBreakpointSmallMin, + SbbBreakpointUltraMin, + SbbBreakpointWideMin, +} from '@sbb-esta/lyne-design-tokens'; +import { setViewport } from '@web/test-runner-commands'; + +const viewportSizes = { + zero: 320, + micro: SbbBreakpointMicroMin, + small: SbbBreakpointSmallMin, + medium: SbbBreakpointMediumMin, + large: SbbBreakpointLargeMin, + wide: SbbBreakpointWideMin, + ultra: SbbBreakpointUltraMin, +} as const; + +export interface DescribeViewportOptions { + viewports?: (keyof typeof viewportSizes)[]; + viewportHeight?: number; +} + +export function describeViewports( + options: DescribeViewportOptions, + fn: (this: Mocha.Suite) => void, +): void; +export function describeViewports(fn: (this: Mocha.Suite) => void): void; +export function describeViewports( + optionsOrFn: DescribeViewportOptions | ((this: Mocha.Suite) => void), + fn?: (this: Mocha.Suite) => void, +): void { + const options = typeof optionsOrFn === 'object' ? optionsOrFn : {}; + fn ??= optionsOrFn as (this: Mocha.Suite) => void; + let viewportSizeTests = Object.entries(viewportSizes); + if (options.viewports?.length) { + viewportSizeTests = viewportSizeTests.filter(([key, _value]) => + options.viewports?.includes(key as keyof typeof viewportSizes), + ); + } + + for (const [size, value] of viewportSizeTests) { + describe(`viewport=${size}`, function () { + before(async () => { + await setViewport({ width: value, height: options.viewportHeight ?? 400 }); + }); + + fn.call(this); + }); + } +} diff --git a/src/components/core/testing/private/fixture.ts b/src/components/core/testing/private/fixture.ts index 7e3788ee48..db042554d3 100644 --- a/src/components/core/testing/private/fixture.ts +++ b/src/components/core/testing/private/fixture.ts @@ -1,8 +1,9 @@ import type { TemplateResult } from 'lit'; -import { isHydratedSsr, isNonHydratedSsr } from '../platform.js'; import { waitForLitRender } from '../wait-for-render.js'; +import { isHydratedSsr, isNonHydratedSsr } from './platform.js'; + // Copied from @lit-labs/testing/lib/fixtures/fixture-options.d.ts interface FixtureOptions { /** diff --git a/src/components/core/testing/platform.ts b/src/components/core/testing/private/platform.ts similarity index 82% rename from src/components/core/testing/platform.ts rename to src/components/core/testing/private/platform.ts index eb23176b41..28a308ec16 100644 --- a/src/components/core/testing/platform.ts +++ b/src/components/core/testing/private/platform.ts @@ -32,3 +32,10 @@ export const isNonHydratedSsr = (): boolean => * Returns true, if this is run in an SSR test group. */ export const isSsr = (): boolean => isHydratedSsr() || isNonHydratedSsr(); + +/** + * This is a custom implementation. + * Returns true, if this is run in the visual regression test group. + */ +export const isVisualRegressionRun = (): boolean => + !isServer && (globalThis as any).testGroup === 'visual-regression'; diff --git a/src/components/core/testing/private/visual-regression-snapshot.ts b/src/components/core/testing/private/visual-regression-snapshot.ts new file mode 100644 index 0000000000..f1a78a9e61 --- /dev/null +++ b/src/components/core/testing/private/visual-regression-snapshot.ts @@ -0,0 +1,94 @@ +import { aTimeout } from '@open-wc/testing'; +import { resetMouse, sendKeys, sendMouse } from '@web/test-runner-commands'; +import { visualDiff } from '@web/test-runner-visual-regression'; + +export function imageName(test: Mocha.Runnable): string { + return test!.fullTitle().replaceAll(', ', '-').replaceAll(' ', '_'); +} + +function findElementCenter(snapshotElement: () => HTMLElement): [number, number] { + const element = snapshotElement(); + // Look for the first sbb-* element and get center of the element to + // move the mouse cursor over it. + const positionElement = element.localName.startsWith('sbb-') + ? element + : element.firstElementChild!; + const position = positionElement.getBoundingClientRect(); + return [ + Math.round(position.x + position.width / 2), + Math.round(position.y + position.height / 2), + ]; +} + +export function testVisualDiff(snapshotElement: () => HTMLElement): void { + it('default', async function () { + await visualDiff(snapshotElement(), imageName(this.test!)); + }); +} + +export function testVisualDiffFocus(snapshotElement: () => HTMLElement): void { + it('focus', async function () { + await sendKeys({ press: 'Tab' }); + await aTimeout(50); + await visualDiff(snapshotElement(), imageName(this.test!)); + }); +} + +export function testVisualDiffHover( + snapshotElement: () => HTMLElement, + stateElement?: (() => HTMLElement) | undefined, +): void { + it('hover', async function () { + const position = findElementCenter(stateElement ?? snapshotElement); + + try { + await sendMouse({ type: 'move', position }); + await aTimeout(5); + await visualDiff(snapshotElement(), imageName(this.test!)); + } finally { + await resetMouse(); + } + }); +} + +export function testVisualDiffActive( + snapshotElement: () => HTMLElement, + stateElement?: (() => HTMLElement) | undefined, +): void { + it('active', async function () { + const position = findElementCenter(stateElement ?? snapshotElement); + + try { + await sendMouse({ type: 'move', position }); + await sendMouse({ type: 'down' }); + await aTimeout(5); + await visualDiff(snapshotElement(), imageName(this.test!)); + } finally { + await resetMouse(); + } + }); +} + +export function visualRegressionSnapshot( + snapshotElement: () => HTMLElement, + stateElement?: () => HTMLElement, +): void { + testVisualDiff(snapshotElement); + testVisualDiffFocus(snapshotElement); + testVisualDiffHover(snapshotElement, stateElement); + testVisualDiffActive(snapshotElement, stateElement); +} + +/** + * Generates styles for the wrapper element in visual regression testing. + * @param options.padding Defaults to 2rem to include shadows and similar styles. + * @param options.backgroundColor Defaults to white. + */ +export function visualRegressionWrapperStyles( + options: { + padding?: string; + backgroundColor?: string; + } = {}, +): string { + return `padding: ${options.padding ?? '2rem'};background-color: ${options.backgroundColor ?? 'var(--sbb-color-white)'};`; +} diff --git a/src/components/core/testing/test-setup-ssr.ts b/src/components/core/testing/test-setup-ssr.ts index 7d08fd62df..76313d001f 100644 --- a/src/components/core/testing/test-setup-ssr.ts +++ b/src/components/core/testing/test-setup-ssr.ts @@ -1,29 +1,33 @@ +import { isServer } from 'lit'; + import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -function setupIconConfig(): void { - const testNamespaces = ['default', 'picto']; - const icon: SbbIconConfig = { - interceptor: ({ namespace, name, request }) => { - if (testNamespaces.includes(namespace)) { - const dimension = name.endsWith('-large') ? 48 : name.endsWith('-medium') ? 36 : 24; - return Promise.resolve( - ` { + if (testNamespaces.includes(namespace)) { + const dimension = name.endsWith('-large') ? 48 : name.endsWith('-medium') ? 36 : 24; + return Promise.resolve( + ` `, - ); - } - return request(); - }, - }; + ); + } + return request(); + }, + }; - mergeConfig({ - icon, - }); -} + mergeConfig({ + icon, + }); + } -setupIconConfig(); + setupIconConfig(); +} diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index 5f90f96f6d..02667333c6 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -1,10 +1,116 @@ +import { getSvgContent } from '../../icon.js'; import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -import { isHydratedSsr } from './platform.js'; +import { isHydratedSsr, isVisualRegressionRun } from './private.js'; -function setupIconConfig(): void { +if (isVisualRegressionRun()) { + const preloadedIcons = [ + 'add-stop', + 'alternative', + 'app-icon-medium', + 'app-icon-small', + 'arrow-long-right-small', + 'arrow-right-small', + 'arrows-circle-small', + 'arrows-right-left-small', + 'backpack-medium', + 'battery-level-empty-small', + 'battery-level-high-small', + 'bicycle-medium', + 'calendar-small', + 'cancellation', + 'chevron-small-down-medium', + 'chevron-small-down-small', + 'chevron-small-left-small', + 'chevron-small-right-small', + 'chevron-small-up-small', + 'circle-cross-small', + 'circle-information-large', + 'circle-information-medium', + 'circle-information-small', + 'circle-minus-small', + 'circle-plus-medium', + 'circle-plus-small', + 'circle-tick-small', + 'clock-small', + 'coins-small', + 'construction', + 'container-small', + 'context-menu-small', + 'cross-small', + 'delay', + 'diamond-small', + 'disruption', + 'dog-medium', + 'dog-small', + 'exclamation-point-small', + 'exit-small', + 'eye-small', + 'face-smiling-small', + 'folder-open-medium', + 'folder-open-small', + 'globe-small', + 'hamburger-menu-small', + 'heart-medium', + 'house-small', + 'info', + 'link-small', + 'location-pin-map-small', + 'magnifying-glass-small', + 'minus-small', + 'missed-connection', + 'pen-medium', + 'pen-small', + 'pie-medium', + 'pie-small', + 'platform-change', + 'plus-medium', + 'plus-small', + 'qrcode-small', + 'replacementbus', + 'reroute', + 'sa-abteilkinderwagen', + 'sa-b', + 'sa-bz', + 'sa-ci', + 'sa-fz', + 'sa-nf', + 'sa-r', + 'sa-rr', + 'sa-rs', + 'sa-wr', + 'shopping-cart-small', + 'swisspass-medium', + 'swisspass-small', + 'tick-small', + 'ticket-route-medium', + 'tickets-class-small', + 'train-medium', + 'train-small', + 'trash-small', + 'travel-backpack-medium', + 'user-small', + 'utilization-high', + 'utilization-low', + 'utilization-medium', + 'utilization-none', + 'walk-fast-small', + 'walk-slow-small', + 'walk-small', + ]; + await Promise.all(preloadedIcons.map((icon) => getSvgContent('default', icon, true))); + + mergeConfig({ + icon: { + interceptor({ namespace, name }) { + throw new Error(`Icon ${namespace}:${name} must be preloaded in test-setup.ts!`); + }, + }, + }); +} else { + // Setup mock configuration for icons const testNamespaces = ['default', 'picto']; const icon: SbbIconConfig = { interceptor: ({ namespace, name, request }) => { @@ -27,8 +133,6 @@ function setupIconConfig(): void { mergeConfig({ icon }); } -setupIconConfig(); - if (isHydratedSsr()) { await import('@lit-labs/ssr-client/lit-element-hydrate-support.js'); } diff --git a/src/visual-regression-app/index.html b/src/visual-regression-app/index.html new file mode 100644 index 0000000000..2a80312eb1 --- /dev/null +++ b/src/visual-regression-app/index.html @@ -0,0 +1,32 @@ + + + + Visual Regression Tests Comparison + + + + + + + + + + diff --git a/src/visual-regression-app/src/components/overview/overview.scss b/src/visual-regression-app/src/components/overview/overview.scss new file mode 100644 index 0000000000..269156620d --- /dev/null +++ b/src/visual-regression-app/src/components/overview/overview.scss @@ -0,0 +1,9 @@ +@use '../../../../components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +.app-overview { + display: flex; + gap: 1rem; + flex-direction: column; +} diff --git a/src/visual-regression-app/src/components/overview/overview.ts b/src/visual-regression-app/src/components/overview/overview.ts new file mode 100644 index 0000000000..5e84d1e028 --- /dev/null +++ b/src/visual-regression-app/src/components/overview/overview.ts @@ -0,0 +1,70 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement } from 'lit/decorators.js'; + +import { screenshots } from '../../screenshots.js'; + +import style from './overview.scss?lit&inline'; + +import '../../../../components/accordion.js'; +import '../../../../components/button/secondary-button-link.js'; +import '../../../../components/card.js'; +import '../../../../components/container.js'; +import '../../../../components/expansion-panel.js'; +import '../../../../components/link-list.js'; +import '../../../../components/link/block-link.js'; +import '../../../../components/title.js'; + +/** + * Overview over all failed or new tests + */ +@customElement('app-overview') +export class Overview extends LitElement { + public static override styles: CSSResultGroup = style; + + public override render(): TemplateResult { + return html` + + Lyne visual regression comparison +
+ + ${screenshots.stats} + + Start comparing + + + + ${screenshots.components.map( + (screenshotComponent) => html` + + + ${screenshotComponent.name} (${screenshotComponent.stats}) + + + + ${screenshotComponent.testCases.map( + (entry) => + html` + ${entry.name} + `, + )} + + + + `, + )} + +
+
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-overview': Overview; + } +} diff --git a/src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss b/src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss new file mode 100644 index 0000000000..be69229048 --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss @@ -0,0 +1,21 @@ +@use '../../../../../../components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +:host { + display: block; +} + +.app-labels { + text-transform: capitalize; +} + +.app-radio-button-group { + margin-block-end: 0.5rem; +} + +.app-scroll-container { + @include sbb.scrollbar; + + overflow: auto; +} diff --git a/src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts b/src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts new file mode 100644 index 0000000000..701e033a80 --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts @@ -0,0 +1,58 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup, nothing } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; + +import type { SbbRadioButtonGroupElement } from '../../../../../../components/radio-button/radio-button-group/radio-button-group.js'; +import type { FailedFiles } from '../../../../interfaces.js'; + +import style from './fullscreen-diff.scss?lit&inline'; + +import '../../../../../../components/chip.js'; +import '../../../../../../components/radio-button.js'; + +/** + * Displays two images in fullscreen to overlay them. + */ +@customElement('app-fullscreen-diff') +export class FullscreenDiff extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public failedFile?: FailedFiles; + + @property() public selectedFile: 'baselineFile' | 'failedFile' | 'diffFile' = 'failedFile'; + + public override render(): TemplateResult { + if (!this.failedFile) { + return html``; + } + return html`
+ ${this.failedFile.browserName} + ${this.failedFile.viewport} +
+ + (this.selectedFile = (event.target as SbbRadioButtonGroupElement).value)} + > + ${!this.failedFile.isNew + ? html`Baseline` + : nothing} + ${this.failedFile.isNew ? 'New' : 'Failed'} + ${this.failedFile.diffFile + ? html`Diff` + : nothing} + +
+ +
`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-fullscreen-diff': FullscreenDiff; + } +} diff --git a/src/visual-regression-app/src/components/test-case/image-diff/image-diff.scss b/src/visual-regression-app/src/components/test-case/image-diff/image-diff.scss new file mode 100644 index 0000000000..51b90d6dbd --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/image-diff/image-diff.scss @@ -0,0 +1,73 @@ +@use '../../../../../components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +:host { + display: block; +} + +.app-container { + display: flex; + flex-direction: column; + gap: var(--sbb-spacing-fixed-1x); +} + +.app-info-bar { + display: flex; + justify-content: space-between; +} + +.app-labels { + text-transform: capitalize; +} + +.app-diff-toggle { + white-space: nowrap; + align-self: center; +} + +.app-image-container { + width: 100%; + display: flex; + align-items: start; + justify-content: stretch; + flex-direction: column; + gap: var(--sbb-spacing-fixed-1x); + + @include sbb.mq($from: small) { + flex-direction: row; + } +} + +.app-image-baseline, +.app-image-failed { + display: flex; + flex: 1; + width: 100%; + background-color: var(--sbb-color-white); +} + +.app-image { + max-width: 100%; +} + +.app-new-test-case-info { + margin: 0.5rem; +} + +.app-image-button { + @include sbb.button-reset; + + display: flex; + width: 100%; + text-align: left; + cursor: pointer; + + &:focus-visible { + @include sbb.focus-outline; + } + + &[hidden] { + display: none; + } +} diff --git a/src/visual-regression-app/src/components/test-case/image-diff/image-diff.ts b/src/visual-regression-app/src/components/test-case/image-diff/image-diff.ts new file mode 100644 index 0000000000..b126bae8e3 --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/image-diff/image-diff.ts @@ -0,0 +1,144 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup, nothing } from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; + +import { SbbOverlayElement } from '../../../../../components/overlay/overlay.js'; +import type { SbbToggleCheckElement } from '../../../../../components/toggle-check/toggle-check.js'; +import type { FailedFiles } from '../../../interfaces.js'; + +import style from './image-diff.scss?lit&inline'; + +import '../../../../../components/chip.js'; +import '../../../../../components/status.js'; +import '../../../../../components/overlay.js'; +import '../../../../../components/toggle-check.js'; + +import './fullscreen-diff/fullscreen-diff.js'; + +const getImageDimension = (img: HTMLImageElement): string => + `${img.naturalWidth}x${img.naturalHeight}px`; + +/** + * Displays two images to compare them. + */ +@customElement('app-image-diff') +export class ImageDiff extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public failedFile?: FailedFiles; + + @state() private _baselineDimension?: string; + @state() private _failedDimension?: string; + + @state() private _showDiff: boolean = true; + + private _toggleDiff(event: Event): void { + this._showDiff = (event.target as SbbToggleCheckElement).checked; + } + + private _setFailedImageDimension(event: Event): void { + this._failedDimension = getImageDimension(event.target as HTMLImageElement); + } + + private _setBaselineImageDimension(event: Event): void { + this._baselineDimension = getImageDimension(event.target as HTMLImageElement); + } + + /** + * To avoid blown up DOM, we create the overlay only when it's needed. + */ + private _showFullscreen(selectedFile: 'baselineFile' | 'failedFile' | 'diffFile'): void { + const sbbOverlayElement: SbbOverlayElement = document.createElement('sbb-overlay'); + const appFullscreenDiff = document.createElement('app-fullscreen-diff'); + + sbbOverlayElement.expanded = true; + appFullscreenDiff.selectedFile = selectedFile; + appFullscreenDiff.failedFile = this.failedFile; + + sbbOverlayElement.appendChild(appFullscreenDiff); + document.body.appendChild(sbbOverlayElement); + sbbOverlayElement.addEventListener(SbbOverlayElement.events.didClose, () => { + document.body.removeChild(sbbOverlayElement); + }); + + sbbOverlayElement.open(); + } + + public override render(): TemplateResult { + if (!this.failedFile) { + return html``; + } + return html`
+
+
+ ${this.failedFile.browserName} + ${this.failedFile.viewport} + ${this._baselineDimension + ? html` + Baseline: ${this._baselineDimension} + ` + : nothing} + + ${this.failedFile.isNew ? 'New' : 'Failed'}: ${this._failedDimension} + +
+ ${!this.failedFile.isNew + ? html` + Show Diff + ` + : nothing} +
+
+
+ ${!this.failedFile.isNew + ? html`` + : html` + New test case + `} +
+
+ + +
+
+
`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-image-diff': ImageDiff; + } +} diff --git a/src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.scss b/src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.scss new file mode 100644 index 0000000000..f1be8fcf83 --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.scss @@ -0,0 +1,14 @@ +@use '../../../../../components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +.app-test-case-filter { + display: flex; + gap: 0.5rem 4rem; + flex-direction: column; + text-transform: capitalize; +} + +sbb-title { + margin: 0; +} diff --git a/src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.ts b/src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.ts new file mode 100644 index 0000000000..385157aa62 --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/test-case-filter/test-case-filter.ts @@ -0,0 +1,93 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; + +import type { SbbTagElement } from '../../../../../components/tag/tag/tag.js'; +import { type ScreenshotTestCase } from '../../../screenshots.js'; +import '../../../../../components/title.js'; +import '../../../../../components/tag.js'; + +import style from './test-case-filter.scss?lit&inline'; + +/** + * Shows filter for viewports and browsers + */ +@customElement('app-test-case-filter') +export class TestCaseFilter extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public testCase?: ScreenshotTestCase; + + /** + * Activate `all`-tag of viewports and browsers. + */ + public reset(): void { + this.shadowRoot!.querySelectorAll(`sbb-tag[value='all']`).forEach( + (tag) => (tag.checked = true), + ); + } + + private _handleViewportChange(event: CustomEvent): void { + this.dispatchEvent( + new CustomEvent('viewportFilterChange', { + bubbles: true, + composed: true, + detail: (event.target as SbbTagElement).value, + }), + ); + } + + private _handleBrowserChange(event: CustomEvent): void { + this.dispatchEvent( + new CustomEvent('browserFilterChange', { + bubbles: true, + composed: true, + detail: (event.target as SbbTagElement).value, + }), + ); + } + + public override render(): TemplateResult { + return html` +
+
+ Viewports + + + All + + ${this.testCase?.viewports?.map( + (viewport) => html` + + ${viewport.name} + + `, + )} + +
+
+ Browsers + + + All + + ${this.testCase?.availableBrowserNames?.map( + (browserName) => html`${browserName}`, + )} + +
+
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-test-case-filter': TestCaseFilter; + } +} diff --git a/src/visual-regression-app/src/components/test-case/test-case.scss b/src/visual-regression-app/src/components/test-case/test-case.scss new file mode 100644 index 0000000000..04b5c1e68e --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/test-case.scss @@ -0,0 +1,49 @@ +@use '../../../../components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +sbb-chip { + width: fit-content; +} + +sbb-title { + margin: 0; +} + +.app-file-name-box { + display: flex; + flex-direction: column; +} + +.app-file-name-ellipsis { + display: block; + @include sbb.ellipsis; +} + +.app-navigation-block { + display: flex; + gap: 0.5rem; + align-items: center; +} + +.app-image-diffs { + display: flex; + flex-direction: column; + gap: 2rem; + padding-block: 1rem; +} + +.app-testcase { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.app-progress { + position: absolute; + inset-block-start: 0; + inset-inline-start: 0; + height: var(--sbb-border-radius-2x); + background-color: var(--sbb-color-black); + width: calc(var(--app-progress) * 100%); +} diff --git a/src/visual-regression-app/src/components/test-case/test-case.ts b/src/visual-regression-app/src/components/test-case/test-case.ts new file mode 100644 index 0000000000..626afb0a8e --- /dev/null +++ b/src/visual-regression-app/src/components/test-case/test-case.ts @@ -0,0 +1,150 @@ +import { + type CSSResultGroup, + html, + LitElement, + type PropertyValues, + type TemplateResult, +} from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; + +import { screenshots, type ScreenshotTestCase } from '../../screenshots.js'; + +import '../../../../components/button/secondary-button-link.js'; +import '../../../../components/chip.js'; +import '../../../../components/container.js'; +import '../../../../components/header.js'; +import '../../../../components/notification.js'; +import '../../../../components/title.js'; + +import type { TestCaseFilter } from './test-case-filter/test-case-filter.js'; +import style from './test-case.scss?lit&inline'; + +import './test-case-filter/test-case-filter.js'; +import './image-diff/image-diff.js'; + +interface Filter { + viewport?: string; + browser?: string; +} + +/** + * Displays a test case with its images. + * Provides filtering functions. + */ +@customElement('app-test-case') +export class TestCase extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public params?: { componentName: string; testCaseName: string }; + + @state() private _testCase?: ScreenshotTestCase; + @state() private _testCaseIndex: number = -1; + @state() private _filter: Filter = {}; + + protected override willUpdate(changedProperties: PropertyValues): void { + super.willUpdate(changedProperties); + + if (changedProperties.has('params')) { + // Reset + this._filter = {}; + this.shadowRoot!.querySelector('app-test-case-filter')?.reset(); + + // Get test case + this._testCaseIndex = screenshots.indexOfTestCase( + this.params!.componentName!, + this.params!.testCaseName!, + ); + if (this._testCaseIndex >= 0) { + this._testCase = screenshots.getByTestCaseIndex(this._testCaseIndex); + } + } + } + + private _progressFraction(): number { + return (this._testCaseIndex + 1) / screenshots.testCaseCount; + } + + private _next(): ScreenshotTestCase | undefined { + return screenshots.getByTestCaseIndex(this._testCaseIndex + 1); + } + + private _previous(): ScreenshotTestCase | undefined { + return screenshots.getByTestCaseIndex(this._testCaseIndex - 1); + } + + private _viewportFilterChanged(event: CustomEvent): void { + this._filter = { + ...this._filter, + viewport: event.detail && event.detail !== 'all' ? event.detail : undefined, + }; + } + + private _browserFilterChanged(event: CustomEvent): void { + this._filter = { + ...this._filter, + browser: event.detail && event.detail !== 'all' ? event.detail : undefined, + }; + } + + public override render(): TemplateResult { + return html` + +
+
+ ${this.params?.componentName} + + ${this.params?.testCaseName} + +
+
+
+ Overview + + +
+
+ ${this._testCase + ? html`
+ + + + +
+ ${this._testCase + ?.filter(this._filter.viewport, this._filter.browser) + .map( + (failedFile) => + html``, + )} +
+
+
` + : html` + + No screenshots found. Please check component and test case name. + + `} + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-test-case': TestCase; + } +} diff --git a/src/visual-regression-app/src/interfaces.ts b/src/visual-regression-app/src/interfaces.ts new file mode 100644 index 0000000000..72e5316788 --- /dev/null +++ b/src/visual-regression-app/src/interfaces.ts @@ -0,0 +1,11 @@ +export interface FailedFiles { + browserName: string; + name: string; + failedFile: string; + diffFile: string; + baselineFile: string; + isNew: boolean; + viewport: string; +} + +export type ScreenshotMap = Record>>; diff --git a/src/visual-regression-app/src/main.ts b/src/visual-regression-app/src/main.ts new file mode 100644 index 0000000000..6d04c5a54e --- /dev/null +++ b/src/visual-regression-app/src/main.ts @@ -0,0 +1,44 @@ +import { Router } from '@lit-labs/router'; +import { html, LitElement, type TemplateResult } from 'lit'; +import { customElement } from 'lit/decorators.js'; + +import '../../components/core/styles/standard-theme.scss'; + +/** + * Main app containing the router outlet. + */ +@customElement('app-main') +export class Main extends LitElement { + private _router = new Router(this, [ + { + path: '/', + render: () => html``, + enter: async () => { + await import('./components/overview/overview.js'); + return true; + }, + }, + { + path: '/compare/:component/:testcase', + render: ({ component, testcase }) => + html``, + enter: async () => { + await import('./components/test-case/test-case.js'); + return true; + }, + }, + ]); + + public override render(): TemplateResult { + return html`${this._router.outlet()}`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-main': Main; + } +} diff --git a/src/visual-regression-app/src/screenshots.ts b/src/visual-regression-app/src/screenshots.ts new file mode 100644 index 0000000000..c7ee30515a --- /dev/null +++ b/src/visual-regression-app/src/screenshots.ts @@ -0,0 +1,142 @@ +// eslint-disable-next-line import-x/no-unresolved +import { screenshotsRaw } from 'virtual:screenshots'; + +import type { FailedFiles, ScreenshotMap } from './interfaces.js'; + +const viewportOrder = ['zero', 'micro', 'small', 'medium', 'large', 'wide', 'ultra']; + +export class ScreenshotStatistics { + public static fromFailedFiles(failedFiles: FailedFiles[]): ScreenshotStatistics { + return failedFiles.reduce( + (current, next) => + current.sum(new ScreenshotStatistics(next.isNew ? 0 : 1, next.isNew ? 1 : 0)), + new ScreenshotStatistics(0, 0), + ); + } + + public static fromList(list: { stats: ScreenshotStatistics }[]): ScreenshotStatistics { + return list.reduce((current, next) => current.sum(next.stats), new ScreenshotStatistics(0, 0)); + } + + public constructor( + public readonly failedTests: number, + public readonly newTests: number, + ) {} + + public sum(other: ScreenshotStatistics): ScreenshotStatistics { + return new ScreenshotStatistics( + this.failedTests + other.failedTests, + this.newTests + other.newTests, + ); + } + + public toString(): string { + return `${this.failedTests} failed, ${this.newTests} new`; + } +} + +export class ScreenshotViewport { + public readonly stats: ScreenshotStatistics; + public readonly browserNames: string[]; + + public constructor( + public readonly name: string, + public readonly browsers: FailedFiles[], + ) { + this.stats = ScreenshotStatistics.fromFailedFiles(this.browsers); + + this.browserNames = this.browsers.map((browser) => browser.browserName); + } + + /** Compare by respecting defined viewport order. */ + public compare(other: ScreenshotViewport): number { + return viewportOrder.indexOf(this.name) - viewportOrder.indexOf(other.name); + } +} + +export class ScreenshotTestCase { + public readonly stats: ScreenshotStatistics; + public readonly availableBrowserNames: string[]; + public readonly path: string; + + public constructor( + public readonly component: string, + public readonly name: string, + public readonly viewports: ScreenshotViewport[], + ) { + this.stats = ScreenshotStatistics.fromList(this.viewports); + this.path = `${this.component}/${this.name}`; + + this.availableBrowserNames = Array.from( + this.viewports.reduce((current, next) => { + next.browserNames.forEach((browserName) => current.add(browserName)); + return current; + }, new Set()), + ); + } + + public filter(viewport?: string, browser?: string): FailedFiles[] { + return this.viewports + .filter((entry) => !viewport || entry.name === viewport) + .flatMap((entry) => + entry.browsers.filter((failedFiles) => !browser || failedFiles.browserName === browser), + ); + } +} + +export class ScreenshotComponent { + public readonly stats: ScreenshotStatistics; + + public constructor( + public readonly name: string, + public readonly testCases: ScreenshotTestCase[], + ) { + this.stats = ScreenshotStatistics.fromList(this.testCases); + } +} + +export class Screenshots { + public readonly components: ScreenshotComponent[]; + public readonly stats: ScreenshotStatistics; + public readonly testCaseCount: number; + public readonly flatTestCases: ScreenshotTestCase[]; + + public constructor(screenshotsRaw: ScreenshotMap) { + const flatTestCases: ScreenshotTestCase[] = []; + + // Convert hierarchical screenshot map to classes + this.components = Object.entries(screenshotsRaw).map( + ([componentName, testCases]) => + new ScreenshotComponent( + componentName, + Object.entries(testCases).map(([testCase, viewports]) => { + const screenshotTestCase = new ScreenshotTestCase( + componentName, + testCase, + Object.entries(viewports) + .map(([viewport, entries]) => new ScreenshotViewport(viewport, entries)) + .sort((a: ScreenshotViewport, b: ScreenshotViewport) => a.compare(b)), + ); + flatTestCases.push(screenshotTestCase); + return screenshotTestCase; + }), + ), + ); + + this.flatTestCases = flatTestCases; + this.testCaseCount = this.flatTestCases.length; + this.stats = ScreenshotStatistics.fromList(this.components); + } + + public indexOfTestCase(componentName: string, testCaseName: string): number { + return this.flatTestCases.findIndex( + (component) => component.component === componentName && component.name === testCaseName, + ); + } + + public getByTestCaseIndex(index: number): ScreenshotTestCase | undefined { + return this.flatTestCases[index]; + } +} + +export const screenshots = new Screenshots(screenshotsRaw); diff --git a/src/visual-regression-app/tsconfig.json b/src/visual-regression-app/tsconfig.json new file mode 100644 index 0000000000..d5efd68ad7 --- /dev/null +++ b/src/visual-regression-app/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "../..", + "baseUrl": ".", + "paths": { + "@sbb-esta/lyne-components": ["../components"], + "@sbb-esta/lyne-components/*": ["../components/*"] + } + }, + "include": ["./**/*.ts", "../vite-env.d.ts"], + "exclude": ["vite.config.ts"] +} diff --git a/src/visual-regression-app/vite.config.ts b/src/visual-regression-app/vite.config.ts new file mode 100644 index 0000000000..f7c234d151 --- /dev/null +++ b/src/visual-regression-app/vite.config.ts @@ -0,0 +1,199 @@ +import { createHash } from 'crypto'; +import { existsSync, readdirSync, readFileSync } from 'fs'; +import { relative } from 'path'; + +import { + defineConfig, + mergeConfig, + type PluginOption, + type ResolvedConfig, + type UserConfig, +} from 'vite'; + +import { distDir } from '../../tools/vite/index.js'; +import rootConfig from '../../vite.config.js'; + +import type { FailedFiles } from './src/interfaces.js'; + +const packageRoot = new URL('.', import.meta.url); +const screenshotsDir = new URL(`./screenshots/`, distDir); + +const extractHierarchicalMap = ( + screenshots: Map[]>, +): Map>> => { + const map = new Map>>(); + + screenshots.forEach((failedFiles, fileName) => { + const component = fileName.match(/^(.*?)_/)![1]; + const name = fileName.match(/_viewport=.*?_(.*?).png$/)![1]; + const viewport = fileName.match(/viewport=(.*?)_/)![1]; + + if (!map.has(component)) { + map.set(component, new Map()); + } + + const componentsMap = map.get(component)!; + + if (!componentsMap.has(name)) { + componentsMap.set(name, new Map()); + } + + const testCaseMap = componentsMap.get(name)!; + + testCaseMap.set( + viewport, + failedFiles.map((failedFile) => ({ ...failedFile, viewport }) satisfies FailedFiles), + ); + }); + + return map; +}; + +function prepareScreenshots(): PluginOption { + let viteConfig: ResolvedConfig; + const virtualModuleId = 'virtual:screenshots'; + const resolvedVirtualModuleId = '\0' + virtualModuleId; + + return { + name: 'prepare screenshot', + configResolved(config) { + viteConfig = config; + }, + resolveId(id) { + if (id === virtualModuleId) { + return resolvedVirtualModuleId; + } + }, + load(id) { + if (id === resolvedVirtualModuleId) { + const failedScreenshotsHash = createHash('sha256'); + const screenshotsFailures = existsSync(screenshotsDir) + ? readdirSync(screenshotsDir, { withFileTypes: true }) + .map((d) => d.name) + .filter((browserName) => + existsSync(new URL(`./${browserName}/failed/`, screenshotsDir)), + ) + .flatMap((browserName) => { + const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); + + return readdirSync(failedDir, { + withFileTypes: true, + }) + .filter((d) => !d.name.endsWith('-diff.png')) + .map((d) => { + const failedFilePath = new URL(`./${d.name}`, failedDir); + const diffFilePath = new URL( + `./${d.name.replace(/.png$/, '-diff.png')}`, + failedDir, + ); + const baselineFilePath = new URL( + `./${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + + const isNew = !existsSync(diffFilePath); + + const assetsScreenshots = 'assets/screenshots/'; + const failedRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, failedFilePath.pathname); + const diffRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); + const baselineRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, baselineFilePath.pathname); + + if (viteConfig.command !== 'serve') { + const failedFileContent = readFileSync(failedFilePath); + // We only add the failed screenshot hashes, as the baseline and comparison (*-diff.png) + // are not relevant to detect whether it is a new difference. + failedScreenshotsHash.update(failedFileContent); + this.emitFile({ + type: 'asset', + fileName: failedRelativeFileName, + source: failedFileContent, + }); + + if (!isNew) { + this.emitFile({ + type: 'asset', + fileName: diffRelativeFileName, + source: readFileSync(diffFilePath), + }); + + this.emitFile({ + type: 'asset', + fileName: baselineRelativeFileName, + source: readFileSync(baselineFilePath), + }); + } + } + + return { + browserName, + name: d.name, + failedFile: failedRelativeFileName, + diffFile: diffRelativeFileName, + baselineFile: baselineRelativeFileName, + isNew, + }; + }); + }) + : []; + + const screenshotsMeta = screenshotsFailures.reduce( + (current, next) => + current.set( + next.name, + current.has(next.name) ? current.get(next.name)!.concat(next) : [next], + ), + new Map(), + ); + + if (viteConfig.command !== 'serve') { + this.emitFile({ + type: 'asset', + fileName: 'diff.json', + source: JSON.stringify({ + changedAmount: screenshotsFailures.filter((f) => !f.isNew).length, + newAmount: screenshotsFailures.filter((f) => f.isNew).length, + hash: failedScreenshotsHash.digest('hex'), + }), + }); + } + + return `export const screenshotsRaw = ${JSON.stringify( + extractHierarchicalMap(screenshotsMeta), + (_key, value) => { + if (value instanceof Map) { + return Object.fromEntries(Array.from(value)); + } else { + return value; + } + }, + )}`; + } + }, + configureServer(server) { + server.middlewares.use((req, res, next) => { + if (req.url?.startsWith('/assets/screenshots/')) { + console.log(req.url); + res.end(readFileSync(new URL(`.${req.url.substring(7)}`, distDir))); + } else { + next(); + } + }); + }, + }; +} + +export default defineConfig(() => + mergeConfig(rootConfig, { + root: packageRoot.pathname, + plugins: [prepareScreenshots()], + build: { + outDir: new URL(`./visual-regression-app/`, distDir).pathname, + emptyOutDir: true, + }, + }), +); diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 9848f7e607..49888528ee 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -9,3 +9,7 @@ declare module '*?lit&inline' { declare module '@custom-elements-manifest/analyzer/cli' { export const cli: (...args) => Promise; } + +declare module 'virtual:screenshots' { + export const screenshotsRaw: import('./visual-regression-app/src/interfaces').ScreenshotMap; +} diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile new file mode 100644 index 0000000000..4d49ebbcc7 --- /dev/null +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -0,0 +1,13 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# Copy screenshots +COPY ./dist/screenshots/Chromium/baseline/ /usr/share/nginx/html/Chromium/baseline/ +COPY ./dist/screenshots/Firefox/baseline/ /usr/share/nginx/html/Firefox/baseline/ +COPY ./dist/screenshots/Webkit/baseline/ /usr/share/nginx/html/Webkit/baseline/ + +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh + +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/baseline.nginx.conf b/tools/visual-regression-testing/baseline.nginx.conf new file mode 100644 index 0000000000..a66a48b9ec --- /dev/null +++ b/tools/visual-regression-testing/baseline.nginx.conf @@ -0,0 +1,50 @@ +tcp_nopush on; +tcp_nodelay on; +types_hash_max_size 2048; + +server { + listen 8080 default_server; + server_name _; + root /usr/share/nginx/html; + index index.html index.htm; + + # Suppresses the nginx version in the Server header. + server_tokens off; + + location / { + expires -1; + add_header Pragma "no-cache"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + try_files $uri $uri/ /index.html =404; + } + + location ~* \.(?:png)$ { + expires 1y; + access_log off; + # We are using a custom etag logic, which uses the sha1 hash of the image as etag. + # The default etag from nginx uses file modified time and file size, which is not good + # enough for our purposes. + etag off; + add_header Cache-Control "public"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header Etag $pngetag; + + set $request_etag $http_if_none_match; + if ($request_etag = false) { + set $request_etag "-"; + } + if ($request_etag = $pngetag) { + return 304 ""; + } + } + + location ~* \.(?:css|js)$ { + expires 1y; + access_log off; + add_header Cache-Control "public"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + } +} \ No newline at end of file diff --git a/tools/visual-regression-testing/diff-app.Dockerfile b/tools/visual-regression-testing/diff-app.Dockerfile new file mode 100644 index 0000000000..f60fce6f08 --- /dev/null +++ b/tools/visual-regression-testing/diff-app.Dockerfile @@ -0,0 +1,11 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# This is currently the same config file as for baseline. Separate into separate configs, if this changes. +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY ./dist/visual-regression-app /usr/share/nginx/html + +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh + +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/etag-map-generation.sh b/tools/visual-regression-testing/etag-map-generation.sh new file mode 100755 index 0000000000..5f88502c31 --- /dev/null +++ b/tools/visual-regression-testing/etag-map-generation.sh @@ -0,0 +1,10 @@ + +# We calculate the sha1 hashes of the png files in order to use it as etag values. +# This allows us to use HTTP caching mechanisms, which should reduce network traffic +# for the baseline comparison. +cd /usr/share/nginx/html + +echo 'map_hash_bucket_size 32768;' +echo 'map $uri $pngetag {' +find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha256sum $1 | cut -d " " -f 1);"' _ {} \; +echo '}' \ No newline at end of file diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts new file mode 100644 index 0000000000..13ebd9665b --- /dev/null +++ b/tools/visual-regression-testing/exec.ts @@ -0,0 +1,104 @@ +// This script checks which OS the visual regression testing is run on +// and if it is not Linux, runs it in a container. + +import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; +import { cpSync, existsSync, mkdirSync } from 'fs'; +import { platform } from 'os'; + +import { startTestRunner } from '@web/test-runner'; +import * as glob from 'glob'; + +if (process.env.GITHUB_ACTIONS) { + // When being run on GitHub Actions we have two use cases. + // Baseline generation for which our expectation is to not fail. + // Diff generation if any test fails, for which we copy only the necessary + // files to dist/screenshots-artifact/ to reduce artifact size. + const runner = await startTestRunner({ autoExitProcess: false }); + if (!runner) { + throw new Error( + `Unexpected state. Test runner not available. Check tools/visual-regression-testing/exec.ts execution.`, + ); + } + await new Promise((r) => runner.on('stopped', r)); + + const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); + const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); + mkdirSync(artifactDir, { recursive: true }); + + if (runner.passed) { + // Tests passed. Do nothing. + process.exit(0); + } + + // When visual regression tests have failed, we only want to pack the relevant screenshots + // into the artifact transfered to the secure workflow, as uploading and downloading the full + // baseline would take far longer. + // Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will + // be moved to /dist/screenshots in the secure workflow. + const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); + for (const failedDir of failedDirs) { + cpSync(new URL(`./${failedDir}`, screenshotDir), new URL(`./${failedDir}`, artifactDir), { + force: true, + recursive: true, + }); + } + + const failedFiles = glob + .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) + .map((p) => p.replace('/failed/', '/baseline/')); + for (const failedFile of failedFiles) { + const baselineFile = new URL(`./${failedFile}`, screenshotDir); + if (existsSync(baselineFile)) { + cpSync(baselineFile, new URL(`./${failedFile}`, artifactDir), { + force: true, + recursive: true, + }); + } + } +} else if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { + await startTestRunner(); +} else { + function executableIsAvailable(name: string): string | null { + try { + execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); + return name; + } catch (error) { + return null; + } + } + + const containerCmd = executableIsAvailable('podman') ?? executableIsAvailable('docker'); + if (!containerCmd) { + console.log('Either docker or podman need to be installed!'); + process.exit(1); + } + + const args = process.argv.slice(2); + const cwd = new URL('../../', import.meta.url); + const tag = 'lyne-vrt'; + const execOptions: ExecSyncOptionsWithStringEncoding = { + encoding: 'utf8', + stdio: 'inherit', + cwd, + }; + const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); + execSync( + `${containerCmd} build ` + + '--file=tools/visual-regression-testing/testing.Dockerfile ' + + `--tag=${tag} .`, + execOptions, + ); + console.log(`\nTest image ready\n`); + mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); + execSync( + `${containerCmd} run -it --rm --ipc=host ` + + `--env=BRANCH="${branch}"` + + `--volume=./dist/screenshots:/dist/screenshots ` + + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + + tag, + execOptions, + ); +} diff --git a/tools/visual-regression-testing/testing.Dockerfile b/tools/visual-regression-testing/testing.Dockerfile new file mode 100644 index 0000000000..1afbaed5c7 --- /dev/null +++ b/tools/visual-regression-testing/testing.Dockerfile @@ -0,0 +1,7 @@ +FROM mcr.microsoft.com/playwright:v1.43.0-jammy + +COPY package.json ./ +COPY yarn.lock ./ +RUN yarn install --frozen-lockfile --non-interactive + +COPY . . diff --git a/tools/visual-regression-testing/testing.Dockerfile.dockerignore b/tools/visual-regression-testing/testing.Dockerfile.dockerignore new file mode 100644 index 0000000000..5de02b1469 --- /dev/null +++ b/tools/visual-regression-testing/testing.Dockerfile.dockerignore @@ -0,0 +1,3 @@ +coverage +dist +node_modules \ No newline at end of file diff --git a/tools/web-test-runner/index.js b/tools/web-test-runner/index.js index 0eb5bcd92d..d4f095b400 100644 --- a/tools/web-test-runner/index.js +++ b/tools/web-test-runner/index.js @@ -2,4 +2,5 @@ export * from './minimal-reporter.js'; export * from './patched-summary-reporter.js'; export * from './ssr-plugin.js'; +export * from './visual-regression-plugin-config.js'; export * from './vite-plugin.js'; diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js new file mode 100644 index 0000000000..e127d69852 --- /dev/null +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -0,0 +1,63 @@ +import { execSync } from 'child_process'; +import { createHash } from 'crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, extname } from 'path'; + +const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); +const baselineUrl = process.env.GITHUB_ACTIONS + ? 'http://localhost:8050/' + : 'https://lyne-visual-regression-baseline.app.sbb.ch/'; + +export const visualRegressionConfig = (update) => + /** @type {Parameters[0]} */ + ({ + update, + baseDir: 'dist/screenshots', + async getBaseline({ filePath, name }) { + const baselineFileUrl = baselineUrl + name + extname(filePath); + const infoFilePath = filePath + '.json'; + const info = existsSync(infoFilePath) ? JSON.parse(readFileSync(infoFilePath, 'utf8')) : {}; + if (existsSync(filePath) && info.branch === branch) { + return readFileSync(filePath); + } else if (existsSync(filePath) && info.etag) { + const response = await fetch(baselineFileUrl, { + method: 'HEAD', + headers: { 'if-none-match': info.etag }, + }); + + if (response.status === 304) { + return readFileSync(filePath); + } else if (response.status === 404) { + return undefined; + } + } + + // If the image address is not reachable, fetch throws, so we wrap it in try/catch. + try { + const response = await fetch(baselineFileUrl); + if (response.ok) { + const etag = response.headers.get('etag'); + const buffer = Buffer.from(await response.arrayBuffer()); + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, buffer); + writeFileSync(infoFilePath, JSON.stringify({ etag }, null, 2), 'utf8'); + + return buffer; + } + } catch (e) { + return undefined; + } + }, + saveBaseline({ filePath, content }) { + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, content); + writeFileSync( + filePath + '.json', + JSON.stringify({ branch, etag: createHash('sha256').update(content).digest('hex') }), + 'utf8', + ); + }, + }); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 5c3b7c9277..c370f9c77e 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -2,6 +2,7 @@ import { defaultReporter } from '@web/test-runner'; import { playwrightLauncher } from '@web/test-runner-playwright'; import { puppeteerLauncher } from '@web/test-runner-puppeteer'; import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins'; +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; import * as sass from 'sass'; import { cpus } from 'node:os'; @@ -9,14 +10,23 @@ import { minimalReporter, patchedSummaryReporter, ssrPlugin, + visualRegressionConfig, vitePlugin, } from './tools/web-test-runner/index.js'; const isCIEnvironment = !!process.env.CI || process.argv.includes('--ci'); const isDebugMode = process.argv.includes('--debug'); +const allBrowsers = process.argv.includes('--all-browsers'); const firefox = process.argv.includes('--firefox'); const webkit = process.argv.includes('--webkit'); const concurrency = process.argv.includes('--parallel') ? {} : { concurrency: 1 }; +const updateBaseImages = + process.argv.includes('--update-visual-baseline') || process.argv.includes('--uv'); +const visualRegressionRun = + process.argv.includes('--group=visual-regression') || + (process.argv.includes('--group') + ? process.argv[process.argv.indexOf('--group') + 1] === 'visual-regression' + : false); const stylesCompiler = new sass.initCompiler(); const renderStyles = () => @@ -24,25 +34,26 @@ const renderStyles = () => loadPaths: ['.', './node_modules/'], }).css; -const browsers = isCIEnvironment - ? [ - // Parallelism has problems, we need force concurrency to 1 - playwrightLauncher({ product: 'chromium', ...concurrency }), - playwrightLauncher({ product: 'firefox', ...concurrency }), - playwrightLauncher({ product: 'webkit', ...concurrency }), - ] - : firefox - ? [playwrightLauncher({ product: 'firefox' })] - : webkit - ? [playwrightLauncher({ product: 'webkit' })] - : isDebugMode - ? [ - puppeteerLauncher({ - launchOptions: { headless: false, devtools: true }, - ...concurrency, - }), - ] - : [playwrightLauncher({ product: 'chromium' })]; +const browsers = + isCIEnvironment || allBrowsers + ? [ + // Parallelism has problems, we need force concurrency to 1 + playwrightLauncher({ product: 'chromium', ...concurrency }), + playwrightLauncher({ product: 'firefox', ...concurrency }), + playwrightLauncher({ product: 'webkit', ...concurrency }), + ] + : firefox + ? [playwrightLauncher({ product: 'firefox' })] + : webkit + ? [playwrightLauncher({ product: 'webkit' })] + : isDebugMode + ? [ + puppeteerLauncher({ + launchOptions: { headless: false, devtools: true }, + ...concurrency, + }), + ] + : [playwrightLauncher({ product: 'chromium' })]; const groupNameOverride = process.argv.includes('--ssr-hydrated') ? 'e2e-ssr-hydrated' @@ -52,8 +63,29 @@ const groupNameOverride = process.argv.includes('--ssr-hydrated') const testRunnerHtml = (testFramework, _config, group) => ` - + + + +