build: implement visual regression testing with @web/test-runner

.github/default.conf

@@ -2,6 +2,9 @@ tcp_nopush          on;
 tcp_nodelay         on;
 types_hash_max_size 2048;
 
+# Suppresses the nginx version in the Server header.
+server_tokens off;
+
 # Determine if it's a valid origin and set it in the $cors variable.
 map "$http_origin" $cors {
   default '';

.github/workflows/container-image-cleanup.yml

@@ -0,0 +1,68 @@
+name: Container Image Cleanup
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: '0 3 * * *'
+
+permissions:
+  packages: write
+
+jobs:
+  container-image-cleanup:
+    runs-on: ubuntu-latest
+    env:
+      CLOSED_PR_RETENTION_DAYS: 5
+      PACKAGE_NAMES: storybook-preview,visual-regression
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const pullRequests = await github.paginate(
+              github.rest.pulls.list.endpoint.merge({ owner, repo, state: 'all' })
+            );
+            const retentionPivot =
+              new Date(Date.now() - (+process.env.CLOSED_PR_RETENTION_DAYS * 24 * 60 * 60 * 1000));
+            const olderThanTwoWeeks = (date) => new Date(date) < retentionPivot;
+            const isExpiredPrTag = (version) => {
+              const prNumber = +version.metadata?.container?.tags
+                ?.find((t) => t.match(/(preview-pr|pr)(\d+)/))?.match(/(preview-pr|pr)(\d+)/)[2];
+              const pr = pullRequests.find((p) => p.number === prNumber);
+              return !!prNumber && pr?.state === 'closed' && olderThanTwoWeeks(pr.closed_at);
+            };
+
+            const packageNames = process.env.PACKAGE_NAME.split(',').map((n) => n.trim());
+            let packageDeletionFailed = false;
+            for (const packageName of packageNames) {
+              const params = {
+                package_type: 'container',
+                package_name: `${repo}/${packageNames}`,
+                username: owner
+              };
+              const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params);
+              for (const version of versions.filter(isExpiredPrTag)) {
+                try {
+                  await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id });
+                  console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`);
+                } catch(e) {
+                  console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`);
+                  console.error(e);
+                  packageDeletionFailed = true;
+                }
+              }
+            }
+
+            if (packageDeletionFailed) {
+              throw new Error('A package deletion failed, please check the log.');
+            }
+      - uses: actions/delete-package-versions@v4
+        with:
+          package-name: lyne-components/storybook-preview
+          package-type: container
+          delete-only-untagged-versions: 'true'
+      - uses: actions/delete-package-versions@v4
+        with:
+          package-name: lyne-components/visual-regression
+          package-type: container
+          delete-only-untagged-versions: 'true'

.github/workflows/continuous-integration-secure.yml

@@ -46,7 +46,7 @@ jobs:
         uses: actions/github-script@v7
         with:
           script: |
-            const environment = process.env.PR_NUMBER ? `preview-pr${process.env.PR_NUMBER}` : 'main';
+            const environment = process.env.PR_NUMBER ? `pr${process.env.PR_NUMBER}` : 'main';
             const payload = { owner: context.repo.owner, repo: context.repo.repo, environment };
             const { data: deployment } = await github.rest.repos.createDeployment({
               ...payload,

.github/workflows/continuous-integration.yml

@@ -7,6 +7,9 @@ concurrency:
 
 permissions: read-all
 
+env:
+  IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -23,6 +26,7 @@ jobs:
 
   integrity:
     runs-on: ubuntu-latest
+    needs: lint
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -38,6 +42,7 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    needs: lint
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -63,6 +68,7 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
+    needs: lint
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -151,3 +157,69 @@ jobs:
           zip: true
           onlyChanged: true
           externals: '**/components/core/styles/**/*.scss'
+
+  visual-regression:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    needs: test
+    services:
+      visual-regression:
+        image: ghcr.io/${{ github.repository }}/visual-regression:baseline
+        ports:
+          - 8050:8080
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: yarn
+      - run: yarn install --frozen-lockfile --non-interactive
+
+      - name: Install browser dependencies
+        run: yarn playwright install-deps
+      - name: Run visual regression baseline generation
+        run: yarn test:visual-regression
+        env:
+          NODE_ENV: production
+      - name: Store visual regression output
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-regression-screenshots
+          path: dist/screenshots-artifact/
+
+  visual-regression-baseline:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: test
+    services:
+      visual-regression:
+        image: ghcr.io/${{ github.repository }}/visual-regression:baseline
+        ports:
+          - 8080:8050
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: yarn
+      - run: yarn install --frozen-lockfile --non-interactive
+
+      - name: Install browser dependencies
+        run: yarn playwright install-deps
+      - name: Run visual regression baseline generation
+        run: yarn test:visual-regression --update-visual-baseline
+        env:
+          NODE_ENV: production
+
+      - name: Build and push visual regression baseline
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+          docker build \
+            --file tools/visual-regression-testing/baseline.Dockerfile \
+            --tag $IMAGE_REPO_VISUAL_REGRESSION:baseline \
+            .
+          docker push $IMAGE_REPO_VISUAL_REGRESSION:baseline
+        env:
+          DOCKER_BUILDKIT: 1

.github/workflows/release-please.yml

@@ -35,7 +35,7 @@ jobs:
     if: needs.release-please.outputs.releases_created
     runs-on: ubuntu-latest
     env:
-      IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook
+      IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook
       VERSION: ${{ needs.release-please.outputs.version }}
     steps:
       - uses: actions/checkout@v4
@@ -66,15 +66,15 @@ jobs:
       - name: Remove files with forbidden extensions
         run: node ./scripts/clean-storybook-files.cjs
       - name: 'Container: Login to GitHub Container Repository'
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
       - name: 'Container: Build image'
-        run: docker build -t $IMAGE_REPO:$VERSION -t $IMAGE_REPO:latest .
+        run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest .
         env:
           DOCKER_BUILDKIT: 1
       - name: 'Container: Publish image'
-        run: docker push $IMAGE_REPO:$VERSION
+        run: docker push $IMAGE_REPO_STORYBOOK:$VERSION
       - name: 'Container: Publish image as latest'
-        run: docker push $IMAGE_REPO:latest
+        run: docker push $IMAGE_REPO_STORYBOOK:latest
 
       - name: Generate chromatic stories
         run: yarn generate:chromatic-stories
@@ -98,7 +98,7 @@ jobs:
     if: needs.release-please.outputs.releases_created != true
     runs-on: ubuntu-latest
     env:
-      IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook
+      IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook
       VERSION: ${{ needs.release-please.outputs.version }}
     steps:
       - uses: actions/checkout@v4
@@ -116,13 +116,13 @@ jobs:
       - name: Remove files with forbidden extensions
         run: node ./scripts/clean-storybook-files.cjs
       - name: 'Container: Login to GitHub Container Repository'
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
       - name: 'Container: Build image'
-        run: docker build -t $IMAGE_REPO:dev .
+        run: docker build --tag $IMAGE_REPO_STORYBOOK:dev .
         env:
           DOCKER_BUILDKIT: 1
       - name: 'Container: Publish image'
-        run: docker push $IMAGE_REPO:dev
+        run: docker push $IMAGE_REPO_STORYBOOK:dev
 
       - name: Generate chromatic stories
         run: yarn generate:chromatic-stories

Dockerfile

@@ -1,4 +1,6 @@
-FROM nginxinc/nginx-unprivileged:stable
+FROM ghcr.io/nginxinc/nginx-unprivileged:stable
+
+LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components
 
 # Copy nginx configuration
 COPY ./.github/default.conf /etc/nginx/conf.d/default.conf

package.json

@@ -28,6 +28,7 @@
     "build:react:production": "vite build --config src/react/vite.config.ts",
     "build:react:development": "NODE_ENV=development vite build --mode development --config src/react/vite.config.ts",
     "build:storybook": "storybook build --quiet --output-dir dist/storybook --stats-json",
+    "build:diff-app": "vite build --config tools/visual-regression-testing/diff-app/vite.config.ts",
     "build": "npm-run-all --sequential build:components build:react build:storybook",
     "docs": "npm-run-all --sequential docs:manifest docs:to-md",
     "docs:manifest": "custom-elements-manifest analyze --config tools/manifest/custom-elements-manifest.config.js",
@@ -48,11 +49,13 @@
     "lint:tsc:components": "tsc --noEmit --project src/components/tsconfig.json",
     "lint:tsc:components-spec": "tsc --noEmit --project src/components/tsconfig.spec.json",
     "start": "storybook dev -p 6006",
+    "start:diff-app": "vite --config tools/visual-regression-testing/diff-app/vite.config.ts",
     "test": "wtr --coverage",
     "test:snapshot": "yarn test:csr --ci --update-snapshots",
     "test:csr": "wtr --group default",
     "test:ssr:hydrated": "wtr --group e2e-ssr-hydrated",
     "test:ssr:non-hydrated": "wtr --group e2e-ssr-non-hydrated",
+    "test:visual-regression": "tsx tools/visual-regression-testing/exec.ts --group=visual-regression --all-browsers",
     "prepare": "husky"
   },
   "dependencies": {
@@ -90,6 +93,7 @@
     "@web/test-runner-commands": "0.9.0",
     "@web/test-runner-playwright": "0.11.0",
     "@web/test-runner-puppeteer": "0.16.0",
+    "@web/test-runner-visual-regression": "0.9.0",
     "chromatic": "11.3.0",
     "custom-elements-manifest": "2.0.0",
     "date-fns": "3.6.0",

src/components/accordion/accordion.e2e.ts

@@ -2,8 +2,8 @@ import { assert, expect } from '@open-wc/testing';
 import { nothing } from 'lit';
 import { html } from 'lit/static-html.js';
 
-import { fixture } from '../core/testing/private.js';
-import { waitForCondition, waitForLitRender, EventSpy, isSsr } from '../core/testing.js';
+import { fixture, isSsr } from '../core/testing/private.js';
+import { waitForCondition, waitForLitRender, EventSpy } from '../core/testing.js';
 import {
   SbbExpansionPanelElement,
   type SbbExpansionPanelHeaderElement,