build: configure visual regression tests with web test runner

sbb-design-systems · Mar 1, 2024 · f63ad5f · f63ad5f
1 parent fdcdb43
commit f63ad5f
Show file tree

Hide file tree

Showing 49 changed files with 127 additions and 5 deletions.
diff --git a/package.json b/package.json
@@ -85,6 +85,7 @@
     "@web/test-runner-commands": "0.9.0",
     "@web/test-runner-playwright": "0.11.0",
     "@web/test-runner-puppeteer": "0.15.0",
+    "@web/test-runner-visual-regression": "0.9.0",
     "chromatic": "11.0.0",
     "custom-elements-manifest": "^2.0.0",
     "date-fns": "3.3.1",

diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-large-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-large-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-large-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-large-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-large-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-large-Webkit.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-medium-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-medium-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-medium-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-medium-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-medium-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-medium-Webkit.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-micro-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-micro-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-micro-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-micro-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-micro-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-micro-Webkit.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-small-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-small-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-small-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-small-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-small-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-small-Webkit.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-ultra-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-ultra-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-ultra-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-ultra-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-ultra-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-ultra-Webkit.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-wide-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-wide-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-wide-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-wide-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-wide-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-wide-Webkit.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-zero-Chromium.png b/src/components/button/button/__snapshot__/baseline/sbb-button-zero-Chromium.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-zero-Firefox.png b/src/components/button/button/__snapshot__/baseline/sbb-button-zero-Firefox.png
diff --git a/src/components/button/button/__snapshot__/baseline/sbb-button-zero-Webkit.png b/src/components/button/button/__snapshot__/baseline/sbb-button-zero-Webkit.png
diff --git a/src/components/button/button/button.spec.ts b/src/components/button/button/button.spec.ts
@@ -1,7 +1,7 @@
 import { expect, fixture } from '@open-wc/testing';
 import { html } from 'lit/static-html.js';
 
-import { waitForLitRender } from '../../core/testing';
+import { testVisualRegressionSnapshot, waitForLitRender } from '../../core/testing';
 
 import '../../form-field';
 import './button';
@@ -111,4 +111,8 @@ describe('sbb-button', () => {
     const button = root.querySelector('sbb-button');
     expect(button).to.have.attribute('data-icon-small');
   });
+
+  testVisualRegressionSnapshot({
+    'sbb-button': html`<sbb-button>My Button</sbb-button>`,
+  });
 });
diff --git a/src/components/core/testing/index.ts b/src/components/core/testing/index.ts
@@ -1,7 +1,9 @@
+export * from './a11y-tree-snapshot';
 export * from './event-spy';
 export * from './fixture';
 export * from './mocha-extensions';
 export * from './platform';
 export * from './scroll';
+export * from './visual-regression-snapshot';
 export * from './wait-for-condition';
 export * from './wait-for-render';
diff --git a/src/components/core/testing/visual-regression-snapshot.ts b/src/components/core/testing/visual-regression-snapshot.ts
@@ -0,0 +1,43 @@
+import { fixture } from '@open-wc/testing';
+import {
+  SbbBreakpointZeroMax,
+  SbbBreakpointMicroMax,
+  SbbBreakpointSmallMax,
+  SbbBreakpointMediumMax,
+  SbbBreakpointLargeMax,
+  SbbBreakpointWideMax,
+  SbbBreakpointUltraMax,
+} from '@sbb-esta/lyne-design-tokens';
+import { setViewport } from '@web/test-runner-commands';
+import { visualDiff } from '@web/test-runner-visual-regression';
+import { html, type TemplateResult } from 'lit';
+
+const viewportSizes = {
+  zero: SbbBreakpointZeroMax,
+  micro: SbbBreakpointMicroMax,
+  small: SbbBreakpointSmallMax,
+  medium: SbbBreakpointMediumMax,
+  large: SbbBreakpointLargeMax,
+  wide: SbbBreakpointWideMax,
+  ultra: SbbBreakpointUltraMax,
+};
+
+export function testVisualRegressionSnapshot(suite: Record<string, TemplateResult>): void {
+  describe('visual regression', () => {
+    for (const [viewport, size] of Object.entries(viewportSizes)) {
+      describe(viewport, () => {
+        before(async () => {
+          await setViewport({ width: size, height: 400 });
+        });
+
+        for (const [name, template] of Object.entries(suite)) {
+          it(name, async function () {
+            this.timeout(10000);
+            const root = await fixture(html`<div style="padding: 0.5rem;">${template}</div>`);
+            await visualDiff(root, `${name}-${viewport}`);
+          });
+        }
+      });
+    }
+  });
+}
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-large-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-large-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-large-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-large-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-large-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-large-Webkit.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-medium-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-medium-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-medium-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-medium-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-medium-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-medium-Webkit.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-micro-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-micro-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-micro-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-micro-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-micro-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-micro-Webkit.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-small-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-small-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-small-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-small-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-small-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-small-Webkit.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-ultra-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-ultra-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-ultra-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-ultra-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-ultra-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-ultra-Webkit.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-wide-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-wide-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-wide-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-wide-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-wide-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-wide-Webkit.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-zero-Chromium.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-zero-Chromium.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-zero-Firefox.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-zero-Firefox.png
diff --git a/src/components/title/__snapshot__/baseline/sbb-title level=1-zero-Webkit.png b/src/components/title/__snapshot__/baseline/sbb-title level=1-zero-Webkit.png
diff --git a/src/components/title/title.spec.ts b/src/components/title/title.spec.ts
@@ -1,7 +1,7 @@
 import { expect, fixture } from '@open-wc/testing';
 import { html } from 'lit/static-html.js';
 
-import { testA11yTreeSnapshot } from '../core/testing/a11y-tree-snapshot';
+import { testA11yTreeSnapshot, testVisualRegressionSnapshot } from '../core/testing';
 
 import './title';
 
@@ -22,4 +22,8 @@ describe('sbb-title', () => {
   });
 
   testA11yTreeSnapshot(html`<sbb-title level="1" visual-level="2">Sample Title Text</sbb-title>`);
+
+  testVisualRegressionSnapshot({
+    'sbb-title level=1': html`<sbb-title>My Title</sbb-title>`,
+  });
 });
diff --git a/web-test-runner.config.js b/web-test-runner.config.js
@@ -1,13 +1,17 @@
+import { existsSync, readFileSync } from 'fs';
+import { dirname, join } from 'path';
 import { Worker } from 'worker_threads';
+
 import { defaultReporter, dotReporter, summaryReporter } from '@web/test-runner';
 import { playwrightLauncher } from '@web/test-runner-playwright';
 import { puppeteerLauncher } from '@web/test-runner-puppeteer';
 import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins';
-import { existsSync, readFileSync } from 'fs';
+import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin';
 import * as glob from 'glob';
 import * as sass from 'sass';
 import { createServer } from 'vite';
 
+const root = new URL('./', import.meta.url);
 const isCIEnvironment = !!process.env.CI || process.argv.includes('--ci');
 const isDebugMode = process.argv.includes('--debug');
 const firefox = process.argv.includes('--firefox');
@@ -59,7 +63,7 @@ const testRunnerHtml = (testFramework, _config, group) => `
 
 // Temporary workaround, until all files are migrated to ssr testing.
 const e2eFiles = glob
-  .sync('**/*.e2e.ts', { cwd: new URL('.', import.meta.url) })
+  .sync('**/*.e2e.ts', { cwd: root })
   .filter((f) => readFileSync(f, 'utf8').includes('${fixture.name}'));
 
 /** @type {import('@web/test-runner').TestRunnerConfig} */
@@ -72,7 +76,19 @@ export default {
   nodeResolve: true,
   reporters: isDebugMode ? [defaultReporter(), summaryReporter()] : [minimalReporter()],
   browsers: browsers,
-  plugins: [vitePlugin(), a11ySnapshotPlugin()],
+  plugins: [
+    vitePlugin(),
+    a11ySnapshotPlugin(),
+    visualRegressionPlugin({
+      update: process.argv.includes('--update-visual-baseline'),
+      getBaselineName: ({ browser, name, testFile }) =>
+        join(dirname(testFile), `__snapshot__/baseline/${name}-${browser}.png`),
+      getDiffName: ({ browser, name, testFile }) =>
+        join(dirname(testFile), `__snapshot__/failed/${name}-${browser}-diff.png`),
+      getFailedName: ({ browser, name, testFile }) =>
+        join(dirname(testFile), `__snapshot__/failed/${name}-${browser}.png`),
+    }),
+  ],
   testFramework: {
     config: {
       timeout: '6000',

diff --git a/yarn.lock b/yarn.lock
@@ -3357,6 +3357,13 @@
   resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.5.tgz#ec10755e871497bcd83efe927e43ec46e8c0747e"
   integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==
 
+"@types/mkdirp@^1.0.1":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.2.tgz#8d0bad7aa793abe551860be1f7ae7f3198c16666"
+  integrity sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/mocha@^10.0.4":
   version "10.0.6"
   resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b"
@@ -3392,6 +3399,20 @@
   resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb"
   integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==
 
+"@types/pixelmatch@^5.2.2":
+  version "5.2.6"
+  resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.6.tgz#fba6de304ac958495f27d85989f5c6bb7499a686"
+  integrity sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==
+  dependencies:
+    "@types/node" "*"
+
+"@types/pngjs@^6.0.0":
+  version "6.0.4"
+  resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-6.0.4.tgz#9a457aebabd944efde1a773a0fa1d74933e8021b"
+  integrity sha512-atAK9xLKOnxiuArxcHovmnOUUGBZOQ3f0vCf43FnoKs6XnqiambT1kkJWmdo71IR+BoXSh+CueeFR0GfH3dTlQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/pretty-hrtime@^1.0.0":
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/@types/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#ee1bd8c9f7a01b3445786aad0ef23aba5f511a44"
@@ -4017,6 +4038,20 @@
     "@web/test-runner-core" "^0.13.0"
     puppeteer "^20.0.0"
 
+"@web/[email protected]":
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/@web/test-runner-visual-regression/-/test-runner-visual-regression-0.9.0.tgz#9875a4871a24f8bf520c97b80ce548e81bce51e6"
+  integrity sha512-06M1WffLy+BJo08s57RumKYUULD/UB8u7DgZ8/MJZYCt+7r4Vt54w34CwSGHCpeDLY8Z/YkxecafvzDjuLnEJQ==
+  dependencies:
+    "@types/mkdirp" "^1.0.1"
+    "@types/pixelmatch" "^5.2.2"
+    "@types/pngjs" "^6.0.0"
+    "@web/test-runner-commands" "^0.9.0"
+    "@web/test-runner-core" "^0.13.0"
+    mkdirp "^1.0.4"
+    pixelmatch "^5.2.1"
+    pngjs "^7.0.0"
+
 "@web/[email protected]":
   version "0.18.0"
   resolved "https://registry.yarnpkg.com/@web/test-runner/-/test-runner-0.18.0.tgz#70a99bb7b4f78555d0944fb53ffd2b1cb3423eb7"
@@ -10276,6 +10311,13 @@ pirates@^4.0.4, pirates@^4.0.6:
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9"
   integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==
 
+pixelmatch@^5.2.1:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a"
+  integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==
+  dependencies:
+    pngjs "^6.0.0"
+
 pkg-dir@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3"
@@ -10316,6 +10358,16 @@ pluralize@^8.0.0:
   resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
   integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==
 
+pngjs@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821"
+  integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==
+
+pngjs@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26"
+  integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==
+
 polished@^4.2.2:
   version "4.3.1"
   resolved "https://registry.yarnpkg.com/polished/-/polished-4.3.1.tgz#5a00ae32715609f83d89f6f31d0f0261c6170548"