From f97b9705ac8c617d4e5b582c3e8abbbc09b9a32f Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 12 Apr 2024 21:43:48 +0200 Subject: [PATCH 01/48] build: implement visual regression testing with @web/test-runner --- .github/default.conf | 3 ++ .github/workflows/continuous-integration.yml | 32 +++++++++++ .github/workflows/release-please.yml | 14 ++--- Dockerfile | 2 +- package.json | 2 + .../button/button/button.snapshot.spec.ts | 48 +++++++++++++++++ src/components/core/testing/private.ts | 3 ++ .../core/testing/private/describe-each.ts | 36 +++++++++++++ .../testing/private/describe-viewports.ts | 32 +++++++++++ .../private/visual-regression-snapshot.ts | 41 ++++++++++++++ .../baseline.Dockerfile | 9 ++++ .../baseline.nginx.conf | 37 +++++++++++++ tools/visual-regression-testing/exec.ts | 52 ++++++++++++++++++ .../testing.Dockerfile | 7 +++ .../testing.Dockerfile.dockerignore | 3 ++ tools/web-test-runner/index.js | 1 + .../visual-regression-plugin-config.js | 54 +++++++++++++++++++ web-test-runner.config.js | 54 +++++++++++-------- yarn.lock | 52 ++++++++++++++++++ 19 files changed, 453 insertions(+), 29 deletions(-) create mode 100644 src/components/button/button/button.snapshot.spec.ts create mode 100644 src/components/core/testing/private/describe-each.ts create mode 100644 src/components/core/testing/private/describe-viewports.ts create mode 100644 src/components/core/testing/private/visual-regression-snapshot.ts create mode 100644 tools/visual-regression-testing/baseline.Dockerfile create mode 100644 tools/visual-regression-testing/baseline.nginx.conf create mode 100644 tools/visual-regression-testing/exec.ts create mode 100644 tools/visual-regression-testing/testing.Dockerfile create mode 100644 tools/visual-regression-testing/testing.Dockerfile.dockerignore create mode 100644 tools/web-test-runner/visual-regression-plugin-config.js diff --git a/.github/default.conf b/.github/default.conf index dee95097b5..bdb9834ab9 100644 --- a/.github/default.conf +++ b/.github/default.conf @@ -2,6 +2,9 @@ tcp_nopush on; tcp_nodelay on; types_hash_max_size 2048; +# Suppresses the nginx version in the Server header. +server_tokens off; + # Determine if it's a valid origin and set it in the $cors variable. map "$http_origin" $cors { default ''; diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 2beaf55455..5a5ab52f7b 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -151,3 +151,35 @@ jobs: zip: true onlyChanged: true externals: '**/components/core/styles/**/*.scss' + + visual-regression: + runs-on: ubuntu-latest + permissions: + packages: write + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: [test] + env: + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run tests + run: yarn test:visual-regression --update-visual-baseline + env: + NODE_ENV: production + + - name: Build and push visual regression baseline + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + docker build \ + --file tools/visual-regression-testing/baseline.Dockerfile \ + --tag $IMAGE_REPO_VISUAL_REGRESSION:baseline \ + . + docker push $IMAGE_REPO_VISUAL_REGRESSION:baseline diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 21720da636..9b8f89edd4 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -35,7 +35,7 @@ jobs: if: needs.release-please.outputs.releases_created runs-on: ubuntu-latest env: - IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook + IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook VERSION: ${{ needs.release-please.outputs.version }} steps: - uses: actions/checkout@v4 @@ -68,13 +68,13 @@ jobs: - name: 'Container: Login to GitHub Container Repository' run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:$VERSION -t $IMAGE_REPO:latest . + run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest . env: DOCKER_BUILDKIT: 1 - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:$VERSION + run: docker push $IMAGE_REPO_STORYBOOK:$VERSION - name: 'Container: Publish image as latest' - run: docker push $IMAGE_REPO:latest + run: docker push $IMAGE_REPO_STORYBOOK:latest - name: Generate chromatic stories run: yarn generate:chromatic-stories @@ -98,7 +98,7 @@ jobs: if: needs.release-please.outputs.releases_created != true runs-on: ubuntu-latest env: - IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook + IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook VERSION: ${{ needs.release-please.outputs.version }} steps: - uses: actions/checkout@v4 @@ -118,11 +118,11 @@ jobs: - name: 'Container: Login to GitHub Container Repository' run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:dev . + run: docker build --tag $IMAGE_REPO_STORYBOOK:dev . env: DOCKER_BUILDKIT: 1 - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:dev + run: docker push $IMAGE_REPO_STORYBOOK:dev - name: Generate chromatic stories run: yarn generate:chromatic-stories diff --git a/Dockerfile b/Dockerfile index 57210bfa3f..2220166b4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nginxinc/nginx-unprivileged:stable +FROM ghcr.io/nginxinc/nginx-unprivileged:stable # Copy nginx configuration COPY ./.github/default.conf /etc/nginx/conf.d/default.conf diff --git a/package.json b/package.json index b2a9fc7121..0c4845ec14 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "test:csr": "wtr --group default", "test:ssr:hydrated": "wtr --group e2e-ssr-hydrated", "test:ssr:non-hydrated": "wtr --group e2e-ssr-non-hydrated", + "test:visual-regression": "tsx tools/visual-regression-testing/exec.ts --group=visual-regression --all-browsers", "prepare": "husky" }, "dependencies": { @@ -90,6 +91,7 @@ "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", "@web/test-runner-puppeteer": "0.16.0", + "@web/test-runner-visual-regression": "0.9.0", "chromatic": "11.3.0", "custom-elements-manifest": "2.0.0", "date-fns": "3.6.0", diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts new file mode 100644 index 0000000000..c675869609 --- /dev/null +++ b/src/components/button/button/button.snapshot.spec.ts @@ -0,0 +1,48 @@ +import { html } from 'lit'; + +import { + describeEach, + describeViewports, + fixture, + visualRegressionSnapshot, +} from '../../core/testing/private.js'; +import type { SbbButtonSize } from '../common.js'; + +import './button.js'; + +describe(`sbb-button`, () => { + describe('visual-regression', () => { + const cases = { + size: ['s', 'm', 'l'] as SbbButtonSize[], + disabled: [false, true], + negative: [false, true], + iconName: [undefined, 'arrow-right-small'], + }; + + describeViewports(() => { + describeEach(cases, ({ size, disabled, negative, iconName }) => { + let root: HTMLElement; + beforeEach(async () => { + root = await fixture(html` +
+ Button +
+ `); + }); + + visualRegressionSnapshot(() => root); + }); + }); + }); +}); diff --git a/src/components/core/testing/private.ts b/src/components/core/testing/private.ts index 57b13775f6..45ecfb614c 100644 --- a/src/components/core/testing/private.ts +++ b/src/components/core/testing/private.ts @@ -1,5 +1,8 @@ export * from './private/a11y-tree-snapshot.js'; +export * from './private/describe-each.js'; +export * from './private/describe-viewports.js'; export * from './private/dispatch-events.js'; export * from './private/event-objects.js'; export * from './private/fixture.js'; export * from './private/type-in-element.js'; +export * from './private/visual-regression-snapshot.js'; diff --git a/src/components/core/testing/private/describe-each.ts b/src/components/core/testing/private/describe-each.ts new file mode 100644 index 0000000000..bfa5d2c1ef --- /dev/null +++ b/src/components/core/testing/private/describe-each.ts @@ -0,0 +1,36 @@ +function partialDescribeEach>( + cases: T, + payload: Record, + suiteRun: (params: { [K in keyof T]: T[K][number] }) => void, +): void { + const [key, ...keys] = Object.keys(cases); + const values = cases[key]; + if (keys.length) { + const partialCases = keys.reduce( + (current, next) => Object.assign(current, { [next]: cases[next] }), + {} as T, + ); + for (const value of values) { + partialDescribeEach(partialCases, { ...payload, [key]: value }, suiteRun); + } + } else { + for (const value of values) { + const finalPayload = { ...payload, [key]: value }; + describe( + Object.entries(finalPayload) + .map(([key, value]) => `${key}=${value}`) + .join(', '), + function () { + suiteRun.call(this, finalPayload); + }, + ); + } + } +} + +export function describeEach>( + cases: T, + suiteRun: (params: { [K in keyof T]: T[K][number] }) => void, +): void { + partialDescribeEach(cases, {} as Record, suiteRun); +} diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts new file mode 100644 index 0000000000..de764fe737 --- /dev/null +++ b/src/components/core/testing/private/describe-viewports.ts @@ -0,0 +1,32 @@ +import { + SbbBreakpointLargeMax, + SbbBreakpointMediumMax, + SbbBreakpointMicroMax, + SbbBreakpointSmallMax, + SbbBreakpointUltraMax, + SbbBreakpointWideMax, + SbbBreakpointZeroMax, +} from '@sbb-esta/lyne-design-tokens'; +import { setViewport } from '@web/test-runner-commands'; + +const viewportSizes = { + zero: SbbBreakpointZeroMax, + micro: SbbBreakpointMicroMax, + small: SbbBreakpointSmallMax, + medium: SbbBreakpointMediumMax, + large: SbbBreakpointLargeMax, + wide: SbbBreakpointWideMax, + ultra: SbbBreakpointUltraMax, +}; + +export function describeViewports(fn: (this: Mocha.Suite) => void): void { + for (const [size, value] of Object.entries(viewportSizes)) { + describe(`viewport=${size}`, function () { + before(async () => { + await setViewport({ width: value, height: 400 }); + }); + + fn.call(this); + }); + } +} diff --git a/src/components/core/testing/private/visual-regression-snapshot.ts b/src/components/core/testing/private/visual-regression-snapshot.ts new file mode 100644 index 0000000000..8c63501b1e --- /dev/null +++ b/src/components/core/testing/private/visual-regression-snapshot.ts @@ -0,0 +1,41 @@ +import { sendKeys, sendMouse } from '@web/test-runner-commands'; +import { visualDiff } from '@web/test-runner-visual-regression'; + +export function imageName(test: Mocha.Runnable): string { + return test!.fullTitle().replaceAll(', ', '-').replaceAll(' ', '_'); +} + +export function visualRegressionSnapshot(snapshotElement: () => HTMLElement): void { + it('default', async function () { + await visualDiff(snapshotElement(), imageName(this.test!)); + }); + + it('focus', async function () { + await sendKeys({ press: 'Tab' }); + await visualDiff(snapshotElement(), imageName(this.test!)); + }); + + it('hover', async function () { + const element = snapshotElement(); + const positionElement = element.localName.startsWith('sbb-') + ? element + : element.firstElementChild!; + const position = positionElement.getBoundingClientRect(); + await sendMouse({ + type: 'move', + position: [ + Math.round(position.x + position.width / 2), + Math.round(position.y + position.height / 2), + ], + }); + + try { + await visualDiff(element, imageName(this.test!)); + } finally { + await sendMouse({ + type: 'move', + position: [0, 0], + }); + } + }); +} diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile new file mode 100644 index 0000000000..99c4074d78 --- /dev/null +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -0,0 +1,9 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY ./dist/screenshots /usr/share/nginx/html + +USER root +RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete +RUN find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; +USER $UID diff --git a/tools/visual-regression-testing/baseline.nginx.conf b/tools/visual-regression-testing/baseline.nginx.conf new file mode 100644 index 0000000000..a2d705e7a4 --- /dev/null +++ b/tools/visual-regression-testing/baseline.nginx.conf @@ -0,0 +1,37 @@ +tcp_nopush on; +tcp_nodelay on; +types_hash_max_size 2048; + +server { + listen 8080 default_server; + server_name _; + root /usr/share/nginx/html; + index index.html index.htm; + + # Suppresses the nginx version in the Server header. + server_tokens off; + + location / { + expires -1; + add_header Pragma "no-cache"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + try_files $uri $uri/ /index.html =404; + } + + location ~* \.(?:png)$ { + expires 1y; + access_log off; + add_header Cache-Control "public"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + } + + location ~* \.(?:css|js)$ { + expires 1y; + access_log off; + add_header Cache-Control "public"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + } +} \ No newline at end of file diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts new file mode 100644 index 0000000000..922c1c693f --- /dev/null +++ b/tools/visual-regression-testing/exec.ts @@ -0,0 +1,52 @@ +// This script serves as checking which OS visual regression testing is run +// and if it is not Linux, runs it in a container. + +import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; +import { platform } from 'os'; + +import { startTestRunner } from '@web/test-runner'; + +const args = process.argv.slice(2); +if (platform() === 'linux' && !process.env.DEBUG) { + startTestRunner(); +} else { + function executableIsAvailable(name: string): string | null { + try { + execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); + return name; + } catch (error) { + return null; + } + } + + const containerCmd = executableIsAvailable('docker') ?? executableIsAvailable('podman'); + if (!containerCmd) { + console.log('Either docker or podman need to be installed!'); + process.exit(1); + } + + const cwd = new URL('../../', import.meta.url); + const tag = 'lyne-vrt'; + const branchName = execSync('git rev-parse --abbrev-ref HEAD'); + const execOptions: ExecSyncOptionsWithStringEncoding = { + encoding: 'utf8', + stdio: 'inherit', + cwd, + }; + execSync( + `${containerCmd} build ` + + '--file=tools/visual-regression-testing/testing.Dockerfile ' + + //`--build-arg=VERSION=${readFileSync(new URL('../../.nvmrc', import.meta.url), 'utf8').replace('v', '')}` + + `--tag=${tag} .`, + execOptions, + ); + console.log(`\nTest image ready\n`); + execSync( + `${containerCmd} run -it --rm --ipc=host ` + + `--env=BRANCH_NAME="${branchName}" ` + + `--volume=./dist/screenshots:/dist/screenshots ` + + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + + tag, + execOptions, + ); +} diff --git a/tools/visual-regression-testing/testing.Dockerfile b/tools/visual-regression-testing/testing.Dockerfile new file mode 100644 index 0000000000..1afbaed5c7 --- /dev/null +++ b/tools/visual-regression-testing/testing.Dockerfile @@ -0,0 +1,7 @@ +FROM mcr.microsoft.com/playwright:v1.43.0-jammy + +COPY package.json ./ +COPY yarn.lock ./ +RUN yarn install --frozen-lockfile --non-interactive + +COPY . . diff --git a/tools/visual-regression-testing/testing.Dockerfile.dockerignore b/tools/visual-regression-testing/testing.Dockerfile.dockerignore new file mode 100644 index 0000000000..5de02b1469 --- /dev/null +++ b/tools/visual-regression-testing/testing.Dockerfile.dockerignore @@ -0,0 +1,3 @@ +coverage +dist +node_modules \ No newline at end of file diff --git a/tools/web-test-runner/index.js b/tools/web-test-runner/index.js index 0eb5bcd92d..d4f095b400 100644 --- a/tools/web-test-runner/index.js +++ b/tools/web-test-runner/index.js @@ -2,4 +2,5 @@ export * from './minimal-reporter.js'; export * from './patched-summary-reporter.js'; export * from './ssr-plugin.js'; +export * from './visual-regression-plugin-config.js'; export * from './vite-plugin.js'; diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js new file mode 100644 index 0000000000..209fe02c12 --- /dev/null +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -0,0 +1,54 @@ +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; +import { dirname, extname, join } from 'path'; + +const baselineUrl = 'http://localhost:8080/'; //'https://lyne-visual-regression-baseline.app.sbb.ch/'; + +export const visualRegressionConfig = (update) => + /** @type {Parameters[0]} */ + ({ + update, + baseDir: 'dist/screenshots', + async getBaseline({ filePath, baseDir, name }) { + if (existsSync(filePath)) { + return readFileSync(filePath); + } + + const cacheFile = join(baseDir, '.cache', name + extname(filePath)); + const cacheFileDetails = cacheFile + '.json'; + mkdirSync(dirname(cacheFile), { recursive: true }); + const baselineFileUrl = baselineUrl + name + extname(filePath); + const downloadFile = async () => { + const response = await fetch(baselineFileUrl); + if (response.ok) { + writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); + writeFileSync( + cacheFileDetails, + JSON.stringify({ etag: response.headers.get('etag') }, null, 2), + 'utf8', + ); + } + }; + + if (existsSync(cacheFileDetails)) { + const details = JSON.parse(readFileSync(cacheFileDetails)); + const response = await fetch(baselineFileUrl, { + method: 'HEAD', + headers: { 'if-none-match': details.etag }, + }); + if (response.status === 200) { + await downloadFile(); + return readFileSync(cacheFile); + } else if (response.status === 404) { + unlinkSync(cacheFile); + unlinkSync(cacheFileDetails); + } else if (response.status === 304) { + return readFileSync(cacheFile); + } else { + console.error(`Unexpected response from baseline service: ${response.status} (${name})`); + } + } else { + await downloadFile(); + return readFileSync(cacheFile); + } + }, + }); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index a410f98812..0bdb2a5195 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -2,6 +2,7 @@ import { defaultReporter } from '@web/test-runner'; import { playwrightLauncher } from '@web/test-runner-playwright'; import { puppeteerLauncher } from '@web/test-runner-puppeteer'; import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins'; +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; import * as sass from 'sass'; import { cpus } from 'node:os'; @@ -9,14 +10,18 @@ import { minimalReporter, patchedSummaryReporter, ssrPlugin, + visualRegressionConfig, vitePlugin, } from './tools/web-test-runner/index.js'; const isCIEnvironment = !!process.env.CI || process.argv.includes('--ci'); const isDebugMode = process.argv.includes('--debug'); +const allBrowsers = process.argv.includes('--all-browsers'); const firefox = process.argv.includes('--firefox'); const webkit = process.argv.includes('--webkit'); const concurrency = process.argv.includes('--parallel') ? {} : { concurrency: 1 }; +const updateBaseImages = + process.argv.includes('--update-visual-baseline') || process.argv.includes('--uv'); const stylesCompiler = new sass.initCompiler(); const renderStyles = () => @@ -24,25 +29,26 @@ const renderStyles = () => loadPaths: ['.', './node_modules/'], }).css; -const browsers = isCIEnvironment - ? [ - // Parallelism has problems, we need force concurrency to 1 - playwrightLauncher({ product: 'chromium', ...concurrency }), - playwrightLauncher({ product: 'firefox', ...concurrency }), - playwrightLauncher({ product: 'webkit', ...concurrency }), - ] - : firefox - ? [playwrightLauncher({ product: 'firefox' })] - : webkit - ? [playwrightLauncher({ product: 'webkit' })] - : isDebugMode - ? [ - puppeteerLauncher({ - launchOptions: { headless: false, devtools: true }, - ...concurrency, - }), - ] - : [playwrightLauncher({ product: 'chromium' })]; +const browsers = + isCIEnvironment || allBrowsers + ? [ + // Parallelism has problems, we need force concurrency to 1 + playwrightLauncher({ product: 'chromium', ...concurrency }), + playwrightLauncher({ product: 'firefox', ...concurrency }), + playwrightLauncher({ product: 'webkit', ...concurrency }), + ] + : firefox + ? [playwrightLauncher({ product: 'firefox' })] + : webkit + ? [playwrightLauncher({ product: 'webkit' })] + : isDebugMode + ? [ + puppeteerLauncher({ + launchOptions: { headless: false, devtools: true }, + ...concurrency, + }), + ] + : [playwrightLauncher({ product: 'chromium' })]; const groupNameOverride = process.argv.includes('--ssr-hydrated') ? 'e2e-ssr-hydrated' @@ -82,11 +88,12 @@ const suppressedLogs = [ /** @type {import('@web/test-runner').TestRunnerConfig} */ export default { - files: ['src/**/*.{e2e,spec}.ts'], + files: ['src/**/*.{e2e,spec,!snapshot.spec}.ts'], groups: [ // Disable ssr tests until stabilized. // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, + { name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }, ], nodeResolve: true, concurrency: resolveConcurrency(), @@ -95,7 +102,12 @@ export default { ? [defaultReporter(), patchedSummaryReporter()] : [minimalReporter()], browsers: browsers, - plugins: [a11ySnapshotPlugin(), ssrPlugin(), vitePlugin()], + plugins: [ + a11ySnapshotPlugin(), + ssrPlugin(), + vitePlugin(), + visualRegressionPlugin(visualRegressionConfig(updateBaseImages)), + ], testFramework: { config: { timeout: '10000', diff --git a/yarn.lock b/yarn.lock index afab216c3b..52564dad9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2954,6 +2954,13 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-5.1.2.tgz#07508b45797cb81ec3f273011b054cd0755eddca" integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA== +"@types/mkdirp@^1.0.1": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.2.tgz#8d0bad7aa793abe551860be1f7ae7f3198c16666" + integrity sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ== + dependencies: + "@types/node" "*" + "@types/mocha@10.0.6": version "10.0.6" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b" @@ -2981,6 +2988,20 @@ resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb" integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g== +"@types/pixelmatch@^5.2.2": + version "5.2.6" + resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.6.tgz#fba6de304ac958495f27d85989f5c6bb7499a686" + integrity sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg== + dependencies: + "@types/node" "*" + +"@types/pngjs@^6.0.0": + version "6.0.4" + resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-6.0.4.tgz#9a457aebabd944efde1a773a0fa1d74933e8021b" + integrity sha512-atAK9xLKOnxiuArxcHovmnOUUGBZOQ3f0vCf43FnoKs6XnqiambT1kkJWmdo71IR+BoXSh+CueeFR0GfH3dTlQ== + dependencies: + "@types/node" "*" + "@types/pretty-hrtime@^1.0.0": version "1.0.3" resolved "https://registry.yarnpkg.com/@types/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#ee1bd8c9f7a01b3445786aad0ef23aba5f511a44" @@ -3650,6 +3671,20 @@ "@web/test-runner-core" "^0.13.0" puppeteer "^22.0.0" +"@web/test-runner-visual-regression@0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@web/test-runner-visual-regression/-/test-runner-visual-regression-0.9.0.tgz#9875a4871a24f8bf520c97b80ce548e81bce51e6" + integrity sha512-06M1WffLy+BJo08s57RumKYUULD/UB8u7DgZ8/MJZYCt+7r4Vt54w34CwSGHCpeDLY8Z/YkxecafvzDjuLnEJQ== + dependencies: + "@types/mkdirp" "^1.0.1" + "@types/pixelmatch" "^5.2.2" + "@types/pngjs" "^6.0.0" + "@web/test-runner-commands" "^0.9.0" + "@web/test-runner-core" "^0.13.0" + mkdirp "^1.0.4" + pixelmatch "^5.2.1" + pngjs "^7.0.0" + "@web/test-runner@0.18.1": version "0.18.1" resolved "https://registry.yarnpkg.com/@web/test-runner/-/test-runner-0.18.1.tgz#5cac11d29f525214e39ae74e3a33ae90551623f3" @@ -9094,6 +9129,13 @@ pirates@^4.0.6: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== +pixelmatch@^5.2.1: + version "5.3.0" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a" + integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q== + dependencies: + pngjs "^6.0.0" + pkg-dir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" @@ -9134,6 +9176,16 @@ pluralize@^8.0.0: resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== +pngjs@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821" + integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg== + +pngjs@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26" + integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow== + polished@^4.2.2: version "4.3.1" resolved "https://registry.yarnpkg.com/polished/-/polished-4.3.1.tgz#5a00ae32715609f83d89f6f31d0f0261c6170548" From 3599793652277a34ce0ff8604c99a7896e8e2049 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Tue, 16 Apr 2024 16:54:02 +0200 Subject: [PATCH 02/48] fix: little fixlis --- tools/visual-regression-testing/exec.ts | 6 ++-- .../visual-regression-plugin-config.js | 29 +++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index 922c1c693f..d71efc10bc 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -2,12 +2,13 @@ // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; +import { mkdirSync } from 'fs'; import { platform } from 'os'; import { startTestRunner } from '@web/test-runner'; const args = process.argv.slice(2); -if (platform() === 'linux' && !process.env.DEBUG) { +if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { startTestRunner(); } else { function executableIsAvailable(name: string): string | null { @@ -19,7 +20,7 @@ if (platform() === 'linux' && !process.env.DEBUG) { } } - const containerCmd = executableIsAvailable('docker') ?? executableIsAvailable('podman'); + const containerCmd = executableIsAvailable('podman') ?? executableIsAvailable('docker'); if (!containerCmd) { console.log('Either docker or podman need to be installed!'); process.exit(1); @@ -41,6 +42,7 @@ if (platform() === 'linux' && !process.env.DEBUG) { execOptions, ); console.log(`\nTest image ready\n`); + mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + `--env=BRANCH_NAME="${branchName}" ` + diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 209fe02c12..5b9ce18713 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -18,14 +18,21 @@ export const visualRegressionConfig = (update) => mkdirSync(dirname(cacheFile), { recursive: true }); const baselineFileUrl = baselineUrl + name + extname(filePath); const downloadFile = async () => { - const response = await fetch(baselineFileUrl); - if (response.ok) { - writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); - writeFileSync( - cacheFileDetails, - JSON.stringify({ etag: response.headers.get('etag') }, null, 2), - 'utf8', - ); + try { + const response = await fetch(baselineFileUrl); + + if (response.ok) { + writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); + writeFileSync( + cacheFileDetails, + JSON.stringify({ etag: response.headers.get('etag') }, null, 2), + 'utf8', + ); + + return readFileSync(cacheFile); + } + } catch { + /* empty */ } }; @@ -36,8 +43,7 @@ export const visualRegressionConfig = (update) => headers: { 'if-none-match': details.etag }, }); if (response.status === 200) { - await downloadFile(); - return readFileSync(cacheFile); + return await downloadFile(); } else if (response.status === 404) { unlinkSync(cacheFile); unlinkSync(cacheFileDetails); @@ -47,8 +53,7 @@ export const visualRegressionConfig = (update) => console.error(`Unexpected response from baseline service: ${response.status} (${name})`); } } else { - await downloadFile(); - return readFileSync(cacheFile); + return await downloadFile(); } }, }); From 7550f49383c2b53222bcd4b1ac853ac7662b7b5f Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Wed, 17 Apr 2024 16:45:42 +0200 Subject: [PATCH 03/48] feat: initial app --- package.json | 2 + .../diff-app/index.html | 32 ++++ .../diff-app/src/main.ts | 8 + .../diff-app/src/vite-env.d.ts | 6 + .../diff-app/tsconfig.json | 8 + .../diff-app/vite.config.ts | 150 ++++++++++++++++++ 6 files changed, 206 insertions(+) create mode 100644 tools/visual-regression-testing/diff-app/index.html create mode 100644 tools/visual-regression-testing/diff-app/src/main.ts create mode 100644 tools/visual-regression-testing/diff-app/src/vite-env.d.ts create mode 100644 tools/visual-regression-testing/diff-app/tsconfig.json create mode 100644 tools/visual-regression-testing/diff-app/vite.config.ts diff --git a/package.json b/package.json index 0c4845ec14..3ee2917906 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "build:react:production": "vite build --config src/react/vite.config.ts", "build:react:development": "NODE_ENV=development vite build --mode development --config src/react/vite.config.ts", "build:storybook": "storybook build --quiet --output-dir dist/storybook --stats-json", + "build:diff-app": "vite build --config tools/visual-regression-testing/diff-app/vite.config.ts", "build": "npm-run-all --sequential build:components build:react build:storybook", "docs": "npm-run-all --sequential docs:manifest docs:to-md", "docs:manifest": "custom-elements-manifest analyze --config tools/manifest/custom-elements-manifest.config.js", @@ -48,6 +49,7 @@ "lint:tsc:components": "tsc --noEmit --project src/components/tsconfig.json", "lint:tsc:components-spec": "tsc --noEmit --project src/components/tsconfig.spec.json", "start": "storybook dev -p 6006", + "start:diff-app": "vite --config tools/visual-regression-testing/diff-app/vite.config.ts", "test": "wtr --coverage", "test:snapshot": "yarn test:csr --ci --update-snapshots", "test:csr": "wtr --group default", diff --git a/tools/visual-regression-testing/diff-app/index.html b/tools/visual-regression-testing/diff-app/index.html new file mode 100644 index 0000000000..b6bc0adf39 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/index.html @@ -0,0 +1,32 @@ + + + + Visual Regression Tests Comparison + + + + + + Hi + + + + diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts new file mode 100644 index 0000000000..efabec6e06 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -0,0 +1,8 @@ +import '../../../../src/components/title.js'; + +import '../../../../src/components/core/styles/global.scss'; + +// eslint-disable-next-line import-x/no-unresolved +import { screenshots } from 'virtual:screenshots'; + +console.log(screenshots); diff --git a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts new file mode 100644 index 0000000000..8b67858995 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts @@ -0,0 +1,6 @@ +/// + +declare module 'virtual:screenshots' { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + export const screenshots: Record; +} diff --git a/tools/visual-regression-testing/diff-app/tsconfig.json b/tools/visual-regression-testing/diff-app/tsconfig.json new file mode 100644 index 0000000000..e4072bd67d --- /dev/null +++ b/tools/visual-regression-testing/diff-app/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "rootDir": "../../..", + "baseUrl": "." + }, + "include": ["./**/*.ts", "./src/vite-env.d.ts"] +} diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts new file mode 100644 index 0000000000..75d4501e46 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -0,0 +1,150 @@ +import { existsSync, readdirSync, readFileSync } from 'fs'; +import { relative } from 'path'; + +import { + defineConfig, + mergeConfig, + type PluginOption, + type ResolvedConfig, + type UserConfig, +} from 'vite'; + +import rootConfig from '../../../vite.config.js'; +import { distDir } from '../../vite/index.js'; + +const packageRoot = new URL('.', import.meta.url); +const screenshotsDir = new URL(`./screenshots/`, distDir); + +export interface FailedFiles { + browserName: string; + name: string; + failedFile: string; + diffFile: string; + baselineFile: string; + isNew: boolean; +} + +function prepareScreenshots(): PluginOption { + let viteConfig: ResolvedConfig; + const virtualModuleId = 'virtual:screenshots'; + const resolvedVirtualModuleId = '\0' + virtualModuleId; + + return { + name: 'prepare screenshot', + configResolved(config) { + viteConfig = config; + }, + resolveId(id) { + if (id === virtualModuleId) { + return resolvedVirtualModuleId; + } + }, + load(id) { + if (id === resolvedVirtualModuleId) { + const browsers = readdirSync(screenshotsDir, { withFileTypes: true }) + .filter((d) => d.name !== '.cache') + .map((d) => d.name); + + const screenshotsMeta = browsers + .filter((browserName) => existsSync(new URL(`./${browserName}/failed/`, screenshotsDir))) + .flatMap((browserName) => { + const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); + + return readdirSync(failedDir, { + withFileTypes: true, + }) + .filter((d) => !d.name.endsWith('-diff.png')) + .map((d) => { + const failedFilePath = new URL(`./${d.name}`, failedDir); + const diffFilePath = new URL( + `./${d.name.replace(/.png$/, '-diff.png')}`, + failedDir, + ); + const baselineFilePath = new URL( + `./${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + const baselineCacheFilePath = new URL( + `./.cache/${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + + const isNew = !existsSync(diffFilePath); + + const assetsScreenshots = 'assets/screenshots/'; + const failedRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, failedFilePath.pathname); + const diffRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); + const baselineRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, baselineFilePath.pathname); + + if (viteConfig.command !== 'serve') { + this.emitFile({ + type: 'asset', + fileName: failedRelativeFileName, + source: readFileSync(failedFilePath), + }); + + if (!isNew) { + this.emitFile({ + type: 'asset', + fileName: diffRelativeFileName, + source: readFileSync(diffFilePath), + }); + + this.emitFile({ + type: 'asset', + fileName: baselineRelativeFileName, + source: readFileSync( + existsSync(baselineFilePath) ? baselineFilePath : baselineCacheFilePath, + ), + }); + } + } + + return { + browserName, + name: d.name, + failedFile: failedRelativeFileName, + diffFile: diffRelativeFileName, + baselineFile: baselineRelativeFileName, + isNew, + }; + }); + }) + .reduce( + (current, next) => + current.set( + next.name, + current.has(next.name) ? current.get(next.name)!.concat(next) : [next], + ), + new Map(), + ); + + return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; + } + }, + configureServer(server) { + server.middlewares.use((req, res, next) => { + if (req.url?.startsWith('/assets/screenshots/')) { + console.log(req.url); + res.end(readFileSync(new URL(`.${req.url.substring(7)}`, distDir))); + } else { + next(); + } + }); + }, + }; +} + +export default defineConfig(() => + mergeConfig(rootConfig, { + root: packageRoot.pathname, + plugins: [prepareScreenshots()], + build: { + outDir: new URL(`./diff-app/`, distDir).pathname, + emptyOutDir: true, + }, + }), +); From dad9fc7390acfe3bf9df456dbb5e92266cfdfb34 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Mon, 15 Apr 2024 09:28:06 +0200 Subject: [PATCH 04/48] build: use local image --- .github/workflows/continuous-integration.yml | 21 +++++++++++++++---- .../baseline.Dockerfile | 5 +++-- .../visual-regression-plugin-config.js | 7 ++++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 5a5ab52f7b..a8597fc777 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -23,6 +23,7 @@ jobs: integrity: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -38,6 +39,12 @@ jobs: test: runs-on: ubuntu-latest + needs: lint + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -63,6 +70,7 @@ jobs: build: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -152,14 +160,19 @@ jobs: onlyChanged: true externals: '**/components/core/styles/**/*.scss' - visual-regression: + visual-regression-baseline: runs-on: ubuntu-latest permissions: packages: write - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - needs: [test] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: test env: IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -170,7 +183,7 @@ jobs: - name: Install browser dependencies run: yarn playwright install-deps - - name: Run tests + - name: Run visual regression baseline generation run: yarn test:visual-regression --update-visual-baseline env: NODE_ENV: production diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index 99c4074d78..c637c6125b 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -4,6 +4,7 @@ COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/def COPY ./dist/screenshots /usr/share/nginx/html USER root -RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete -RUN find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; +RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete && \ + find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; && \ + find /usr/share/nginx/html/*/ -type d -name .cache -prune -exec rm -rf {} \; USER $UID diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 5b9ce18713..9030e4d482 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -1,7 +1,9 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; import { dirname, extname, join } from 'path'; -const baselineUrl = 'http://localhost:8080/'; //'https://lyne-visual-regression-baseline.app.sbb.ch/'; +const baselineUrl = process.env.CI + ? 'http://localhost:8050/' + : 'https://lyne-visual-regression-baseline.app.sbb.ch/'; export const visualRegressionConfig = (update) => /** @type {Parameters[0]} */ @@ -45,8 +47,7 @@ export const visualRegressionConfig = (update) => if (response.status === 200) { return await downloadFile(); } else if (response.status === 404) { - unlinkSync(cacheFile); - unlinkSync(cacheFileDetails); + [cacheFile, cacheFileDetails].forEach(unlinkSync); } else if (response.status === 304) { return readFileSync(cacheFile); } else { From 04f64e8d20a2d6e6f85f4d6cf702f5abf8c37608 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Wed, 17 Apr 2024 10:30:40 +0200 Subject: [PATCH 05/48] chore: clean-up --- tools/visual-regression-testing/exec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index d71efc10bc..8f493c5f55 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -1,4 +1,4 @@ -// This script serves as checking which OS visual regression testing is run +// This script checks which OS the visual regression testing is run on // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; @@ -28,7 +28,6 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { const cwd = new URL('../../', import.meta.url); const tag = 'lyne-vrt'; - const branchName = execSync('git rev-parse --abbrev-ref HEAD'); const execOptions: ExecSyncOptionsWithStringEncoding = { encoding: 'utf8', stdio: 'inherit', @@ -37,7 +36,6 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { execSync( `${containerCmd} build ` + '--file=tools/visual-regression-testing/testing.Dockerfile ' + - //`--build-arg=VERSION=${readFileSync(new URL('../../.nvmrc', import.meta.url), 'utf8').replace('v', '')}` + `--tag=${tag} .`, execOptions, ); @@ -45,7 +43,6 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + - `--env=BRANCH_NAME="${branchName}" ` + `--volume=./dist/screenshots:/dist/screenshots ` + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + tag, From 7bcfe6677850160799d63c3e6a5fd486204fe2ea Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 19 Apr 2024 17:28:24 +0200 Subject: [PATCH 06/48] feat: add container image cleanup and etag logic --- ...age-cleanup.yml => container-image-cleanup.yml} | 9 +++++++-- .github/workflows/continuous-integration.yml | 2 ++ .../visual-regression-testing/baseline.Dockerfile | 14 +++++++++++--- .../visual-regression-testing/baseline.nginx.conf | 13 +++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) rename .github/workflows/{preview-image-cleanup.yml => container-image-cleanup.yml} (90%) diff --git a/.github/workflows/preview-image-cleanup.yml b/.github/workflows/container-image-cleanup.yml similarity index 90% rename from .github/workflows/preview-image-cleanup.yml rename to .github/workflows/container-image-cleanup.yml index 3a9a8df931..15fb21b859 100644 --- a/.github/workflows/preview-image-cleanup.yml +++ b/.github/workflows/container-image-cleanup.yml @@ -1,4 +1,4 @@ -name: Preview Image Cleanup +name: Container Image Cleanup on: workflow_dispatch: {} @@ -9,7 +9,7 @@ permissions: packages: write jobs: - preview-image: + container-image-cleanup: runs-on: ubuntu-latest env: CLOSED_PR_RETENTION_DAYS: 14 @@ -59,3 +59,8 @@ jobs: package-name: lyne-components/storybook-preview package-type: container delete-only-untagged-versions: 'true' + - uses: actions/delete-package-versions@v4 + with: + package-name: lyne-components/visual-regression + package-type: container + delete-only-untagged-versions: 'true' diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index a8597fc777..d0e48d2603 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -196,3 +196,5 @@ jobs: --tag $IMAGE_REPO_VISUAL_REGRESSION:baseline \ . docker push $IMAGE_REPO_VISUAL_REGRESSION:baseline + env: + DOCKER_BUILDKIT: 1 diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index c637c6125b..f1b9ecc96b 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -4,7 +4,15 @@ COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/def COPY ./dist/screenshots /usr/share/nginx/html USER root -RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete && \ - find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; && \ - find /usr/share/nginx/html/*/ -type d -name .cache -prune -exec rm -rf {} \; +# We calculate the sha1 hashes of the png files in order to use it as etag values. +# This allows us to use HTTP caching mechanisms, which should reduce network traffic +# for the baseline comparison. +RUN cd /usr/share/nginx/html && \ + find ./*/ -type f ! -iname "*.png" -delete && \ + find ./*/ -type d -name failed -prune -exec rm -rf {} \; && \ + find ./*/ -type d -name .cache -prune -exec rm -rf {} \; && \ + echo 'map_hash_bucket_size 16384;' > /etc/nginx/conf.d/1etags.conf && \ + echo 'map $uri $pngetag {' >> /etc/nginx/conf.d/1etags.conf && \ + find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha1sum $1 | cut -d " " -f 1);"' _ {} \; >> /etc/nginx/conf.d/1etags.conf && \ + echo '}' >> /etc/nginx/conf.d/1etags.conf USER $UID diff --git a/tools/visual-regression-testing/baseline.nginx.conf b/tools/visual-regression-testing/baseline.nginx.conf index a2d705e7a4..a66a48b9ec 100644 --- a/tools/visual-regression-testing/baseline.nginx.conf +++ b/tools/visual-regression-testing/baseline.nginx.conf @@ -22,9 +22,22 @@ server { location ~* \.(?:png)$ { expires 1y; access_log off; + # We are using a custom etag logic, which uses the sha1 hash of the image as etag. + # The default etag from nginx uses file modified time and file size, which is not good + # enough for our purposes. + etag off; add_header Cache-Control "public"; add_header X-Frame-Options DENY; add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header Etag $pngetag; + + set $request_etag $http_if_none_match; + if ($request_etag = false) { + set $request_etag "-"; + } + if ($request_etag = $pngetag) { + return 304 ""; + } } location ~* \.(?:css|js)$ { From d54a5f38a1d59c47d3d615a046b73c605a5a65fc Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 26 Apr 2024 15:50:29 +0200 Subject: [PATCH 07/48] feat: implement workflow --- .github/workflows/container-image-cleanup.yml | 44 ++--- .../continuous-integration-secure.yml | 2 +- .github/workflows/continuous-integration.yml | 39 +++- .github/workflows/release-please.yml | 4 +- Dockerfile | 2 + .../button/button/button.snapshot.spec.ts | 61 +++--- src/components/core/testing.ts | 1 - src/components/core/testing/private.ts | 1 + .../core/testing/private/fixture.ts | 3 +- .../core/testing/{ => private}/platform.ts | 7 + src/components/core/testing/test-setup.ts | 2 +- .../baseline.Dockerfile | 23 +-- .../diff-app.Dockerfile | 11 ++ .../diff-app/vite.config.ts | 174 ++++++++++-------- .../etag-map-generation.sh | 10 + tools/visual-regression-testing/exec.ts | 61 +++++- .../visual-regression-plugin-config.js | 87 ++++----- web-test-runner.config.js | 23 ++- 18 files changed, 347 insertions(+), 208 deletions(-) rename src/components/core/testing/{ => private}/platform.ts (82%) create mode 100644 tools/visual-regression-testing/diff-app.Dockerfile create mode 100755 tools/visual-regression-testing/etag-map-generation.sh diff --git a/.github/workflows/container-image-cleanup.yml b/.github/workflows/container-image-cleanup.yml index 15fb21b859..0788b6f553 100644 --- a/.github/workflows/container-image-cleanup.yml +++ b/.github/workflows/container-image-cleanup.yml @@ -3,7 +3,7 @@ name: Container Image Cleanup on: workflow_dispatch: {} schedule: - - cron: '0 5 * * *' + - cron: '0 3 * * *' permissions: packages: write @@ -12,9 +12,8 @@ jobs: container-image-cleanup: runs-on: ubuntu-latest env: - CLOSED_PR_RETENTION_DAYS: 14 - PACKAGE_NAME: storybook-preview - PR_TAG_PREFIX: preview-pr + CLOSED_PR_RETENTION_DAYS: 5 + PACKAGE_NAMES: storybook-preview,visual-regression steps: - uses: actions/github-script@v7 with: @@ -23,31 +22,34 @@ jobs: const pullRequests = await github.paginate( github.rest.pulls.list.endpoint.merge({ owner, repo, state: 'all' }) ); - const twoWeeksAgo = + const retentionPivot = new Date(Date.now() - (+process.env.CLOSED_PR_RETENTION_DAYS * 24 * 60 * 60 * 1000)); - const olderThanTwoWeeks = (date) => new Date(date) < twoWeeksAgo; + const olderThanTwoWeeks = (date) => new Date(date) < retentionPivot; const isExpiredPrTag = (version) => { const prNumber = +version.metadata?.container?.tags - ?.find((t) => t.startsWith(process.env.PR_TAG_PREFIX))?.split(process.env.PR_TAG_PREFIX)[1]; + ?.find((t) => t.match(/(preview-pr|pr)(\d+)/))?.match(/(preview-pr|pr)(\d+)/)[2]; const pr = pullRequests.find((p) => p.number === prNumber); return !!prNumber && pr?.state === 'closed' && olderThanTwoWeeks(pr.closed_at); }; - const params = { - package_type: 'container', - package_name: `${repo}/${process.env.PACKAGE_NAME}`, - username: owner - }; - const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + const packageNames = process.env.PACKAGE_NAME.split(',').map((n) => n.trim()); let packageDeletionFailed = false; - for (const version of versions.filter(isExpiredPrTag)) { - try { - await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); - console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); - } catch(e) { - console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); - console.error(e); - packageDeletionFailed = true; + for (const packageName of packageNames) { + const params = { + package_type: 'container', + package_name: `${repo}/${packageNames}`, + username: owner + }; + const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + for (const version of versions.filter(isExpiredPrTag)) { + try { + await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); + console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); + } catch(e) { + console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); + console.error(e); + packageDeletionFailed = true; + } } } diff --git a/.github/workflows/continuous-integration-secure.yml b/.github/workflows/continuous-integration-secure.yml index bd7bf654ec..be56029386 100644 --- a/.github/workflows/continuous-integration-secure.yml +++ b/.github/workflows/continuous-integration-secure.yml @@ -46,7 +46,7 @@ jobs: uses: actions/github-script@v7 with: script: | - const environment = process.env.PR_NUMBER ? `preview-pr${process.env.PR_NUMBER}` : 'main'; + const environment = process.env.PR_NUMBER ? `pr${process.env.PR_NUMBER}` : 'main'; const payload = { owner: context.repo.owner, repo: context.repo.repo, environment }; const { data: deployment } = await github.rest.repos.createDeployment({ ...payload, diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d0e48d2603..9050f779fc 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -7,6 +7,9 @@ concurrency: permissions: read-all +env: + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + jobs: lint: runs-on: ubuntu-latest @@ -40,11 +43,6 @@ jobs: test: runs-on: ubuntu-latest needs: lint - services: - visual-regression: - image: ghcr.io/${{ github.repository }}/visual-regression:baseline - ports: - - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -160,14 +158,41 @@ jobs: onlyChanged: true externals: '**/components/core/styles/**/*.scss' + visual-regression: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + needs: test + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8050:8080 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run visual regression baseline generation + run: yarn test:visual-regression + env: + NODE_ENV: production + - name: Store visual regression output + uses: actions/upload-artifact@v4 + with: + name: visual-regression-screenshots + path: dist/screenshots-artifact/ + visual-regression-baseline: runs-on: ubuntu-latest permissions: packages: write if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: test - env: - IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression services: visual-regression: image: ghcr.io/${{ github.repository }}/visual-regression:baseline diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 9b8f89edd4..9d800563ab 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -66,7 +66,7 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest . env: @@ -116,7 +116,7 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' run: docker build --tag $IMAGE_REPO_STORYBOOK:dev . env: diff --git a/Dockerfile b/Dockerfile index 2220166b4f..bd0cd09679 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM ghcr.io/nginxinc/nginx-unprivileged:stable +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + # Copy nginx configuration COPY ./.github/default.conf /etc/nginx/conf.d/default.conf diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts index c675869609..abe7497126 100644 --- a/src/components/button/button/button.snapshot.spec.ts +++ b/src/components/button/button/button.snapshot.spec.ts @@ -4,6 +4,7 @@ import { describeEach, describeViewports, fixture, + isVisualRegressionRun, visualRegressionSnapshot, } from '../../core/testing/private.js'; import type { SbbButtonSize } from '../common.js'; @@ -11,38 +12,40 @@ import type { SbbButtonSize } from '../common.js'; import './button.js'; describe(`sbb-button`, () => { - describe('visual-regression', () => { - const cases = { - size: ['s', 'm', 'l'] as SbbButtonSize[], - disabled: [false, true], - negative: [false, true], - iconName: [undefined, 'arrow-right-small'], - }; + if (isVisualRegressionRun()) { + describe('visual-regression', () => { + const cases = { + size: ['s', 'm', 'l'] as SbbButtonSize[], + disabled: [false, true], + negative: [false, true], + iconName: [undefined, 'arrow-right-small'], + }; - describeViewports(() => { - describeEach(cases, ({ size, disabled, negative, iconName }) => { - let root: HTMLElement; - beforeEach(async () => { - root = await fixture(html` -
- Button { + describeEach(cases, ({ size, disabled, negative, iconName }) => { + let root: HTMLElement; + beforeEach(async () => { + root = await fixture(html` +
-
- `); - }); + Button +
+ `); + }); - visualRegressionSnapshot(() => root); + visualRegressionSnapshot(() => root); + }); }); }); - }); + } }); diff --git a/src/components/core/testing.ts b/src/components/core/testing.ts index 032a69bed9..428ae3f973 100644 --- a/src/components/core/testing.ts +++ b/src/components/core/testing.ts @@ -1,6 +1,5 @@ export * from './testing/event-spy.js'; export * from './testing/mocha-extensions.js'; -export * from './testing/platform.js'; export * from './testing/scroll.js'; export * from './testing/wait-for-condition.js'; export * from './testing/wait-for-render.js'; diff --git a/src/components/core/testing/private.ts b/src/components/core/testing/private.ts index 45ecfb614c..ba1753f948 100644 --- a/src/components/core/testing/private.ts +++ b/src/components/core/testing/private.ts @@ -4,5 +4,6 @@ export * from './private/describe-viewports.js'; export * from './private/dispatch-events.js'; export * from './private/event-objects.js'; export * from './private/fixture.js'; +export * from './private/platform.js'; export * from './private/type-in-element.js'; export * from './private/visual-regression-snapshot.js'; diff --git a/src/components/core/testing/private/fixture.ts b/src/components/core/testing/private/fixture.ts index 7e3788ee48..db042554d3 100644 --- a/src/components/core/testing/private/fixture.ts +++ b/src/components/core/testing/private/fixture.ts @@ -1,8 +1,9 @@ import type { TemplateResult } from 'lit'; -import { isHydratedSsr, isNonHydratedSsr } from '../platform.js'; import { waitForLitRender } from '../wait-for-render.js'; +import { isHydratedSsr, isNonHydratedSsr } from './platform.js'; + // Copied from @lit-labs/testing/lib/fixtures/fixture-options.d.ts interface FixtureOptions { /** diff --git a/src/components/core/testing/platform.ts b/src/components/core/testing/private/platform.ts similarity index 82% rename from src/components/core/testing/platform.ts rename to src/components/core/testing/private/platform.ts index eb23176b41..28a308ec16 100644 --- a/src/components/core/testing/platform.ts +++ b/src/components/core/testing/private/platform.ts @@ -32,3 +32,10 @@ export const isNonHydratedSsr = (): boolean => * Returns true, if this is run in an SSR test group. */ export const isSsr = (): boolean => isHydratedSsr() || isNonHydratedSsr(); + +/** + * This is a custom implementation. + * Returns true, if this is run in the visual regression test group. + */ +export const isVisualRegressionRun = (): boolean => + !isServer && (globalThis as any).testGroup === 'visual-regression'; diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index 5f90f96f6d..a795c33515 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -2,7 +2,7 @@ import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -import { isHydratedSsr } from './platform.js'; +import { isHydratedSsr } from './private.js'; function setupIconConfig(): void { const testNamespaces = ['default', 'picto']; diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index f1b9ecc96b..4d49ebbcc7 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -1,18 +1,13 @@ FROM ghcr.io/nginxinc/nginx-unprivileged:stable +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# Copy screenshots +COPY ./dist/screenshots/Chromium/baseline/ /usr/share/nginx/html/Chromium/baseline/ +COPY ./dist/screenshots/Firefox/baseline/ /usr/share/nginx/html/Firefox/baseline/ +COPY ./dist/screenshots/Webkit/baseline/ /usr/share/nginx/html/Webkit/baseline/ + COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf -COPY ./dist/screenshots /usr/share/nginx/html +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh -USER root -# We calculate the sha1 hashes of the png files in order to use it as etag values. -# This allows us to use HTTP caching mechanisms, which should reduce network traffic -# for the baseline comparison. -RUN cd /usr/share/nginx/html && \ - find ./*/ -type f ! -iname "*.png" -delete && \ - find ./*/ -type d -name failed -prune -exec rm -rf {} \; && \ - find ./*/ -type d -name .cache -prune -exec rm -rf {} \; && \ - echo 'map_hash_bucket_size 16384;' > /etc/nginx/conf.d/1etags.conf && \ - echo 'map $uri $pngetag {' >> /etc/nginx/conf.d/1etags.conf && \ - find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha1sum $1 | cut -d " " -f 1);"' _ {} \; >> /etc/nginx/conf.d/1etags.conf && \ - echo '}' >> /etc/nginx/conf.d/1etags.conf -USER $UID +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/diff-app.Dockerfile b/tools/visual-regression-testing/diff-app.Dockerfile new file mode 100644 index 0000000000..1bd7ea0d99 --- /dev/null +++ b/tools/visual-regression-testing/diff-app.Dockerfile @@ -0,0 +1,11 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# This is currently the same config file as for baseline. Separate into separate configs, if this changes. +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY ./dist/diff-app /usr/share/nginx/html + +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh + +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts index 75d4501e46..981c615246 100644 --- a/tools/visual-regression-testing/diff-app/vite.config.ts +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -1,3 +1,4 @@ +import { createHash } from 'crypto'; import { existsSync, readdirSync, readFileSync } from 'fs'; import { relative } from 'path'; @@ -41,86 +42,101 @@ function prepareScreenshots(): PluginOption { }, load(id) { if (id === resolvedVirtualModuleId) { - const browsers = readdirSync(screenshotsDir, { withFileTypes: true }) - .filter((d) => d.name !== '.cache') - .map((d) => d.name); - - const screenshotsMeta = browsers - .filter((browserName) => existsSync(new URL(`./${browserName}/failed/`, screenshotsDir))) - .flatMap((browserName) => { - const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); - - return readdirSync(failedDir, { - withFileTypes: true, - }) - .filter((d) => !d.name.endsWith('-diff.png')) - .map((d) => { - const failedFilePath = new URL(`./${d.name}`, failedDir); - const diffFilePath = new URL( - `./${d.name.replace(/.png$/, '-diff.png')}`, - failedDir, - ); - const baselineFilePath = new URL( - `./${browserName}/baseline/${d.name}`, - screenshotsDir, - ); - const baselineCacheFilePath = new URL( - `./.cache/${browserName}/baseline/${d.name}`, - screenshotsDir, - ); - - const isNew = !existsSync(diffFilePath); - - const assetsScreenshots = 'assets/screenshots/'; - const failedRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, failedFilePath.pathname); - const diffRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); - const baselineRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, baselineFilePath.pathname); - - if (viteConfig.command !== 'serve') { - this.emitFile({ - type: 'asset', - fileName: failedRelativeFileName, - source: readFileSync(failedFilePath), + const failedScreenshotsHash = createHash('sha256'); + const screenshotsFailures = existsSync(screenshotsDir) + ? readdirSync(screenshotsDir, { withFileTypes: true }) + .map((d) => d.name) + .filter((browserName) => + existsSync(new URL(`./${browserName}/failed/`, screenshotsDir)), + ) + .flatMap((browserName) => { + const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); + + return readdirSync(failedDir, { + withFileTypes: true, + }) + .filter((d) => !d.name.endsWith('-diff.png')) + .map((d) => { + const failedFilePath = new URL(`./${d.name}`, failedDir); + const diffFilePath = new URL( + `./${d.name.replace(/.png$/, '-diff.png')}`, + failedDir, + ); + const baselineFilePath = new URL( + `./${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + + const isNew = !existsSync(diffFilePath); + + const assetsScreenshots = 'assets/screenshots/'; + const failedRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, failedFilePath.pathname); + const diffRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); + const baselineRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, baselineFilePath.pathname); + + if (viteConfig.command !== 'serve') { + const failedFileContent = readFileSync(failedFilePath); + // We only add the failed screenshot hashes, as the baseline and comparison (*-diff.png) + // are not relevant to detect whether it is a new difference. + failedScreenshotsHash.update(failedFileContent); + this.emitFile({ + type: 'asset', + fileName: failedRelativeFileName, + source: failedFileContent, + }); + + if (!isNew) { + this.emitFile({ + type: 'asset', + fileName: diffRelativeFileName, + source: readFileSync(diffFilePath), + }); + + this.emitFile({ + type: 'asset', + fileName: baselineRelativeFileName, + source: readFileSync(baselineFilePath), + }); + } + } + + return { + browserName, + name: d.name, + failedFile: failedRelativeFileName, + diffFile: diffRelativeFileName, + baselineFile: baselineRelativeFileName, + isNew, + }; }); - - if (!isNew) { - this.emitFile({ - type: 'asset', - fileName: diffRelativeFileName, - source: readFileSync(diffFilePath), - }); - - this.emitFile({ - type: 'asset', - fileName: baselineRelativeFileName, - source: readFileSync( - existsSync(baselineFilePath) ? baselineFilePath : baselineCacheFilePath, - ), - }); - } - } - - return { - browserName, - name: d.name, - failedFile: failedRelativeFileName, - diffFile: diffRelativeFileName, - baselineFile: baselineRelativeFileName, - isNew, - }; - }); - }) - .reduce( - (current, next) => - current.set( - next.name, - current.has(next.name) ? current.get(next.name)!.concat(next) : [next], - ), - new Map(), - ); + }) + : []; + + const screenshotsMeta = screenshotsFailures.reduce( + (current, next) => + current.set( + next.name, + current.has(next.name) ? current.get(next.name)!.concat(next) : [next], + ), + new Map(), + ); + + if (viteConfig.command !== 'serve') { + this.emitFile({ + type: 'asset', + fileName: 'diff.json', + source: JSON.stringify({ + changedAmount: screenshotsFailures.filter((f) => !f.isNew).length, + newAmount: screenshotsFailures.filter((f) => f.isNew).length, + hash: failedScreenshotsHash.digest('hex'), + }), + }); + } return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; } diff --git a/tools/visual-regression-testing/etag-map-generation.sh b/tools/visual-regression-testing/etag-map-generation.sh new file mode 100755 index 0000000000..5f88502c31 --- /dev/null +++ b/tools/visual-regression-testing/etag-map-generation.sh @@ -0,0 +1,10 @@ + +# We calculate the sha1 hashes of the png files in order to use it as etag values. +# This allows us to use HTTP caching mechanisms, which should reduce network traffic +# for the baseline comparison. +cd /usr/share/nginx/html + +echo 'map_hash_bucket_size 32768;' +echo 'map $uri $pngetag {' +find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha256sum $1 | cut -d " " -f 1);"' _ {} \; +echo '}' \ No newline at end of file diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index 8f493c5f55..13ebd9665b 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -2,14 +2,61 @@ // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; -import { mkdirSync } from 'fs'; +import { cpSync, existsSync, mkdirSync } from 'fs'; import { platform } from 'os'; import { startTestRunner } from '@web/test-runner'; +import * as glob from 'glob'; -const args = process.argv.slice(2); -if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { - startTestRunner(); +if (process.env.GITHUB_ACTIONS) { + // When being run on GitHub Actions we have two use cases. + // Baseline generation for which our expectation is to not fail. + // Diff generation if any test fails, for which we copy only the necessary + // files to dist/screenshots-artifact/ to reduce artifact size. + const runner = await startTestRunner({ autoExitProcess: false }); + if (!runner) { + throw new Error( + `Unexpected state. Test runner not available. Check tools/visual-regression-testing/exec.ts execution.`, + ); + } + await new Promise((r) => runner.on('stopped', r)); + + const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); + const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); + mkdirSync(artifactDir, { recursive: true }); + + if (runner.passed) { + // Tests passed. Do nothing. + process.exit(0); + } + + // When visual regression tests have failed, we only want to pack the relevant screenshots + // into the artifact transfered to the secure workflow, as uploading and downloading the full + // baseline would take far longer. + // Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will + // be moved to /dist/screenshots in the secure workflow. + const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); + for (const failedDir of failedDirs) { + cpSync(new URL(`./${failedDir}`, screenshotDir), new URL(`./${failedDir}`, artifactDir), { + force: true, + recursive: true, + }); + } + + const failedFiles = glob + .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) + .map((p) => p.replace('/failed/', '/baseline/')); + for (const failedFile of failedFiles) { + const baselineFile = new URL(`./${failedFile}`, screenshotDir); + if (existsSync(baselineFile)) { + cpSync(baselineFile, new URL(`./${failedFile}`, artifactDir), { + force: true, + recursive: true, + }); + } + } +} else if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { + await startTestRunner(); } else { function executableIsAvailable(name: string): string | null { try { @@ -26,6 +73,7 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { process.exit(1); } + const args = process.argv.slice(2); const cwd = new URL('../../', import.meta.url); const tag = 'lyne-vrt'; const execOptions: ExecSyncOptionsWithStringEncoding = { @@ -33,6 +81,10 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { stdio: 'inherit', cwd, }; + const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); execSync( `${containerCmd} build ` + '--file=tools/visual-regression-testing/testing.Dockerfile ' + @@ -43,6 +95,7 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + + `--env=BRANCH="${branch}"` + `--volume=./dist/screenshots:/dist/screenshots ` + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + tag, diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 9030e4d482..e127d69852 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -1,7 +1,13 @@ -import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; -import { dirname, extname, join } from 'path'; +import { execSync } from 'child_process'; +import { createHash } from 'crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, extname } from 'path'; -const baselineUrl = process.env.CI +const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); +const baselineUrl = process.env.GITHUB_ACTIONS ? 'http://localhost:8050/' : 'https://lyne-visual-regression-baseline.app.sbb.ch/'; @@ -10,51 +16,48 @@ export const visualRegressionConfig = (update) => ({ update, baseDir: 'dist/screenshots', - async getBaseline({ filePath, baseDir, name }) { - if (existsSync(filePath)) { - return readFileSync(filePath); - } - - const cacheFile = join(baseDir, '.cache', name + extname(filePath)); - const cacheFileDetails = cacheFile + '.json'; - mkdirSync(dirname(cacheFile), { recursive: true }); + async getBaseline({ filePath, name }) { const baselineFileUrl = baselineUrl + name + extname(filePath); - const downloadFile = async () => { - try { - const response = await fetch(baselineFileUrl); - - if (response.ok) { - writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); - writeFileSync( - cacheFileDetails, - JSON.stringify({ etag: response.headers.get('etag') }, null, 2), - 'utf8', - ); - - return readFileSync(cacheFile); - } - } catch { - /* empty */ - } - }; - - if (existsSync(cacheFileDetails)) { - const details = JSON.parse(readFileSync(cacheFileDetails)); + const infoFilePath = filePath + '.json'; + const info = existsSync(infoFilePath) ? JSON.parse(readFileSync(infoFilePath, 'utf8')) : {}; + if (existsSync(filePath) && info.branch === branch) { + return readFileSync(filePath); + } else if (existsSync(filePath) && info.etag) { const response = await fetch(baselineFileUrl, { method: 'HEAD', - headers: { 'if-none-match': details.etag }, + headers: { 'if-none-match': info.etag }, }); - if (response.status === 200) { - return await downloadFile(); + + if (response.status === 304) { + return readFileSync(filePath); } else if (response.status === 404) { - [cacheFile, cacheFileDetails].forEach(unlinkSync); - } else if (response.status === 304) { - return readFileSync(cacheFile); - } else { - console.error(`Unexpected response from baseline service: ${response.status} (${name})`); + return undefined; } - } else { - return await downloadFile(); } + + // If the image address is not reachable, fetch throws, so we wrap it in try/catch. + try { + const response = await fetch(baselineFileUrl); + if (response.ok) { + const etag = response.headers.get('etag'); + const buffer = Buffer.from(await response.arrayBuffer()); + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, buffer); + writeFileSync(infoFilePath, JSON.stringify({ etag }, null, 2), 'utf8'); + + return buffer; + } + } catch (e) { + return undefined; + } + }, + saveBaseline({ filePath, content }) { + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, content); + writeFileSync( + filePath + '.json', + JSON.stringify({ branch, etag: createHash('sha256').update(content).digest('hex') }), + 'utf8', + ); }, }); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 0bdb2a5195..d257457f6e 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -22,6 +22,11 @@ const webkit = process.argv.includes('--webkit'); const concurrency = process.argv.includes('--parallel') ? {} : { concurrency: 1 }; const updateBaseImages = process.argv.includes('--update-visual-baseline') || process.argv.includes('--uv'); +const visualRegressionRun = + process.argv.includes('--group=visual-regression') || + (process.argv.includes('--group') + ? process.argv[process.argv.indexOf('--group') + 1] === 'visual-regression' + : false); const stylesCompiler = new sass.initCompiler(); const renderStyles = () => @@ -86,15 +91,21 @@ const suppressedLogs = [ '[vite] connecting...', ]; +const groups = [ + // Disable ssr tests until stabilized. + // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, + // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, +]; + +// The visual regression test group is only added when expicitely set, as the tests are very expensive. +if (visualRegressionRun) { + groups.push({ name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }); +} + /** @type {import('@web/test-runner').TestRunnerConfig} */ export default { files: ['src/**/*.{e2e,spec,!snapshot.spec}.ts'], - groups: [ - // Disable ssr tests until stabilized. - // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, - // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, - { name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }, - ], + groups, nodeResolve: true, concurrency: resolveConcurrency(), reporters: From 0ac126101a759c36ef70ab4b64f84aba3fb6a2ec Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 26 Apr 2024 17:36:11 +0200 Subject: [PATCH 08/48] fix: lint --- src/components/accordion/accordion.e2e.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/accordion/accordion.e2e.ts b/src/components/accordion/accordion.e2e.ts index a01e2ff39c..be20fbb172 100644 --- a/src/components/accordion/accordion.e2e.ts +++ b/src/components/accordion/accordion.e2e.ts @@ -2,8 +2,8 @@ import { assert, expect } from '@open-wc/testing'; import { nothing } from 'lit'; import { html } from 'lit/static-html.js'; -import { fixture } from '../core/testing/private.js'; -import { waitForCondition, waitForLitRender, EventSpy, isSsr } from '../core/testing.js'; +import { fixture, isSsr } from '../core/testing/private.js'; +import { waitForCondition, waitForLitRender, EventSpy } from '../core/testing.js'; import { SbbExpansionPanelElement, type SbbExpansionPanelHeaderElement, From 26e845b25cb142183a295fc33c5d185b93febd73 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Wed, 1 May 2024 17:01:22 +0200 Subject: [PATCH 09/48] fix: fix screenshot sizes --- .../testing/private/describe-viewports.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts index de764fe737..6c538dfab1 100644 --- a/src/components/core/testing/private/describe-viewports.ts +++ b/src/components/core/testing/private/describe-viewports.ts @@ -1,29 +1,29 @@ import { - SbbBreakpointLargeMax, - SbbBreakpointMediumMax, - SbbBreakpointMicroMax, - SbbBreakpointSmallMax, - SbbBreakpointUltraMax, - SbbBreakpointWideMax, - SbbBreakpointZeroMax, + SbbBreakpointLargeMin, + SbbBreakpointMediumMin, + SbbBreakpointMicroMin, + SbbBreakpointSmallMin, + SbbBreakpointUltraMin, + SbbBreakpointWideMin, + SbbBreakpointZeroMin, } from '@sbb-esta/lyne-design-tokens'; import { setViewport } from '@web/test-runner-commands'; const viewportSizes = { - zero: SbbBreakpointZeroMax, - micro: SbbBreakpointMicroMax, - small: SbbBreakpointSmallMax, - medium: SbbBreakpointMediumMax, - large: SbbBreakpointLargeMax, - wide: SbbBreakpointWideMax, - ultra: SbbBreakpointUltraMax, + zero: SbbBreakpointZeroMin, + micro: SbbBreakpointMicroMin, + small: SbbBreakpointSmallMin, + medium: SbbBreakpointMediumMin, + large: SbbBreakpointLargeMin, + wide: SbbBreakpointWideMin, + ultra: SbbBreakpointUltraMin, }; -export function describeViewports(fn: (this: Mocha.Suite) => void): void { +export function describeViewports(fn: (this: Mocha.Suite) => void, viewportHeight = 400): void { for (const [size, value] of Object.entries(viewportSizes)) { describe(`viewport=${size}`, function () { before(async () => { - await setViewport({ width: value, height: 400 }); + await setViewport({ width: value, height: viewportHeight }); }); fn.call(this); From 3cfbde2955ed168459589982d42655cb9e2094ee Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 11:47:28 +0200 Subject: [PATCH 10/48] feat: diff-app first version --- eslint.config.js | 7 + package.json | 1 + .../diff-app/index.html | 3 +- .../src/components/overview/overview.scss | 9 + .../src/components/overview/overview.ts | 72 ++++++ .../fullscreen-diff/fullscreen-diff.scss | 21 ++ .../fullscreen-diff/fullscreen-diff.ts | 58 +++++ .../test-case/image-diff/image-diff.scss | 69 ++++++ .../test-case/image-diff/image-diff.ts | 147 ++++++++++++ .../test-case-filter/test-case-filter.scss | 14 ++ .../test-case-filter/test-case-filter.ts | 84 +++++++ .../src/components/test-case/test-case.scss | 49 ++++ .../src/components/test-case/test-case.ts | 123 ++++++++++ .../diff-app/src/main.ts | 10 +- .../diff-app/src/routes.ts | 15 ++ .../diff-app/src/screenshots.ts | 218 ++++++++++++++++++ .../diff-app/src/vite-env.d.ts | 8 +- .../diff-app/vite.config.ts | 2 +- yarn.lock | 25 ++ 19 files changed, 927 insertions(+), 8 deletions(-) create mode 100644 tools/visual-regression-testing/diff-app/src/components/overview/overview.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/overview/overview.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts create mode 100644 tools/visual-regression-testing/diff-app/src/routes.ts create mode 100644 tools/visual-regression-testing/diff-app/src/screenshots.ts diff --git a/eslint.config.js b/eslint.config.js index 673978b295..1daf442a54 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -47,6 +47,13 @@ export default [ 'plugin:import-x/typescript', ), eslintPluginLyne.default.configs.recommended, + { + files: ['tools/visual-regression-testing/diff-app/**/*.ts'], + rules: { + 'lyne/custom-element-class-name-rule': 'off', + 'import-x/namespace': 'off', + }, + }, { files: ['**/*.ts'], rules: { diff --git a/package.json b/package.json index ce8cbbfa7e..94da49411d 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "@types/react-dom": "18.3.0", "@typescript-eslint/eslint-plugin": "7.8.0", "@typescript-eslint/parser": "7.8.0", + "@vaadin/router": "1.7.5", "@web/test-runner": "0.18.1", "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", diff --git a/tools/visual-regression-testing/diff-app/index.html b/tools/visual-regression-testing/diff-app/index.html index b6bc0adf39..54182ed5c0 100644 --- a/tools/visual-regression-testing/diff-app/index.html +++ b/tools/visual-regression-testing/diff-app/index.html @@ -2,6 +2,7 @@ Visual Regression Tests Comparison + - Hi +
diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.scss b/tools/visual-regression-testing/diff-app/src/components/overview/overview.scss new file mode 100644 index 0000000000..0458d025a8 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.scss @@ -0,0 +1,9 @@ +@use '../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +.app-overview { + display: flex; + gap: 1rem; + flex-direction: column; +} diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts new file mode 100644 index 0000000000..8e1a9032e7 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts @@ -0,0 +1,72 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement } from 'lit/decorators.js'; + +import { screenshotService } from '../../screenshots.js'; + +import style from './overview.scss?lit&inline'; + +import '../../../../../../src/components/accordion.js'; +import '../../../../../../src/components/button/secondary-button-link.js'; +import '../../../../../../src/components/card.js'; +import '../../../../../../src/components/container.js'; +import '../../../../../../src/components/expansion-panel.js'; +import '../../../../../../src/components/link-list.js'; +import '../../../../../../src/components/link/block-link.js'; +import '../../../../../../src/components/title.js'; + +/** + * Overview over all failed or new tests + */ +@customElement('app-overview') +export class Overview extends LitElement { + public static override styles: CSSResultGroup = style; + + public override render(): TemplateResult { + return html` + + Lyne visual regression comparison +
+ + ${screenshotService.screenshots.stats} + + Start comparing + + + + ${screenshotService.screenshots.components.map( + (screenshotComponent) => html` + + + ${screenshotComponent.name} (${screenshotComponent.stats}) + + + + ${screenshotComponent.testCases.map( + (entry) => + html` + ${entry.name} + `, + )} + + + + `, + )} + +
+ + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-overview': Overview; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss new file mode 100644 index 0000000000..3af3e700ae --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss @@ -0,0 +1,21 @@ +@use '../../../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +:host { + display: block; +} + +.app-labels { + text-transform: capitalize; +} + +.app-radio-button-group { + margin-block-end: 0.5rem; +} + +.app-scroll-container { + @include sbb.scrollbar; + + overflow: auto; +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts new file mode 100644 index 0000000000..5e6ef0208a --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts @@ -0,0 +1,58 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup, nothing } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; + +import type { SbbRadioButtonGroupElement } from '../../../../../../../../src/components/radio-button/radio-button-group/radio-button-group.js'; +import type { ScreenshotFailedFiles } from '../../../../screenshots.js'; + +import style from './fullscreen-diff.scss?lit&inline'; + +import '../../../../../../../../src/components/chip.js'; +import '../../../../../../../../src/components/radio-button.js'; + +/** + * Displays two images in fullscreen to overlay them. + */ +@customElement('app-fullscreen-diff') +export class FullscreenDiff extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public failedFile?: ScreenshotFailedFiles; + + @property() public selectedFile: 'baselineFile' | 'failedFile' | 'diffFile' = 'failedFile'; + + public override render(): TemplateResult { + if (!this.failedFile) { + return html``; + } + return html`
+ ${this.failedFile.browserName} + ${this.failedFile.viewport} +
+ + (this.selectedFile = (event.target as SbbRadioButtonGroupElement).value)} + > + ${!this.failedFile.isNew + ? html`Baseline` + : nothing} + ${this.failedFile.isNew ? 'New' : 'Failed'} + ${this.failedFile.diffFile + ? html`Diff` + : nothing} + +
+ +
`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-fullscreen-diff': FullscreenDiff; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss new file mode 100644 index 0000000000..9359cd30ff --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss @@ -0,0 +1,69 @@ +@use '../../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +:host { + display: block; +} + +.app-container { + display: flex; + flex-direction: column; + gap: var(--sbb-spacing-fixed-1x); +} + +.app-info-bar { + display: flex; + justify-content: space-between; +} + +.app-labels { + text-transform: capitalize; +} + +.app-diff-toggle { + white-space: nowrap; + align-self: center; +} + +.app-image-container { + width: 100%; + display: flex; + align-items: start; + justify-content: stretch; + flex-direction: column; + gap: var(--sbb-spacing-fixed-1x); + + @include sbb.mq($from: small) { + flex-direction: row; + } +} + +.app-image-baseline, +.app-image-failed { + display: flex; + flex: 1; + width: 100%; + background-color: var(--sbb-color-white); +} + +.app-image { + max-width: 100%; +} + +.app-new-test-case-info { + margin: 0.5rem; +} + +.app-image-button { + @include sbb.button-reset; + + display: block; + width: 100%; + text-align: left; + cursor: pointer; + + &:focus-visible { + @include sbb.focus-outline; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts new file mode 100644 index 0000000000..cb0797fa53 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts @@ -0,0 +1,147 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup, nothing } from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; + +import { SbbOverlayElement } from '../../../../../../../src/components/overlay/overlay.js'; +import type { SbbToggleCheckElement } from '../../../../../../../src/components/toggle-check/toggle-check.js'; +import type { ScreenshotFailedFiles } from '../../../screenshots.js'; + +import style from './image-diff.scss?lit&inline'; + +import '../../../../../../../src/components/chip.js'; +import '../../../../../../../src/components/status.js'; +import '../../../../../../../src/components/overlay.js'; +import '../../../../../../../src/components/toggle-check.js'; + +import './fullscreen-diff/fullscreen-diff.js'; + +const getImageDimension = (img: HTMLImageElement): string => + `${img.naturalWidth}x${img.naturalHeight}px`; + +/** + * Displays two images to compare them. + */ +@customElement('app-image-diff') +export class ImageDiff extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public failedFile?: ScreenshotFailedFiles; + + @state() private _baselineDimension?: string; + @state() private _failedDimension?: string; + + @state() private _showDiff: boolean = true; + + private _toggleDiff(event: Event): void { + this._showDiff = (event.target as SbbToggleCheckElement).checked; + } + + private _setFailedImageDimension(event: Event): void { + this._failedDimension = getImageDimension(event.target as HTMLImageElement); + } + + private _setBaselineImageDimension(event: Event): void { + this._baselineDimension = getImageDimension(event.target as HTMLImageElement); + } + + /** + * To avoid blown up DOM, we create the overlay only when it's needed. + */ + private _showFullscreen(selectedFile: 'baselineFile' | 'failedFile' | 'diffFile'): void { + const sbbOverlayElement: SbbOverlayElement = document.createElement('sbb-overlay'); + const appFullscreenDiff = document.createElement('app-fullscreen-diff'); + + appFullscreenDiff.selectedFile = selectedFile; + appFullscreenDiff.failedFile = this.failedFile; + + sbbOverlayElement.appendChild(appFullscreenDiff); + document.body.appendChild(sbbOverlayElement); + sbbOverlayElement.addEventListener(SbbOverlayElement.events.didClose, () => { + document.body.removeChild(sbbOverlayElement); + }); + + sbbOverlayElement.open(); + } + + public override render(): TemplateResult { + if (!this.failedFile) { + return html``; + } + return html`
+
+
+ ${this.failedFile.browserName} + ${this.failedFile.viewport} + ${this._baselineDimension + ? html` + Baseline: ${this._baselineDimension} + ` + : nothing} + + ${this.failedFile.isNew ? 'New' : 'Failed'}: ${this._failedDimension} + +
+ ${!this.failedFile.isNew + ? html` + Show Diff + ` + : nothing} +
+
+
+ ${!this.failedFile.isNew + ? html`` + : html` + New test case + `} +
+
+ ${this._showDiff && !this.failedFile.isNew + ? html`` + : html``} +
+
+
`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-image-diff': ImageDiff; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss new file mode 100644 index 0000000000..ecae9f0162 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss @@ -0,0 +1,14 @@ +@use '../../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +.app-test-case-filter { + display: flex; + gap: 0.5rem 4rem; + flex-direction: column; + text-transform: capitalize; +} + +sbb-title { + margin: 0; +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts new file mode 100644 index 0000000000..86057eeb67 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts @@ -0,0 +1,84 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; + +import type { SbbTagElement } from '../../../../../../../src/components/tag/tag/tag.js'; +import { type ScreenshotTestCase } from '../../../screenshots.js'; +import '../../../../../../../src/components/title.js'; +import '../../../../../../../src/components/tag.js'; + +import style from './test-case-filter.scss?lit&inline'; + +/** + * Shows filter for viewports and browsers + */ +@customElement('app-test-case-filter') +export class TestCaseFilter extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public testCase?: ScreenshotTestCase; + + private _handleViewportChange(event: CustomEvent): void { + this.dispatchEvent( + new CustomEvent('viewportFilterChange', { + bubbles: true, + composed: true, + detail: (event.target as SbbTagElement).value, + }), + ); + } + + private _handleBrowserChange(event: CustomEvent): void { + this.dispatchEvent( + new CustomEvent('browserFilterChange', { + bubbles: true, + composed: true, + detail: (event.target as SbbTagElement).value, + }), + ); + } + + public override render(): TemplateResult { + return html` +
+
+ Viewports + + + All + + ${this.testCase?.viewports?.map( + (viewport) => html` + + ${viewport.name} + + `, + )} + +
+
+ Browsers + + + All + + ${this.testCase?.availableBrowserNames?.map( + (browserName) => html`${browserName}`, + )} + +
+
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-test-case-filter': TestCaseFilter; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss new file mode 100644 index 0000000000..cd90c65dcd --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss @@ -0,0 +1,49 @@ +@use '../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +sbb-chip { + width: fit-content; +} + +sbb-title { + margin: 0; +} + +.app-file-name-box { + display: flex; + flex-direction: column; +} + +.app-file-name-ellipsis { + display: block; + @include sbb.ellipsis; +} + +.app-navigation-block { + display: flex; + gap: 0.5rem; + align-items: center; +} + +.app-image-diffs { + display: flex; + flex-direction: column; + gap: 2rem; + padding-block: 1rem; +} + +.app-testcase { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.app-progress { + position: absolute; + inset-block-start: 0; + inset-inline-start: 0; + height: var(--sbb-border-radius-2x); + background-color: var(--sbb-color-black); + width: calc(var(--app-progress) * 100%); +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts new file mode 100644 index 0000000000..b665bdaf43 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts @@ -0,0 +1,123 @@ +import type { BeforeEnterObserver, RouterLocation } from '@vaadin/router'; +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; + +import { screenshotService, type ScreenshotTestCase } from '../../screenshots.js'; + +import '../../../../../../src/components/button/secondary-button-link.js'; +import '../../../../../../src/components/chip.js'; +import '../../../../../../src/components/container.js'; +import '../../../../../../src/components/header.js'; +import '../../../../../../src/components/notification.js'; +import '../../../../../../src/components/title.js'; + +import style from './test-case.scss?lit&inline'; + +import './test-case-filter/test-case-filter.js'; +import './image-diff/image-diff.js'; + +interface Filter { + viewport?: string; + browser?: string; +} + +/** + * Displays a test case with its images. + * Provides filtering functions. + */ +@customElement('app-test-case') +export class TestCase extends LitElement implements BeforeEnterObserver { + public static override styles: CSSResultGroup = style; + + @state() private _testCaseName?: string; + @state() private _componentName?: string; + @state() private _testCase?: ScreenshotTestCase; + @state() private _filter: Filter = {}; + + // Called by router + public onBeforeEnter(location: RouterLocation): void { + this._testCaseName = location.params.testcase as string; + this._componentName = location.params.component as string; + + this._testCase = screenshotService.setCurrentTestCase(this._componentName, this._testCaseName); + } + + private _viewportFilterChanged(event: CustomEvent): void { + this._filter = { + ...this._filter, + viewport: event.detail && event.detail !== 'all' ? event.detail : undefined, + }; + } + + private _browserFilterChanged(event: CustomEvent): void { + this._filter = { + ...this._filter, + browser: event.detail && event.detail !== 'all' ? event.detail : undefined, + }; + } + + public override render(): TemplateResult { + return html` + +
+
+ ${this._componentName} + + ${this._testCaseName} + +
+
+
+ Overview + + +
+ + ${this._testCase + ? html`
+ + + + +
+ ${this._testCase + ?.filter(this._filter.viewport, this._filter.browser) + .map( + (failedFile) => + html``, + )} +
+ +
` + : html` + + No screenshots found. Please check component and test case name. + + `} + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-test-case': TestCase; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts index efabec6e06..3cf7b340e6 100644 --- a/tools/visual-regression-testing/diff-app/src/main.ts +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -1,8 +1,8 @@ -import '../../../../src/components/title.js'; +import { Router } from '@vaadin/router'; -import '../../../../src/components/core/styles/global.scss'; +import { routes } from './routes.js'; -// eslint-disable-next-line import-x/no-unresolved -import { screenshots } from 'virtual:screenshots'; +import '../../../../src/components/core/styles/standard-theme.scss'; -console.log(screenshots); +export const router = new Router(document.querySelector('#outlet')); +router.setRoutes(routes); diff --git a/tools/visual-regression-testing/diff-app/src/routes.ts b/tools/visual-regression-testing/diff-app/src/routes.ts new file mode 100644 index 0000000000..5fa9bfe2fb --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/routes.ts @@ -0,0 +1,15 @@ +import type { Route } from '@vaadin/router'; + +import './components/overview/overview.js'; +import './components/test-case/test-case.js'; + +export const routes: Route[] = [ + { + path: '/', + component: 'app-overview', + }, + { + path: '/compare/:component/:testcase', + component: 'app-test-case', + }, +]; diff --git a/tools/visual-regression-testing/diff-app/src/screenshots.ts b/tools/visual-regression-testing/diff-app/src/screenshots.ts new file mode 100644 index 0000000000..71c49eb205 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/screenshots.ts @@ -0,0 +1,218 @@ +// eslint-disable-next-line import-x/no-unresolved +import { screenshotsRaw } from 'virtual:screenshots'; + +import type { FailedFiles } from '../vite.config.js'; + +const viewportOrder = ['zero', 'micro', 'small', 'medium', 'large', 'wide', 'ultra']; + +// TODO: discuss whether to include it in creation of screenshotsRaw +const extractHierarchicalMap = ( + screenshots: Record, +): Map>> => { + const map = new Map>>(); + + Object.entries(screenshots).forEach(([fileName, failedFiles]) => { + const component = fileName.match(/^(.*?)_/)![1]; + const name = fileName.match(/_viewport=.*?_(.*?).png$/)![1]; + const viewport = fileName.match(/viewport=(.*?)_/)![1]; + + if (!map.has(component)) { + map.set(component, new Map()); + } + + const componentsMap = map.get(component)!; + + if (!componentsMap.has(name)) { + componentsMap.set(name, new Map()); + } + + const testCaseMap = componentsMap.get(name)!; + + testCaseMap.set( + viewport, + failedFiles.map((failedFile) => ({ ...failedFile, viewport })), + ); + }); + return map; +}; + +export interface ScreenshotFailedFiles extends FailedFiles { + viewport: string; +} + +export class ScreenshotStatistics { + public static fromFailedFiles(failedFiles: ScreenshotFailedFiles[]): ScreenshotStatistics { + return failedFiles.reduce( + (current, next) => + current.sum(new ScreenshotStatistics(next.isNew ? 0 : 1, next.isNew ? 1 : 0)), + new ScreenshotStatistics(0, 0), + ); + } + + public static fromList(list: { stats: ScreenshotStatistics }[]): ScreenshotStatistics { + return list.reduce((current, next) => current.sum(next.stats), new ScreenshotStatistics(0, 0)); + } + + public constructor( + public readonly failedTests: number, + public readonly newTests: number, + ) {} + + public sum(other: ScreenshotStatistics): ScreenshotStatistics { + return new ScreenshotStatistics( + this.failedTests + other.failedTests, + this.newTests + other.newTests, + ); + } + + public toString(): string { + return `${this.failedTests} failed, ${this.newTests} new`; + } +} + +export class ScreenshotViewport { + public readonly stats: ScreenshotStatistics; + public readonly browserNames: string[]; + + public constructor( + public readonly name: string, + public readonly browsers: ScreenshotFailedFiles[], + ) { + this.stats = ScreenshotStatistics.fromFailedFiles(this.browsers); + + this.browserNames = this.browsers.map((browser) => browser.browserName); + } + + /** Compare by respecting defined viewport order. */ + public compare(other: ScreenshotViewport): number { + return viewportOrder.indexOf(this.name) - viewportOrder.indexOf(other.name); + } +} + +export class ScreenshotTestCase { + public readonly stats: ScreenshotStatistics; + public readonly availableBrowserNames: string[]; + public readonly path: string; + + public constructor( + public readonly component: string, + public readonly name: string, + public readonly viewports: ScreenshotViewport[], + ) { + this.stats = ScreenshotStatistics.fromList(this.viewports); + this.path = `${this.component}/${this.name}`; + + this.availableBrowserNames = Array.from( + this.viewports.reduce((current, next) => { + next.browserNames.forEach((browserName) => current.add(browserName)); + return current; + }, new Set()), + ); + } + + public filter(viewport?: string, browser?: string): ScreenshotFailedFiles[] { + return this.viewports + .filter((entry) => !viewport || entry.name === viewport) + .flatMap((entry) => + entry.browsers.filter((failedFiles) => !browser || failedFiles.browserName === browser), + ); + } +} + +export class ScreenshotComponent { + public readonly stats: ScreenshotStatistics; + + public constructor( + public readonly name: string, + public readonly testCases: ScreenshotTestCase[], + ) { + this.stats = ScreenshotStatistics.fromList(this.testCases); + } +} + +export class Screenshots { + public readonly components: ScreenshotComponent[]; + public readonly stats: ScreenshotStatistics; + public readonly testCaseCount: number; + public readonly flatTestCases: ScreenshotTestCase[]; + + public constructor(screenshots: Record) { + const flatTestCases: ScreenshotTestCase[] = []; + + // Convert hierarchical screenshot map to classes + this.components = Array.from(extractHierarchicalMap(screenshots).entries()).map( + ([componentName, testCases]) => + new ScreenshotComponent( + componentName, + Array.from(testCases.entries()).map(([testCase, viewports]) => { + const screenshotTestCase = new ScreenshotTestCase( + componentName, + testCase, + Array.from(viewports.entries()) + .map(([viewport, entries]) => new ScreenshotViewport(viewport, entries)) + .sort((a: ScreenshotViewport, b: ScreenshotViewport) => a.compare(b)), + ); + flatTestCases.push(screenshotTestCase); + return screenshotTestCase; + }), + ), + ); + + this.flatTestCases = flatTestCases; + this.testCaseCount = this.flatTestCases.length; + this.stats = ScreenshotStatistics.fromList(this.components); + } + + public indexOfTestCase(componentName: string, testCaseName: string): number { + return this.flatTestCases.findIndex( + (component) => component.component === componentName && component.name === testCaseName, + ); + } + + public getByTestCaseIndex(index: number): ScreenshotTestCase | undefined { + return this.flatTestCases[index]; + } +} + +export class ScreenshotService { + public readonly screenshots; + private _currentIndex: number = -1; + + public get progressFraction(): number { + return (this._currentIndex + 1) / this.screenshots.testCaseCount; + } + + public get current(): ScreenshotTestCase | undefined { + return this._current; + } + private _current?: ScreenshotTestCase; + + public constructor(screenshots: Screenshots) { + this.screenshots = screenshots; + this._currentIndex = 0; + this._current = screenshots.getByTestCaseIndex(this._currentIndex); + } + + public get next(): ScreenshotTestCase | undefined { + return this.screenshots.getByTestCaseIndex(this._currentIndex + 1); + } + + public get previous(): ScreenshotTestCase | undefined { + return this.screenshots.getByTestCaseIndex(this._currentIndex - 1); + } + + public setCurrentTestCase( + componentName: string, + testCaseName: string, + ): ScreenshotTestCase | undefined { + const testCaseIndex = this.screenshots.indexOfTestCase(componentName, testCaseName); + + this._currentIndex = testCaseIndex; + if (testCaseIndex >= 0) { + this._current = this.screenshots.getByTestCaseIndex(testCaseIndex); + return this._current; + } + } +} + +export const screenshotService = new ScreenshotService(new Screenshots(screenshotsRaw)); diff --git a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts index 8b67858995..d6c5e3c360 100644 --- a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts +++ b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts @@ -2,5 +2,11 @@ declare module 'virtual:screenshots' { // eslint-disable-next-line @typescript-eslint/consistent-type-imports - export const screenshots: Record; + export const screenshotsRaw: Record; +} + +declare module '*?lit&inline' { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const src: import('lit').CSSResultGroup; + export default src; } diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts index 981c615246..3cadf5763d 100644 --- a/tools/visual-regression-testing/diff-app/vite.config.ts +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -138,7 +138,7 @@ function prepareScreenshots(): PluginOption { }); } - return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; + return `export const screenshotsRaw = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; } }, configureServer(server) { diff --git a/yarn.lock b/yarn.lock index e4190e7b5e..848804e18e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3185,6 +3185,26 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== +"@vaadin/router@1.7.5": + version "1.7.5" + resolved "https://registry.yarnpkg.com/@vaadin/router/-/router-1.7.5.tgz#af4b8f05a86e2890ea875f9bca9b406d74f9bae4" + integrity sha512-uRN3vd1ihgd596bF/NMZqpgxau0nlvIc0/JDd1EwStFNbZID/xIVse5LXdQhIyUKLmSl4T0GeCQK505xerWX0w== + dependencies: + "@vaadin/vaadin-usage-statistics" "^2.1.0" + path-to-regexp "2.4.0" + +"@vaadin/vaadin-development-mode-detector@^2.0.0": + version "2.0.6" + resolved "https://registry.yarnpkg.com/@vaadin/vaadin-development-mode-detector/-/vaadin-development-mode-detector-2.0.6.tgz#2acb568975c18b7965e56ed765638dcf183d8dd1" + integrity sha512-N6a5nLT/ytEUlpPo+nvdCKIGoyNjPsj3rzPGvGYK8x9Ceg76OTe1xI/GtN71mRW9e2HUScR0kCNOkl1Z63YDjw== + +"@vaadin/vaadin-usage-statistics@^2.1.0": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@vaadin/vaadin-usage-statistics/-/vaadin-usage-statistics-2.1.2.tgz#a0a211b298e647742b60c6d4d1d8834381e3c77f" + integrity sha512-xKs1PvRfTXsG0eWWcImLXWjv7D+f1vfoIvovppv6pZ5QX8xgcxWUdNgERlOOdGt3CTuxQXukTBW3+Qfva+OXSg== + dependencies: + "@vaadin/vaadin-development-mode-detector" "^2.0.0" + "@vitest/expect@1.3.1": version "1.3.1" resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.3.1.tgz#d4c14b89c43a25fd400a6b941f51ba27fe0cb918" @@ -8920,6 +8940,11 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== +path-to-regexp@2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" + integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== + path-to-regexp@^6.2.1: version "6.2.2" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.2.tgz#324377a83e5049cbecadc5554d6a63a9a4866b36" From c851fd2a332851b8362f2b4fa77ba8b2932d9891 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 14:42:45 +0200 Subject: [PATCH 11/48] fix: a11y cleanup --- src/components/core/testing/private/a11y-tree-snapshot.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/core/testing/private/a11y-tree-snapshot.ts b/src/components/core/testing/private/a11y-tree-snapshot.ts index db975d0875..57e4c576ba 100644 --- a/src/components/core/testing/private/a11y-tree-snapshot.ts +++ b/src/components/core/testing/private/a11y-tree-snapshot.ts @@ -1,4 +1,4 @@ -import { aTimeout, expect, fixture } from '@open-wc/testing'; +import { aTimeout, expect, fixture, fixtureCleanup } from '@open-wc/testing'; import { a11ySnapshot } from '@web/test-runner-commands'; import type { TemplateResult } from 'lit'; import { html } from 'lit/static-html.js'; @@ -18,6 +18,7 @@ async function a11yTreeEqualSnapshot(): Promise { const htmlWrapper = await fixture(html`

${JSON.stringify(snapshot, null, 2)}

`); await expect(htmlWrapper).to.be.equalSnapshot(); + fixtureCleanup(); } /** From decbd57d5d20fefad8ec1380c1603ddd3843e092 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 15:37:16 +0200 Subject: [PATCH 12/48] fix: fix viewport --- src/components/core/testing/private/describe-viewports.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts index 6c538dfab1..09364e05dd 100644 --- a/src/components/core/testing/private/describe-viewports.ts +++ b/src/components/core/testing/private/describe-viewports.ts @@ -5,12 +5,11 @@ import { SbbBreakpointSmallMin, SbbBreakpointUltraMin, SbbBreakpointWideMin, - SbbBreakpointZeroMin, } from '@sbb-esta/lyne-design-tokens'; import { setViewport } from '@web/test-runner-commands'; const viewportSizes = { - zero: SbbBreakpointZeroMin, + zero: 320, micro: SbbBreakpointMicroMin, small: SbbBreakpointSmallMin, medium: SbbBreakpointMediumMin, From 2e546d8fedd93b323a4f163ddd74c49ae4fb16e9 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 15:59:16 +0200 Subject: [PATCH 13/48] fix: fix diff app image size --- .../test-case/image-diff/image-diff.scss | 6 ++- .../test-case/image-diff/image-diff.ts | 43 +++++++++---------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss index 9359cd30ff..bf876ea77d 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss @@ -58,7 +58,7 @@ .app-image-button { @include sbb.button-reset; - display: block; + display: flex; width: 100%; text-align: left; cursor: pointer; @@ -66,4 +66,8 @@ &:focus-visible { @include sbb.focus-outline; } + + &[hidden] { + display: none; + } } diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts index cb0797fa53..411bb43fcd 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts @@ -50,6 +50,7 @@ export class ImageDiff extends LitElement { const sbbOverlayElement: SbbOverlayElement = document.createElement('sbb-overlay'); const appFullscreenDiff = document.createElement('app-fullscreen-diff'); + sbbOverlayElement.expanded = true; appFullscreenDiff.selectedFile = selectedFile; appFullscreenDiff.failedFile = this.failedFile; @@ -110,29 +111,25 @@ export class ImageDiff extends LitElement { `}
- ${this._showDiff && !this.failedFile.isNew - ? html`` - : html``} + +
`; From fdf7007f515e576b986652d930f0259d268e1340 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 16:28:14 +0200 Subject: [PATCH 14/48] fix: activate sbb-icons in tests --- src/components/core/testing/test-setup.ts | 4 +++- web-test-runner.config.js | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index a795c33515..c1bf4bd5e1 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -27,7 +27,9 @@ function setupIconConfig(): void { mergeConfig({ icon }); } -setupIconConfig(); +if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { + setupIconConfig(); +} if (isHydratedSsr()) { await import('@lit-labs/ssr-client/lit-element-hydrate-support.js'); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 209a2289e2..5e4ee272ca 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -69,6 +69,7 @@ const testRunnerHtml = (testFramework, _config, group) => ` From e444d5b91509c9c8822f2b15a04c202861e63422 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 17:25:17 +0200 Subject: [PATCH 15/48] fix: activate sbb-icons in tests 2 --- src/components/core/testing/test-setup-ssr.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/core/testing/test-setup-ssr.ts b/src/components/core/testing/test-setup-ssr.ts index 7d08fd62df..035fcef3f4 100644 --- a/src/components/core/testing/test-setup-ssr.ts +++ b/src/components/core/testing/test-setup-ssr.ts @@ -26,4 +26,6 @@ function setupIconConfig(): void { }); } -setupIconConfig(); +if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { + setupIconConfig(); +} From 69fa947ab1c05a07009b6c2d5ea79db06af45898 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Mon, 6 May 2024 12:06:23 +0200 Subject: [PATCH 16/48] fix: icon handling --- src/components/core/testing/test-setup-ssr.ts | 38 ++++++++++--------- src/components/core/testing/test-setup.ts | 4 +- web-test-runner.config.js | 1 - 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/components/core/testing/test-setup-ssr.ts b/src/components/core/testing/test-setup-ssr.ts index 035fcef3f4..76313d001f 100644 --- a/src/components/core/testing/test-setup-ssr.ts +++ b/src/components/core/testing/test-setup-ssr.ts @@ -1,31 +1,33 @@ +import { isServer } from 'lit'; + import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -function setupIconConfig(): void { - const testNamespaces = ['default', 'picto']; - const icon: SbbIconConfig = { - interceptor: ({ namespace, name, request }) => { - if (testNamespaces.includes(namespace)) { - const dimension = name.endsWith('-large') ? 48 : name.endsWith('-medium') ? 36 : 24; - return Promise.resolve( - ` { + if (testNamespaces.includes(namespace)) { + const dimension = name.endsWith('-large') ? 48 : name.endsWith('-medium') ? 36 : 24; + return Promise.resolve( + ` `, - ); - } - return request(); - }, - }; + ); + } + return request(); + }, + }; - mergeConfig({ - icon, - }); -} + mergeConfig({ + icon, + }); + } -if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { setupIconConfig(); } diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index c1bf4bd5e1..8489618e69 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -2,7 +2,7 @@ import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -import { isHydratedSsr } from './private.js'; +import { isHydratedSsr, isVisualRegressionRun } from './private.js'; function setupIconConfig(): void { const testNamespaces = ['default', 'picto']; @@ -27,7 +27,7 @@ function setupIconConfig(): void { mergeConfig({ icon }); } -if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { +if (!isVisualRegressionRun()) { setupIconConfig(); } diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 5e4ee272ca..209a2289e2 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -69,7 +69,6 @@ const testRunnerHtml = (testFramework, _config, group) => ` From 89e7a97f796045033c43c210add9d7945a54a313 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Mon, 6 May 2024 16:47:57 +0200 Subject: [PATCH 17/48] refactor: replace vaadin router by lit labs router --- package.json | 2 +- .../diff-app/index.html | 3 +- .../test-case-filter/test-case-filter.ts | 9 ++++ .../src/components/test-case/test-case.ts | 37 ++++++++++----- .../diff-app/src/main.ts | 47 +++++++++++++++++-- .../diff-app/src/routes.ts | 15 ------ yarn.lock | 32 +++---------- 7 files changed, 84 insertions(+), 61 deletions(-) delete mode 100644 tools/visual-regression-testing/diff-app/src/routes.ts diff --git a/package.json b/package.json index 2f9c7e667f..0ae1963d91 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "@custom-elements-manifest/to-markdown": "0.1.0", "@eslint/eslintrc": "3.0.2", "@eslint/js": "9.1.1", + "@lit-labs/router": "^0.1.3", "@lit-labs/testing": "0.2.3", "@lit/react": "1.0.5", "@open-wc/lit-helpers": "0.7.0", @@ -89,7 +90,6 @@ "@types/react-dom": "18.3.0", "@typescript-eslint/eslint-plugin": "7.8.0", "@typescript-eslint/parser": "7.8.0", - "@vaadin/router": "1.7.5", "@web/test-runner": "0.18.1", "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", diff --git a/tools/visual-regression-testing/diff-app/index.html b/tools/visual-regression-testing/diff-app/index.html index 54182ed5c0..2a80312eb1 100644 --- a/tools/visual-regression-testing/diff-app/index.html +++ b/tools/visual-regression-testing/diff-app/index.html @@ -26,8 +26,7 @@ /> -
- + diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts index 86057eeb67..c7509cae00 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts @@ -17,6 +17,15 @@ export class TestCaseFilter extends LitElement { @property() public testCase?: ScreenshotTestCase; + /** + * Activate `all`-tag of viewports and browsers. + */ + public reset(): void { + this.shadowRoot!.querySelectorAll(`sbb-tag[value='all']`).forEach( + (tag) => (tag.checked = true), + ); + } + private _handleViewportChange(event: CustomEvent): void { this.dispatchEvent( new CustomEvent('viewportFilterChange', { diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts index b665bdaf43..79ae973a20 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts @@ -1,6 +1,11 @@ -import type { BeforeEnterObserver, RouterLocation } from '@vaadin/router'; -import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; -import { customElement, state } from 'lit/decorators.js'; +import { + LitElement, + html, + type TemplateResult, + type CSSResultGroup, + type PropertyValues, +} from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; import { screenshotService, type ScreenshotTestCase } from '../../screenshots.js'; @@ -11,6 +16,7 @@ import '../../../../../../src/components/header.js'; import '../../../../../../src/components/notification.js'; import '../../../../../../src/components/title.js'; +import type { TestCaseFilter } from './test-case-filter/test-case-filter.js'; import style from './test-case.scss?lit&inline'; import './test-case-filter/test-case-filter.js'; @@ -26,20 +32,25 @@ interface Filter { * Provides filtering functions. */ @customElement('app-test-case') -export class TestCase extends LitElement implements BeforeEnterObserver { +export class TestCase extends LitElement { public static override styles: CSSResultGroup = style; - @state() private _testCaseName?: string; - @state() private _componentName?: string; + @property() public params?: { componentName: string; testCaseName: string }; + @state() private _testCase?: ScreenshotTestCase; @state() private _filter: Filter = {}; - // Called by router - public onBeforeEnter(location: RouterLocation): void { - this._testCaseName = location.params.testcase as string; - this._componentName = location.params.component as string; + protected override willUpdate(changedProperties: PropertyValues): void { + super.willUpdate(changedProperties); - this._testCase = screenshotService.setCurrentTestCase(this._componentName, this._testCaseName); + if (changedProperties.has('params')) { + this._filter = {}; + this.shadowRoot!.querySelector('app-test-case-filter')?.reset(); + this._testCase = screenshotService.setCurrentTestCase( + this.params!.componentName!, + this.params!.testCaseName!, + ); + } } private _viewportFilterChanged(event: CustomEvent): void { @@ -64,9 +75,9 @@ export class TestCase extends LitElement implements BeforeEnterObserver { style="--app-progress: ${screenshotService.progressFraction}" >
- ${this._componentName} + ${this.params?.componentName} - ${this._testCaseName} + ${this.params?.testCaseName}
diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts index 3cf7b340e6..d89e69c28e 100644 --- a/tools/visual-regression-testing/diff-app/src/main.ts +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -1,8 +1,45 @@ -import { Router } from '@vaadin/router'; - -import { routes } from './routes.js'; +import { Router } from '@lit-labs/router'; +import { html, LitElement, type TemplateResult } from 'lit'; +import { customElement } from 'lit/decorators.js'; import '../../../../src/components/core/styles/standard-theme.scss'; -export const router = new Router(document.querySelector('#outlet')); -router.setRoutes(routes); +/** + * Main app containing the router outlet. + */ +@customElement('app-main') +export class Main extends LitElement { + private _router = new Router(this, [ + { + path: '/', + render: () => html``, + enter: async () => { + await import('./components/overview/overview.js'); + return true; + }, + }, + { + path: '/compare/:component/:testcase', + render: ({ component, testcase }) => + html``, + + enter: async () => { + await import('./components/test-case/test-case.js'); + return true; + }, + }, + ]); + + public override render(): TemplateResult { + return html`${this._router.outlet()}`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-main': Main; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/routes.ts b/tools/visual-regression-testing/diff-app/src/routes.ts deleted file mode 100644 index 5fa9bfe2fb..0000000000 --- a/tools/visual-regression-testing/diff-app/src/routes.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Route } from '@vaadin/router'; - -import './components/overview/overview.js'; -import './components/test-case/test-case.js'; - -export const routes: Route[] = [ - { - path: '/', - component: 'app-overview', - }, - { - path: '/compare/:component/:testcase', - component: 'app-test-case', - }, -]; diff --git a/yarn.lock b/yarn.lock index 7ab430cad2..febb6dd840 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1508,6 +1508,13 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@lit-labs/router@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@lit-labs/router/-/router-0.1.3.tgz#6be268eec6bbcbf0d28ee66688440cc46f587882" + integrity sha512-G+HHo57KsArG58LOI8DLtipFfC9tVV4lGaDy2I8hYQvS2P/pV5wQObrpFYPZswse8D47y8VuHNXNdVPQOVc5MA== + dependencies: + lit "^2.0.0 || ^3.0.0" + "@lit-labs/ssr-client@^1.1.4", "@lit-labs/ssr-client@^1.1.7": version "1.1.7" resolved "https://registry.yarnpkg.com/@lit-labs/ssr-client/-/ssr-client-1.1.7.tgz#fd0cf4ce986ee903dc67ad647193f2323ac15a6d" @@ -3185,26 +3192,6 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@vaadin/router@1.7.5": - version "1.7.5" - resolved "https://registry.yarnpkg.com/@vaadin/router/-/router-1.7.5.tgz#af4b8f05a86e2890ea875f9bca9b406d74f9bae4" - integrity sha512-uRN3vd1ihgd596bF/NMZqpgxau0nlvIc0/JDd1EwStFNbZID/xIVse5LXdQhIyUKLmSl4T0GeCQK505xerWX0w== - dependencies: - "@vaadin/vaadin-usage-statistics" "^2.1.0" - path-to-regexp "2.4.0" - -"@vaadin/vaadin-development-mode-detector@^2.0.0": - version "2.0.6" - resolved "https://registry.yarnpkg.com/@vaadin/vaadin-development-mode-detector/-/vaadin-development-mode-detector-2.0.6.tgz#2acb568975c18b7965e56ed765638dcf183d8dd1" - integrity sha512-N6a5nLT/ytEUlpPo+nvdCKIGoyNjPsj3rzPGvGYK8x9Ceg76OTe1xI/GtN71mRW9e2HUScR0kCNOkl1Z63YDjw== - -"@vaadin/vaadin-usage-statistics@^2.1.0": - version "2.1.2" - resolved "https://registry.yarnpkg.com/@vaadin/vaadin-usage-statistics/-/vaadin-usage-statistics-2.1.2.tgz#a0a211b298e647742b60c6d4d1d8834381e3c77f" - integrity sha512-xKs1PvRfTXsG0eWWcImLXWjv7D+f1vfoIvovppv6pZ5QX8xgcxWUdNgERlOOdGt3CTuxQXukTBW3+Qfva+OXSg== - dependencies: - "@vaadin/vaadin-development-mode-detector" "^2.0.0" - "@vitest/expect@1.3.1": version "1.3.1" resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.3.1.tgz#d4c14b89c43a25fd400a6b941f51ba27fe0cb918" @@ -8940,11 +8927,6 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== -path-to-regexp@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-to-regexp@^6.2.1: version "6.2.2" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.2.tgz#324377a83e5049cbecadc5554d6a63a9a4866b36" From 3e4960bcecb814b61b6b573ec12fb33e2805c617 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Mon, 6 May 2024 21:10:16 +0200 Subject: [PATCH 18/48] refactor: remove service because obsolete since router refactoring Signed-off-by: Jeremias Peier --- .../src/components/overview/overview.ts | 12 +++--- .../src/components/test-case/test-case.ts | 42 ++++++++++++------ .../diff-app/src/screenshots.ts | 43 +------------------ 3 files changed, 35 insertions(+), 62 deletions(-) diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts index 8e1a9032e7..cf835c5219 100644 --- a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts @@ -1,7 +1,7 @@ import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; import { customElement } from 'lit/decorators.js'; -import { screenshotService } from '../../screenshots.js'; +import { screenshots } from '../../screenshots.js'; import style from './overview.scss?lit&inline'; @@ -27,20 +27,18 @@ export class Overview extends LitElement { Lyne visual regression comparison
- ${screenshotService.screenshots.stats} + ${screenshots.stats} Start comparing - ${screenshotService.screenshots.components.map( + ${screenshots.components.map( (screenshotComponent) => html` - + ${screenshotComponent.name} (${screenshotComponent.stats}) diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts index 79ae973a20..000ecb9984 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts @@ -1,13 +1,13 @@ import { - LitElement, - html, - type TemplateResult, type CSSResultGroup, + html, + LitElement, type PropertyValues, + type TemplateResult, } from 'lit'; import { customElement, property, state } from 'lit/decorators.js'; -import { screenshotService, type ScreenshotTestCase } from '../../screenshots.js'; +import { screenshots, type ScreenshotTestCase } from '../../screenshots.js'; import '../../../../../../src/components/button/secondary-button-link.js'; import '../../../../../../src/components/chip.js'; @@ -38,21 +38,40 @@ export class TestCase extends LitElement { @property() public params?: { componentName: string; testCaseName: string }; @state() private _testCase?: ScreenshotTestCase; + @state() private _testCaseIndex: number = -1; @state() private _filter: Filter = {}; protected override willUpdate(changedProperties: PropertyValues): void { super.willUpdate(changedProperties); if (changedProperties.has('params')) { + // Reset this._filter = {}; this.shadowRoot!.querySelector('app-test-case-filter')?.reset(); - this._testCase = screenshotService.setCurrentTestCase( + + // Get test case + this._testCaseIndex = screenshots.indexOfTestCase( this.params!.componentName!, this.params!.testCaseName!, ); + if (this._testCaseIndex >= 0) { + this._testCase = screenshots.getByTestCaseIndex(this._testCaseIndex); + } } } + private _progressFraction(): number { + return (this._testCaseIndex + 1) / screenshots.testCaseCount; + } + + private _next(): ScreenshotTestCase | undefined { + return screenshots.getByTestCaseIndex(this._testCaseIndex + 1); + } + + private _previous(): ScreenshotTestCase | undefined { + return screenshots.getByTestCaseIndex(this._testCaseIndex - 1); + } + private _viewportFilterChanged(event: CustomEvent): void { this._filter = { ...this._filter, @@ -70,10 +89,7 @@ export class TestCase extends LitElement { public override render(): TemplateResult { return html` -
+
${this.params?.componentName} @@ -84,16 +100,16 @@ export class TestCase extends LitElement {
Overview
diff --git a/tools/visual-regression-testing/diff-app/src/screenshots.ts b/tools/visual-regression-testing/diff-app/src/screenshots.ts index 71c49eb205..f14f1415b5 100644 --- a/tools/visual-regression-testing/diff-app/src/screenshots.ts +++ b/tools/visual-regression-testing/diff-app/src/screenshots.ts @@ -174,45 +174,4 @@ export class Screenshots { } } -export class ScreenshotService { - public readonly screenshots; - private _currentIndex: number = -1; - - public get progressFraction(): number { - return (this._currentIndex + 1) / this.screenshots.testCaseCount; - } - - public get current(): ScreenshotTestCase | undefined { - return this._current; - } - private _current?: ScreenshotTestCase; - - public constructor(screenshots: Screenshots) { - this.screenshots = screenshots; - this._currentIndex = 0; - this._current = screenshots.getByTestCaseIndex(this._currentIndex); - } - - public get next(): ScreenshotTestCase | undefined { - return this.screenshots.getByTestCaseIndex(this._currentIndex + 1); - } - - public get previous(): ScreenshotTestCase | undefined { - return this.screenshots.getByTestCaseIndex(this._currentIndex - 1); - } - - public setCurrentTestCase( - componentName: string, - testCaseName: string, - ): ScreenshotTestCase | undefined { - const testCaseIndex = this.screenshots.indexOfTestCase(componentName, testCaseName); - - this._currentIndex = testCaseIndex; - if (testCaseIndex >= 0) { - this._current = this.screenshots.getByTestCaseIndex(testCaseIndex); - return this._current; - } - } -} - -export const screenshotService = new ScreenshotService(new Screenshots(screenshotsRaw)); +export const screenshots = new Screenshots(screenshotsRaw); From e215e76371b803019ede7c850174be54655d7b33 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 12 Apr 2024 21:43:48 +0200 Subject: [PATCH 19/48] build: implement visual regression testing with @web/test-runner --- .github/default.conf | 3 ++ .github/workflows/continuous-integration.yml | 32 +++++++++++ .github/workflows/release-please.yml | 14 ++--- Dockerfile | 2 +- package.json | 2 + .../button/button/button.snapshot.spec.ts | 48 +++++++++++++++++ src/components/core/testing/private.ts | 3 ++ .../core/testing/private/describe-each.ts | 36 +++++++++++++ .../testing/private/describe-viewports.ts | 32 +++++++++++ .../private/visual-regression-snapshot.ts | 41 ++++++++++++++ .../baseline.Dockerfile | 9 ++++ .../baseline.nginx.conf | 37 +++++++++++++ tools/visual-regression-testing/exec.ts | 52 ++++++++++++++++++ .../testing.Dockerfile | 7 +++ .../testing.Dockerfile.dockerignore | 3 ++ tools/web-test-runner/index.js | 1 + .../visual-regression-plugin-config.js | 54 +++++++++++++++++++ web-test-runner.config.js | 54 +++++++++++-------- yarn.lock | 52 ++++++++++++++++++ 19 files changed, 453 insertions(+), 29 deletions(-) create mode 100644 src/components/button/button/button.snapshot.spec.ts create mode 100644 src/components/core/testing/private/describe-each.ts create mode 100644 src/components/core/testing/private/describe-viewports.ts create mode 100644 src/components/core/testing/private/visual-regression-snapshot.ts create mode 100644 tools/visual-regression-testing/baseline.Dockerfile create mode 100644 tools/visual-regression-testing/baseline.nginx.conf create mode 100644 tools/visual-regression-testing/exec.ts create mode 100644 tools/visual-regression-testing/testing.Dockerfile create mode 100644 tools/visual-regression-testing/testing.Dockerfile.dockerignore create mode 100644 tools/web-test-runner/visual-regression-plugin-config.js diff --git a/.github/default.conf b/.github/default.conf index dee95097b5..bdb9834ab9 100644 --- a/.github/default.conf +++ b/.github/default.conf @@ -2,6 +2,9 @@ tcp_nopush on; tcp_nodelay on; types_hash_max_size 2048; +# Suppresses the nginx version in the Server header. +server_tokens off; + # Determine if it's a valid origin and set it in the $cors variable. map "$http_origin" $cors { default ''; diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 2beaf55455..5a5ab52f7b 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -151,3 +151,35 @@ jobs: zip: true onlyChanged: true externals: '**/components/core/styles/**/*.scss' + + visual-regression: + runs-on: ubuntu-latest + permissions: + packages: write + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: [test] + env: + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run tests + run: yarn test:visual-regression --update-visual-baseline + env: + NODE_ENV: production + + - name: Build and push visual regression baseline + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + docker build \ + --file tools/visual-regression-testing/baseline.Dockerfile \ + --tag $IMAGE_REPO_VISUAL_REGRESSION:baseline \ + . + docker push $IMAGE_REPO_VISUAL_REGRESSION:baseline diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 21720da636..9b8f89edd4 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -35,7 +35,7 @@ jobs: if: needs.release-please.outputs.releases_created runs-on: ubuntu-latest env: - IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook + IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook VERSION: ${{ needs.release-please.outputs.version }} steps: - uses: actions/checkout@v4 @@ -68,13 +68,13 @@ jobs: - name: 'Container: Login to GitHub Container Repository' run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:$VERSION -t $IMAGE_REPO:latest . + run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest . env: DOCKER_BUILDKIT: 1 - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:$VERSION + run: docker push $IMAGE_REPO_STORYBOOK:$VERSION - name: 'Container: Publish image as latest' - run: docker push $IMAGE_REPO:latest + run: docker push $IMAGE_REPO_STORYBOOK:latest - name: Generate chromatic stories run: yarn generate:chromatic-stories @@ -98,7 +98,7 @@ jobs: if: needs.release-please.outputs.releases_created != true runs-on: ubuntu-latest env: - IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/lyne-components/storybook + IMAGE_REPO_STORYBOOK: ghcr.io/${{ github.repository }}/storybook VERSION: ${{ needs.release-please.outputs.version }} steps: - uses: actions/checkout@v4 @@ -118,11 +118,11 @@ jobs: - name: 'Container: Login to GitHub Container Repository' run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:dev . + run: docker build --tag $IMAGE_REPO_STORYBOOK:dev . env: DOCKER_BUILDKIT: 1 - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:dev + run: docker push $IMAGE_REPO_STORYBOOK:dev - name: Generate chromatic stories run: yarn generate:chromatic-stories diff --git a/Dockerfile b/Dockerfile index 57210bfa3f..2220166b4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nginxinc/nginx-unprivileged:stable +FROM ghcr.io/nginxinc/nginx-unprivileged:stable # Copy nginx configuration COPY ./.github/default.conf /etc/nginx/conf.d/default.conf diff --git a/package.json b/package.json index 5c37848609..2dbf1c6b35 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "test:csr": "wtr --group default", "test:ssr:hydrated": "wtr --group e2e-ssr-hydrated", "test:ssr:non-hydrated": "wtr --group e2e-ssr-non-hydrated", + "test:visual-regression": "tsx tools/visual-regression-testing/exec.ts --group=visual-regression --all-browsers", "prepare": "husky" }, "dependencies": { @@ -90,6 +91,7 @@ "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", "@web/test-runner-puppeteer": "0.16.0", + "@web/test-runner-visual-regression": "0.9.0", "chromatic": "11.3.2", "custom-elements-manifest": "2.1.0", "date-fns": "3.6.0", diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts new file mode 100644 index 0000000000..c675869609 --- /dev/null +++ b/src/components/button/button/button.snapshot.spec.ts @@ -0,0 +1,48 @@ +import { html } from 'lit'; + +import { + describeEach, + describeViewports, + fixture, + visualRegressionSnapshot, +} from '../../core/testing/private.js'; +import type { SbbButtonSize } from '../common.js'; + +import './button.js'; + +describe(`sbb-button`, () => { + describe('visual-regression', () => { + const cases = { + size: ['s', 'm', 'l'] as SbbButtonSize[], + disabled: [false, true], + negative: [false, true], + iconName: [undefined, 'arrow-right-small'], + }; + + describeViewports(() => { + describeEach(cases, ({ size, disabled, negative, iconName }) => { + let root: HTMLElement; + beforeEach(async () => { + root = await fixture(html` +
+ Button +
+ `); + }); + + visualRegressionSnapshot(() => root); + }); + }); + }); +}); diff --git a/src/components/core/testing/private.ts b/src/components/core/testing/private.ts index 57b13775f6..45ecfb614c 100644 --- a/src/components/core/testing/private.ts +++ b/src/components/core/testing/private.ts @@ -1,5 +1,8 @@ export * from './private/a11y-tree-snapshot.js'; +export * from './private/describe-each.js'; +export * from './private/describe-viewports.js'; export * from './private/dispatch-events.js'; export * from './private/event-objects.js'; export * from './private/fixture.js'; export * from './private/type-in-element.js'; +export * from './private/visual-regression-snapshot.js'; diff --git a/src/components/core/testing/private/describe-each.ts b/src/components/core/testing/private/describe-each.ts new file mode 100644 index 0000000000..bfa5d2c1ef --- /dev/null +++ b/src/components/core/testing/private/describe-each.ts @@ -0,0 +1,36 @@ +function partialDescribeEach>( + cases: T, + payload: Record, + suiteRun: (params: { [K in keyof T]: T[K][number] }) => void, +): void { + const [key, ...keys] = Object.keys(cases); + const values = cases[key]; + if (keys.length) { + const partialCases = keys.reduce( + (current, next) => Object.assign(current, { [next]: cases[next] }), + {} as T, + ); + for (const value of values) { + partialDescribeEach(partialCases, { ...payload, [key]: value }, suiteRun); + } + } else { + for (const value of values) { + const finalPayload = { ...payload, [key]: value }; + describe( + Object.entries(finalPayload) + .map(([key, value]) => `${key}=${value}`) + .join(', '), + function () { + suiteRun.call(this, finalPayload); + }, + ); + } + } +} + +export function describeEach>( + cases: T, + suiteRun: (params: { [K in keyof T]: T[K][number] }) => void, +): void { + partialDescribeEach(cases, {} as Record, suiteRun); +} diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts new file mode 100644 index 0000000000..de764fe737 --- /dev/null +++ b/src/components/core/testing/private/describe-viewports.ts @@ -0,0 +1,32 @@ +import { + SbbBreakpointLargeMax, + SbbBreakpointMediumMax, + SbbBreakpointMicroMax, + SbbBreakpointSmallMax, + SbbBreakpointUltraMax, + SbbBreakpointWideMax, + SbbBreakpointZeroMax, +} from '@sbb-esta/lyne-design-tokens'; +import { setViewport } from '@web/test-runner-commands'; + +const viewportSizes = { + zero: SbbBreakpointZeroMax, + micro: SbbBreakpointMicroMax, + small: SbbBreakpointSmallMax, + medium: SbbBreakpointMediumMax, + large: SbbBreakpointLargeMax, + wide: SbbBreakpointWideMax, + ultra: SbbBreakpointUltraMax, +}; + +export function describeViewports(fn: (this: Mocha.Suite) => void): void { + for (const [size, value] of Object.entries(viewportSizes)) { + describe(`viewport=${size}`, function () { + before(async () => { + await setViewport({ width: value, height: 400 }); + }); + + fn.call(this); + }); + } +} diff --git a/src/components/core/testing/private/visual-regression-snapshot.ts b/src/components/core/testing/private/visual-regression-snapshot.ts new file mode 100644 index 0000000000..8c63501b1e --- /dev/null +++ b/src/components/core/testing/private/visual-regression-snapshot.ts @@ -0,0 +1,41 @@ +import { sendKeys, sendMouse } from '@web/test-runner-commands'; +import { visualDiff } from '@web/test-runner-visual-regression'; + +export function imageName(test: Mocha.Runnable): string { + return test!.fullTitle().replaceAll(', ', '-').replaceAll(' ', '_'); +} + +export function visualRegressionSnapshot(snapshotElement: () => HTMLElement): void { + it('default', async function () { + await visualDiff(snapshotElement(), imageName(this.test!)); + }); + + it('focus', async function () { + await sendKeys({ press: 'Tab' }); + await visualDiff(snapshotElement(), imageName(this.test!)); + }); + + it('hover', async function () { + const element = snapshotElement(); + const positionElement = element.localName.startsWith('sbb-') + ? element + : element.firstElementChild!; + const position = positionElement.getBoundingClientRect(); + await sendMouse({ + type: 'move', + position: [ + Math.round(position.x + position.width / 2), + Math.round(position.y + position.height / 2), + ], + }); + + try { + await visualDiff(element, imageName(this.test!)); + } finally { + await sendMouse({ + type: 'move', + position: [0, 0], + }); + } + }); +} diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile new file mode 100644 index 0000000000..99c4074d78 --- /dev/null +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -0,0 +1,9 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY ./dist/screenshots /usr/share/nginx/html + +USER root +RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete +RUN find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; +USER $UID diff --git a/tools/visual-regression-testing/baseline.nginx.conf b/tools/visual-regression-testing/baseline.nginx.conf new file mode 100644 index 0000000000..a2d705e7a4 --- /dev/null +++ b/tools/visual-regression-testing/baseline.nginx.conf @@ -0,0 +1,37 @@ +tcp_nopush on; +tcp_nodelay on; +types_hash_max_size 2048; + +server { + listen 8080 default_server; + server_name _; + root /usr/share/nginx/html; + index index.html index.htm; + + # Suppresses the nginx version in the Server header. + server_tokens off; + + location / { + expires -1; + add_header Pragma "no-cache"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + try_files $uri $uri/ /index.html =404; + } + + location ~* \.(?:png)$ { + expires 1y; + access_log off; + add_header Cache-Control "public"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + } + + location ~* \.(?:css|js)$ { + expires 1y; + access_log off; + add_header Cache-Control "public"; + add_header X-Frame-Options DENY; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + } +} \ No newline at end of file diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts new file mode 100644 index 0000000000..922c1c693f --- /dev/null +++ b/tools/visual-regression-testing/exec.ts @@ -0,0 +1,52 @@ +// This script serves as checking which OS visual regression testing is run +// and if it is not Linux, runs it in a container. + +import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; +import { platform } from 'os'; + +import { startTestRunner } from '@web/test-runner'; + +const args = process.argv.slice(2); +if (platform() === 'linux' && !process.env.DEBUG) { + startTestRunner(); +} else { + function executableIsAvailable(name: string): string | null { + try { + execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); + return name; + } catch (error) { + return null; + } + } + + const containerCmd = executableIsAvailable('docker') ?? executableIsAvailable('podman'); + if (!containerCmd) { + console.log('Either docker or podman need to be installed!'); + process.exit(1); + } + + const cwd = new URL('../../', import.meta.url); + const tag = 'lyne-vrt'; + const branchName = execSync('git rev-parse --abbrev-ref HEAD'); + const execOptions: ExecSyncOptionsWithStringEncoding = { + encoding: 'utf8', + stdio: 'inherit', + cwd, + }; + execSync( + `${containerCmd} build ` + + '--file=tools/visual-regression-testing/testing.Dockerfile ' + + //`--build-arg=VERSION=${readFileSync(new URL('../../.nvmrc', import.meta.url), 'utf8').replace('v', '')}` + + `--tag=${tag} .`, + execOptions, + ); + console.log(`\nTest image ready\n`); + execSync( + `${containerCmd} run -it --rm --ipc=host ` + + `--env=BRANCH_NAME="${branchName}" ` + + `--volume=./dist/screenshots:/dist/screenshots ` + + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + + tag, + execOptions, + ); +} diff --git a/tools/visual-regression-testing/testing.Dockerfile b/tools/visual-regression-testing/testing.Dockerfile new file mode 100644 index 0000000000..1afbaed5c7 --- /dev/null +++ b/tools/visual-regression-testing/testing.Dockerfile @@ -0,0 +1,7 @@ +FROM mcr.microsoft.com/playwright:v1.43.0-jammy + +COPY package.json ./ +COPY yarn.lock ./ +RUN yarn install --frozen-lockfile --non-interactive + +COPY . . diff --git a/tools/visual-regression-testing/testing.Dockerfile.dockerignore b/tools/visual-regression-testing/testing.Dockerfile.dockerignore new file mode 100644 index 0000000000..5de02b1469 --- /dev/null +++ b/tools/visual-regression-testing/testing.Dockerfile.dockerignore @@ -0,0 +1,3 @@ +coverage +dist +node_modules \ No newline at end of file diff --git a/tools/web-test-runner/index.js b/tools/web-test-runner/index.js index 0eb5bcd92d..d4f095b400 100644 --- a/tools/web-test-runner/index.js +++ b/tools/web-test-runner/index.js @@ -2,4 +2,5 @@ export * from './minimal-reporter.js'; export * from './patched-summary-reporter.js'; export * from './ssr-plugin.js'; +export * from './visual-regression-plugin-config.js'; export * from './vite-plugin.js'; diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js new file mode 100644 index 0000000000..209fe02c12 --- /dev/null +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -0,0 +1,54 @@ +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; +import { dirname, extname, join } from 'path'; + +const baselineUrl = 'http://localhost:8080/'; //'https://lyne-visual-regression-baseline.app.sbb.ch/'; + +export const visualRegressionConfig = (update) => + /** @type {Parameters[0]} */ + ({ + update, + baseDir: 'dist/screenshots', + async getBaseline({ filePath, baseDir, name }) { + if (existsSync(filePath)) { + return readFileSync(filePath); + } + + const cacheFile = join(baseDir, '.cache', name + extname(filePath)); + const cacheFileDetails = cacheFile + '.json'; + mkdirSync(dirname(cacheFile), { recursive: true }); + const baselineFileUrl = baselineUrl + name + extname(filePath); + const downloadFile = async () => { + const response = await fetch(baselineFileUrl); + if (response.ok) { + writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); + writeFileSync( + cacheFileDetails, + JSON.stringify({ etag: response.headers.get('etag') }, null, 2), + 'utf8', + ); + } + }; + + if (existsSync(cacheFileDetails)) { + const details = JSON.parse(readFileSync(cacheFileDetails)); + const response = await fetch(baselineFileUrl, { + method: 'HEAD', + headers: { 'if-none-match': details.etag }, + }); + if (response.status === 200) { + await downloadFile(); + return readFileSync(cacheFile); + } else if (response.status === 404) { + unlinkSync(cacheFile); + unlinkSync(cacheFileDetails); + } else if (response.status === 304) { + return readFileSync(cacheFile); + } else { + console.error(`Unexpected response from baseline service: ${response.status} (${name})`); + } + } else { + await downloadFile(); + return readFileSync(cacheFile); + } + }, + }); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 5c3b7c9277..eb8101b0bf 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -2,6 +2,7 @@ import { defaultReporter } from '@web/test-runner'; import { playwrightLauncher } from '@web/test-runner-playwright'; import { puppeteerLauncher } from '@web/test-runner-puppeteer'; import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins'; +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; import * as sass from 'sass'; import { cpus } from 'node:os'; @@ -9,14 +10,18 @@ import { minimalReporter, patchedSummaryReporter, ssrPlugin, + visualRegressionConfig, vitePlugin, } from './tools/web-test-runner/index.js'; const isCIEnvironment = !!process.env.CI || process.argv.includes('--ci'); const isDebugMode = process.argv.includes('--debug'); +const allBrowsers = process.argv.includes('--all-browsers'); const firefox = process.argv.includes('--firefox'); const webkit = process.argv.includes('--webkit'); const concurrency = process.argv.includes('--parallel') ? {} : { concurrency: 1 }; +const updateBaseImages = + process.argv.includes('--update-visual-baseline') || process.argv.includes('--uv'); const stylesCompiler = new sass.initCompiler(); const renderStyles = () => @@ -24,25 +29,26 @@ const renderStyles = () => loadPaths: ['.', './node_modules/'], }).css; -const browsers = isCIEnvironment - ? [ - // Parallelism has problems, we need force concurrency to 1 - playwrightLauncher({ product: 'chromium', ...concurrency }), - playwrightLauncher({ product: 'firefox', ...concurrency }), - playwrightLauncher({ product: 'webkit', ...concurrency }), - ] - : firefox - ? [playwrightLauncher({ product: 'firefox' })] - : webkit - ? [playwrightLauncher({ product: 'webkit' })] - : isDebugMode - ? [ - puppeteerLauncher({ - launchOptions: { headless: false, devtools: true }, - ...concurrency, - }), - ] - : [playwrightLauncher({ product: 'chromium' })]; +const browsers = + isCIEnvironment || allBrowsers + ? [ + // Parallelism has problems, we need force concurrency to 1 + playwrightLauncher({ product: 'chromium', ...concurrency }), + playwrightLauncher({ product: 'firefox', ...concurrency }), + playwrightLauncher({ product: 'webkit', ...concurrency }), + ] + : firefox + ? [playwrightLauncher({ product: 'firefox' })] + : webkit + ? [playwrightLauncher({ product: 'webkit' })] + : isDebugMode + ? [ + puppeteerLauncher({ + launchOptions: { headless: false, devtools: true }, + ...concurrency, + }), + ] + : [playwrightLauncher({ product: 'chromium' })]; const groupNameOverride = process.argv.includes('--ssr-hydrated') ? 'e2e-ssr-hydrated' @@ -82,11 +88,12 @@ const suppressedLogs = [ /** @type {import('@web/test-runner').TestRunnerConfig} */ export default { - files: ['src/**/*.{e2e,spec}.ts'], + files: ['src/**/*.{e2e,spec,!snapshot.spec}.ts'], groups: [ // Disable ssr tests until stabilized. // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, + { name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }, ], nodeResolve: true, concurrency: resolveConcurrency(), @@ -95,7 +102,12 @@ export default { ? [defaultReporter(), patchedSummaryReporter()] : [minimalReporter()], browsers: browsers, - plugins: [a11ySnapshotPlugin(), ssrPlugin(), vitePlugin()], + plugins: [ + a11ySnapshotPlugin(), + ssrPlugin(), + vitePlugin(), + visualRegressionPlugin(visualRegressionConfig(updateBaseImages)), + ], testFramework: { config: { timeout: '10000', diff --git a/yarn.lock b/yarn.lock index a2c5c328d3..e080350c88 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2834,6 +2834,13 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-5.1.2.tgz#07508b45797cb81ec3f273011b054cd0755eddca" integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA== +"@types/mkdirp@^1.0.1": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.2.tgz#8d0bad7aa793abe551860be1f7ae7f3198c16666" + integrity sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ== + dependencies: + "@types/node" "*" + "@types/mocha@10.0.6": version "10.0.6" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b" @@ -2861,6 +2868,20 @@ resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb" integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g== +"@types/pixelmatch@^5.2.2": + version "5.2.6" + resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.6.tgz#fba6de304ac958495f27d85989f5c6bb7499a686" + integrity sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg== + dependencies: + "@types/node" "*" + +"@types/pngjs@^6.0.0": + version "6.0.4" + resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-6.0.4.tgz#9a457aebabd944efde1a773a0fa1d74933e8021b" + integrity sha512-atAK9xLKOnxiuArxcHovmnOUUGBZOQ3f0vCf43FnoKs6XnqiambT1kkJWmdo71IR+BoXSh+CueeFR0GfH3dTlQ== + dependencies: + "@types/node" "*" + "@types/pretty-hrtime@^1.0.0": version "1.0.3" resolved "https://registry.yarnpkg.com/@types/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#ee1bd8c9f7a01b3445786aad0ef23aba5f511a44" @@ -3474,6 +3495,20 @@ "@web/test-runner-core" "^0.13.0" puppeteer "^22.0.0" +"@web/test-runner-visual-regression@0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@web/test-runner-visual-regression/-/test-runner-visual-regression-0.9.0.tgz#9875a4871a24f8bf520c97b80ce548e81bce51e6" + integrity sha512-06M1WffLy+BJo08s57RumKYUULD/UB8u7DgZ8/MJZYCt+7r4Vt54w34CwSGHCpeDLY8Z/YkxecafvzDjuLnEJQ== + dependencies: + "@types/mkdirp" "^1.0.1" + "@types/pixelmatch" "^5.2.2" + "@types/pngjs" "^6.0.0" + "@web/test-runner-commands" "^0.9.0" + "@web/test-runner-core" "^0.13.0" + mkdirp "^1.0.4" + pixelmatch "^5.2.1" + pngjs "^7.0.0" + "@web/test-runner@0.18.2": version "0.18.2" resolved "https://registry.yarnpkg.com/@web/test-runner/-/test-runner-0.18.2.tgz#3c93f0f16c2969e9d0549f110a7d4227aeaac389" @@ -8892,6 +8927,13 @@ pirates@^4.0.6: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== +pixelmatch@^5.2.1: + version "5.3.0" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a" + integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q== + dependencies: + pngjs "^6.0.0" + pkg-dir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" @@ -8932,6 +8974,16 @@ pluralize@^8.0.0: resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== +pngjs@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821" + integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg== + +pngjs@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26" + integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow== + polished@^4.2.2: version "4.3.1" resolved "https://registry.yarnpkg.com/polished/-/polished-4.3.1.tgz#5a00ae32715609f83d89f6f31d0f0261c6170548" From c7576170e0b645cfd113d3b1556738e8b4ff66a5 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Tue, 16 Apr 2024 16:54:02 +0200 Subject: [PATCH 20/48] fix: little fixlis --- tools/visual-regression-testing/exec.ts | 6 ++-- .../visual-regression-plugin-config.js | 29 +++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index 922c1c693f..d71efc10bc 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -2,12 +2,13 @@ // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; +import { mkdirSync } from 'fs'; import { platform } from 'os'; import { startTestRunner } from '@web/test-runner'; const args = process.argv.slice(2); -if (platform() === 'linux' && !process.env.DEBUG) { +if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { startTestRunner(); } else { function executableIsAvailable(name: string): string | null { @@ -19,7 +20,7 @@ if (platform() === 'linux' && !process.env.DEBUG) { } } - const containerCmd = executableIsAvailable('docker') ?? executableIsAvailable('podman'); + const containerCmd = executableIsAvailable('podman') ?? executableIsAvailable('docker'); if (!containerCmd) { console.log('Either docker or podman need to be installed!'); process.exit(1); @@ -41,6 +42,7 @@ if (platform() === 'linux' && !process.env.DEBUG) { execOptions, ); console.log(`\nTest image ready\n`); + mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + `--env=BRANCH_NAME="${branchName}" ` + diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 209fe02c12..5b9ce18713 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -18,14 +18,21 @@ export const visualRegressionConfig = (update) => mkdirSync(dirname(cacheFile), { recursive: true }); const baselineFileUrl = baselineUrl + name + extname(filePath); const downloadFile = async () => { - const response = await fetch(baselineFileUrl); - if (response.ok) { - writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); - writeFileSync( - cacheFileDetails, - JSON.stringify({ etag: response.headers.get('etag') }, null, 2), - 'utf8', - ); + try { + const response = await fetch(baselineFileUrl); + + if (response.ok) { + writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); + writeFileSync( + cacheFileDetails, + JSON.stringify({ etag: response.headers.get('etag') }, null, 2), + 'utf8', + ); + + return readFileSync(cacheFile); + } + } catch { + /* empty */ } }; @@ -36,8 +43,7 @@ export const visualRegressionConfig = (update) => headers: { 'if-none-match': details.etag }, }); if (response.status === 200) { - await downloadFile(); - return readFileSync(cacheFile); + return await downloadFile(); } else if (response.status === 404) { unlinkSync(cacheFile); unlinkSync(cacheFileDetails); @@ -47,8 +53,7 @@ export const visualRegressionConfig = (update) => console.error(`Unexpected response from baseline service: ${response.status} (${name})`); } } else { - await downloadFile(); - return readFileSync(cacheFile); + return await downloadFile(); } }, }); From ab6e622a652927fbbd29877c9458f3b5d101e059 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Wed, 17 Apr 2024 16:45:42 +0200 Subject: [PATCH 21/48] feat: initial app --- package.json | 2 + .../diff-app/index.html | 32 ++++ .../diff-app/src/main.ts | 8 + .../diff-app/src/vite-env.d.ts | 6 + .../diff-app/tsconfig.json | 8 + .../diff-app/vite.config.ts | 150 ++++++++++++++++++ 6 files changed, 206 insertions(+) create mode 100644 tools/visual-regression-testing/diff-app/index.html create mode 100644 tools/visual-regression-testing/diff-app/src/main.ts create mode 100644 tools/visual-regression-testing/diff-app/src/vite-env.d.ts create mode 100644 tools/visual-regression-testing/diff-app/tsconfig.json create mode 100644 tools/visual-regression-testing/diff-app/vite.config.ts diff --git a/package.json b/package.json index 2dbf1c6b35..0246fe0d98 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "build:react:production": "vite build --config src/react/vite.config.ts", "build:react:development": "NODE_ENV=development vite build --mode development --config src/react/vite.config.ts", "build:storybook": "storybook build --quiet --output-dir dist/storybook --stats-json", + "build:diff-app": "vite build --config tools/visual-regression-testing/diff-app/vite.config.ts", "build": "npm-run-all --sequential build:components build:react build:storybook", "docs": "npm-run-all --sequential docs:manifest docs:to-md", "docs:manifest": "custom-elements-manifest analyze --config tools/manifest/custom-elements-manifest.config.js", @@ -48,6 +49,7 @@ "lint:tsc:components": "tsc --noEmit --project src/components/tsconfig.json", "lint:tsc:components-spec": "tsc --noEmit --project src/components/tsconfig.spec.json", "start": "storybook dev -p 6006", + "start:diff-app": "vite --config tools/visual-regression-testing/diff-app/vite.config.ts", "test": "wtr --coverage", "test:snapshot": "yarn test:csr --ci --update-snapshots", "test:csr": "wtr --group default", diff --git a/tools/visual-regression-testing/diff-app/index.html b/tools/visual-regression-testing/diff-app/index.html new file mode 100644 index 0000000000..b6bc0adf39 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/index.html @@ -0,0 +1,32 @@ + + + + Visual Regression Tests Comparison + + + + + + Hi + + + + diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts new file mode 100644 index 0000000000..efabec6e06 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -0,0 +1,8 @@ +import '../../../../src/components/title.js'; + +import '../../../../src/components/core/styles/global.scss'; + +// eslint-disable-next-line import-x/no-unresolved +import { screenshots } from 'virtual:screenshots'; + +console.log(screenshots); diff --git a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts new file mode 100644 index 0000000000..8b67858995 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts @@ -0,0 +1,6 @@ +/// + +declare module 'virtual:screenshots' { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + export const screenshots: Record; +} diff --git a/tools/visual-regression-testing/diff-app/tsconfig.json b/tools/visual-regression-testing/diff-app/tsconfig.json new file mode 100644 index 0000000000..e4072bd67d --- /dev/null +++ b/tools/visual-regression-testing/diff-app/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "rootDir": "../../..", + "baseUrl": "." + }, + "include": ["./**/*.ts", "./src/vite-env.d.ts"] +} diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts new file mode 100644 index 0000000000..75d4501e46 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -0,0 +1,150 @@ +import { existsSync, readdirSync, readFileSync } from 'fs'; +import { relative } from 'path'; + +import { + defineConfig, + mergeConfig, + type PluginOption, + type ResolvedConfig, + type UserConfig, +} from 'vite'; + +import rootConfig from '../../../vite.config.js'; +import { distDir } from '../../vite/index.js'; + +const packageRoot = new URL('.', import.meta.url); +const screenshotsDir = new URL(`./screenshots/`, distDir); + +export interface FailedFiles { + browserName: string; + name: string; + failedFile: string; + diffFile: string; + baselineFile: string; + isNew: boolean; +} + +function prepareScreenshots(): PluginOption { + let viteConfig: ResolvedConfig; + const virtualModuleId = 'virtual:screenshots'; + const resolvedVirtualModuleId = '\0' + virtualModuleId; + + return { + name: 'prepare screenshot', + configResolved(config) { + viteConfig = config; + }, + resolveId(id) { + if (id === virtualModuleId) { + return resolvedVirtualModuleId; + } + }, + load(id) { + if (id === resolvedVirtualModuleId) { + const browsers = readdirSync(screenshotsDir, { withFileTypes: true }) + .filter((d) => d.name !== '.cache') + .map((d) => d.name); + + const screenshotsMeta = browsers + .filter((browserName) => existsSync(new URL(`./${browserName}/failed/`, screenshotsDir))) + .flatMap((browserName) => { + const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); + + return readdirSync(failedDir, { + withFileTypes: true, + }) + .filter((d) => !d.name.endsWith('-diff.png')) + .map((d) => { + const failedFilePath = new URL(`./${d.name}`, failedDir); + const diffFilePath = new URL( + `./${d.name.replace(/.png$/, '-diff.png')}`, + failedDir, + ); + const baselineFilePath = new URL( + `./${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + const baselineCacheFilePath = new URL( + `./.cache/${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + + const isNew = !existsSync(diffFilePath); + + const assetsScreenshots = 'assets/screenshots/'; + const failedRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, failedFilePath.pathname); + const diffRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); + const baselineRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, baselineFilePath.pathname); + + if (viteConfig.command !== 'serve') { + this.emitFile({ + type: 'asset', + fileName: failedRelativeFileName, + source: readFileSync(failedFilePath), + }); + + if (!isNew) { + this.emitFile({ + type: 'asset', + fileName: diffRelativeFileName, + source: readFileSync(diffFilePath), + }); + + this.emitFile({ + type: 'asset', + fileName: baselineRelativeFileName, + source: readFileSync( + existsSync(baselineFilePath) ? baselineFilePath : baselineCacheFilePath, + ), + }); + } + } + + return { + browserName, + name: d.name, + failedFile: failedRelativeFileName, + diffFile: diffRelativeFileName, + baselineFile: baselineRelativeFileName, + isNew, + }; + }); + }) + .reduce( + (current, next) => + current.set( + next.name, + current.has(next.name) ? current.get(next.name)!.concat(next) : [next], + ), + new Map(), + ); + + return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; + } + }, + configureServer(server) { + server.middlewares.use((req, res, next) => { + if (req.url?.startsWith('/assets/screenshots/')) { + console.log(req.url); + res.end(readFileSync(new URL(`.${req.url.substring(7)}`, distDir))); + } else { + next(); + } + }); + }, + }; +} + +export default defineConfig(() => + mergeConfig(rootConfig, { + root: packageRoot.pathname, + plugins: [prepareScreenshots()], + build: { + outDir: new URL(`./diff-app/`, distDir).pathname, + emptyOutDir: true, + }, + }), +); From 04cccc229871e555a996a7b16dbe1e4e6ce6176a Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Mon, 15 Apr 2024 09:28:06 +0200 Subject: [PATCH 22/48] build: use local image --- .github/workflows/continuous-integration.yml | 21 +++++++++++++++---- .../baseline.Dockerfile | 5 +++-- .../visual-regression-plugin-config.js | 7 ++++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 5a5ab52f7b..a8597fc777 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -23,6 +23,7 @@ jobs: integrity: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -38,6 +39,12 @@ jobs: test: runs-on: ubuntu-latest + needs: lint + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -63,6 +70,7 @@ jobs: build: runs-on: ubuntu-latest + needs: lint steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -152,14 +160,19 @@ jobs: onlyChanged: true externals: '**/components/core/styles/**/*.scss' - visual-regression: + visual-regression-baseline: runs-on: ubuntu-latest permissions: packages: write - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - needs: [test] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: test env: IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -170,7 +183,7 @@ jobs: - name: Install browser dependencies run: yarn playwright install-deps - - name: Run tests + - name: Run visual regression baseline generation run: yarn test:visual-regression --update-visual-baseline env: NODE_ENV: production diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index 99c4074d78..c637c6125b 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -4,6 +4,7 @@ COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/def COPY ./dist/screenshots /usr/share/nginx/html USER root -RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete -RUN find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; +RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete && \ + find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; && \ + find /usr/share/nginx/html/*/ -type d -name .cache -prune -exec rm -rf {} \; USER $UID diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 5b9ce18713..9030e4d482 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -1,7 +1,9 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; import { dirname, extname, join } from 'path'; -const baselineUrl = 'http://localhost:8080/'; //'https://lyne-visual-regression-baseline.app.sbb.ch/'; +const baselineUrl = process.env.CI + ? 'http://localhost:8050/' + : 'https://lyne-visual-regression-baseline.app.sbb.ch/'; export const visualRegressionConfig = (update) => /** @type {Parameters[0]} */ @@ -45,8 +47,7 @@ export const visualRegressionConfig = (update) => if (response.status === 200) { return await downloadFile(); } else if (response.status === 404) { - unlinkSync(cacheFile); - unlinkSync(cacheFileDetails); + [cacheFile, cacheFileDetails].forEach(unlinkSync); } else if (response.status === 304) { return readFileSync(cacheFile); } else { From f09e07625dfe174993714a3b81377a672b90172a Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Wed, 17 Apr 2024 10:30:40 +0200 Subject: [PATCH 23/48] chore: clean-up --- tools/visual-regression-testing/exec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index d71efc10bc..8f493c5f55 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -1,4 +1,4 @@ -// This script serves as checking which OS visual regression testing is run +// This script checks which OS the visual regression testing is run on // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; @@ -28,7 +28,6 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { const cwd = new URL('../../', import.meta.url); const tag = 'lyne-vrt'; - const branchName = execSync('git rev-parse --abbrev-ref HEAD'); const execOptions: ExecSyncOptionsWithStringEncoding = { encoding: 'utf8', stdio: 'inherit', @@ -37,7 +36,6 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { execSync( `${containerCmd} build ` + '--file=tools/visual-regression-testing/testing.Dockerfile ' + - //`--build-arg=VERSION=${readFileSync(new URL('../../.nvmrc', import.meta.url), 'utf8').replace('v', '')}` + `--tag=${tag} .`, execOptions, ); @@ -45,7 +43,6 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + - `--env=BRANCH_NAME="${branchName}" ` + `--volume=./dist/screenshots:/dist/screenshots ` + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + tag, From 3a103bdcb0107881276877ce96eb8aa8e65fa8cc Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 19 Apr 2024 17:28:24 +0200 Subject: [PATCH 24/48] feat: add container image cleanup and etag logic --- ...age-cleanup.yml => container-image-cleanup.yml} | 9 +++++++-- .github/workflows/continuous-integration.yml | 2 ++ .../visual-regression-testing/baseline.Dockerfile | 14 +++++++++++--- .../visual-regression-testing/baseline.nginx.conf | 13 +++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) rename .github/workflows/{preview-image-cleanup.yml => container-image-cleanup.yml} (90%) diff --git a/.github/workflows/preview-image-cleanup.yml b/.github/workflows/container-image-cleanup.yml similarity index 90% rename from .github/workflows/preview-image-cleanup.yml rename to .github/workflows/container-image-cleanup.yml index 3a9a8df931..15fb21b859 100644 --- a/.github/workflows/preview-image-cleanup.yml +++ b/.github/workflows/container-image-cleanup.yml @@ -1,4 +1,4 @@ -name: Preview Image Cleanup +name: Container Image Cleanup on: workflow_dispatch: {} @@ -9,7 +9,7 @@ permissions: packages: write jobs: - preview-image: + container-image-cleanup: runs-on: ubuntu-latest env: CLOSED_PR_RETENTION_DAYS: 14 @@ -59,3 +59,8 @@ jobs: package-name: lyne-components/storybook-preview package-type: container delete-only-untagged-versions: 'true' + - uses: actions/delete-package-versions@v4 + with: + package-name: lyne-components/visual-regression + package-type: container + delete-only-untagged-versions: 'true' diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index a8597fc777..d0e48d2603 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -196,3 +196,5 @@ jobs: --tag $IMAGE_REPO_VISUAL_REGRESSION:baseline \ . docker push $IMAGE_REPO_VISUAL_REGRESSION:baseline + env: + DOCKER_BUILDKIT: 1 diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index c637c6125b..f1b9ecc96b 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -4,7 +4,15 @@ COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/def COPY ./dist/screenshots /usr/share/nginx/html USER root -RUN find /usr/share/nginx/html/*/ -type f ! -iname "*.png" -delete && \ - find /usr/share/nginx/html/*/ -type d -name failed -prune -exec rm -rf {} \; && \ - find /usr/share/nginx/html/*/ -type d -name .cache -prune -exec rm -rf {} \; +# We calculate the sha1 hashes of the png files in order to use it as etag values. +# This allows us to use HTTP caching mechanisms, which should reduce network traffic +# for the baseline comparison. +RUN cd /usr/share/nginx/html && \ + find ./*/ -type f ! -iname "*.png" -delete && \ + find ./*/ -type d -name failed -prune -exec rm -rf {} \; && \ + find ./*/ -type d -name .cache -prune -exec rm -rf {} \; && \ + echo 'map_hash_bucket_size 16384;' > /etc/nginx/conf.d/1etags.conf && \ + echo 'map $uri $pngetag {' >> /etc/nginx/conf.d/1etags.conf && \ + find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha1sum $1 | cut -d " " -f 1);"' _ {} \; >> /etc/nginx/conf.d/1etags.conf && \ + echo '}' >> /etc/nginx/conf.d/1etags.conf USER $UID diff --git a/tools/visual-regression-testing/baseline.nginx.conf b/tools/visual-regression-testing/baseline.nginx.conf index a2d705e7a4..a66a48b9ec 100644 --- a/tools/visual-regression-testing/baseline.nginx.conf +++ b/tools/visual-regression-testing/baseline.nginx.conf @@ -22,9 +22,22 @@ server { location ~* \.(?:png)$ { expires 1y; access_log off; + # We are using a custom etag logic, which uses the sha1 hash of the image as etag. + # The default etag from nginx uses file modified time and file size, which is not good + # enough for our purposes. + etag off; add_header Cache-Control "public"; add_header X-Frame-Options DENY; add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header Etag $pngetag; + + set $request_etag $http_if_none_match; + if ($request_etag = false) { + set $request_etag "-"; + } + if ($request_etag = $pngetag) { + return 304 ""; + } } location ~* \.(?:css|js)$ { From f4b2359a33e6e5e929e5f91494e6340734199dbc Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 26 Apr 2024 15:50:29 +0200 Subject: [PATCH 25/48] feat: implement workflow --- .github/workflows/container-image-cleanup.yml | 44 ++--- .../continuous-integration-secure.yml | 2 +- .github/workflows/continuous-integration.yml | 39 +++- .github/workflows/release-please.yml | 4 +- Dockerfile | 2 + .../button/button/button.snapshot.spec.ts | 61 +++--- src/components/core/testing.ts | 1 - src/components/core/testing/private.ts | 1 + .../core/testing/private/fixture.ts | 3 +- .../core/testing/{ => private}/platform.ts | 7 + src/components/core/testing/test-setup.ts | 2 +- .../baseline.Dockerfile | 23 +-- .../diff-app.Dockerfile | 11 ++ .../diff-app/vite.config.ts | 174 ++++++++++-------- .../etag-map-generation.sh | 10 + tools/visual-regression-testing/exec.ts | 61 +++++- .../visual-regression-plugin-config.js | 87 ++++----- web-test-runner.config.js | 23 ++- 18 files changed, 347 insertions(+), 208 deletions(-) rename src/components/core/testing/{ => private}/platform.ts (82%) create mode 100644 tools/visual-regression-testing/diff-app.Dockerfile create mode 100755 tools/visual-regression-testing/etag-map-generation.sh diff --git a/.github/workflows/container-image-cleanup.yml b/.github/workflows/container-image-cleanup.yml index 15fb21b859..0788b6f553 100644 --- a/.github/workflows/container-image-cleanup.yml +++ b/.github/workflows/container-image-cleanup.yml @@ -3,7 +3,7 @@ name: Container Image Cleanup on: workflow_dispatch: {} schedule: - - cron: '0 5 * * *' + - cron: '0 3 * * *' permissions: packages: write @@ -12,9 +12,8 @@ jobs: container-image-cleanup: runs-on: ubuntu-latest env: - CLOSED_PR_RETENTION_DAYS: 14 - PACKAGE_NAME: storybook-preview - PR_TAG_PREFIX: preview-pr + CLOSED_PR_RETENTION_DAYS: 5 + PACKAGE_NAMES: storybook-preview,visual-regression steps: - uses: actions/github-script@v7 with: @@ -23,31 +22,34 @@ jobs: const pullRequests = await github.paginate( github.rest.pulls.list.endpoint.merge({ owner, repo, state: 'all' }) ); - const twoWeeksAgo = + const retentionPivot = new Date(Date.now() - (+process.env.CLOSED_PR_RETENTION_DAYS * 24 * 60 * 60 * 1000)); - const olderThanTwoWeeks = (date) => new Date(date) < twoWeeksAgo; + const olderThanTwoWeeks = (date) => new Date(date) < retentionPivot; const isExpiredPrTag = (version) => { const prNumber = +version.metadata?.container?.tags - ?.find((t) => t.startsWith(process.env.PR_TAG_PREFIX))?.split(process.env.PR_TAG_PREFIX)[1]; + ?.find((t) => t.match(/(preview-pr|pr)(\d+)/))?.match(/(preview-pr|pr)(\d+)/)[2]; const pr = pullRequests.find((p) => p.number === prNumber); return !!prNumber && pr?.state === 'closed' && olderThanTwoWeeks(pr.closed_at); }; - const params = { - package_type: 'container', - package_name: `${repo}/${process.env.PACKAGE_NAME}`, - username: owner - }; - const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + const packageNames = process.env.PACKAGE_NAME.split(',').map((n) => n.trim()); let packageDeletionFailed = false; - for (const version of versions.filter(isExpiredPrTag)) { - try { - await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); - console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); - } catch(e) { - console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); - console.error(e); - packageDeletionFailed = true; + for (const packageName of packageNames) { + const params = { + package_type: 'container', + package_name: `${repo}/${packageNames}`, + username: owner + }; + const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + for (const version of versions.filter(isExpiredPrTag)) { + try { + await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); + console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); + } catch(e) { + console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); + console.error(e); + packageDeletionFailed = true; + } } } diff --git a/.github/workflows/continuous-integration-secure.yml b/.github/workflows/continuous-integration-secure.yml index 93b3f8fb07..e7a4ae398e 100644 --- a/.github/workflows/continuous-integration-secure.yml +++ b/.github/workflows/continuous-integration-secure.yml @@ -46,7 +46,7 @@ jobs: uses: actions/github-script@v7 with: script: | - const environment = process.env.PR_NUMBER ? `preview-pr${process.env.PR_NUMBER}` : 'main'; + const environment = process.env.PR_NUMBER ? `pr${process.env.PR_NUMBER}` : 'main'; const payload = { owner: context.repo.owner, repo: context.repo.repo, environment }; const { data: deployment } = await github.rest.repos.createDeployment({ ...payload, diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d0e48d2603..9050f779fc 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -7,6 +7,9 @@ concurrency: permissions: read-all +env: + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + jobs: lint: runs-on: ubuntu-latest @@ -40,11 +43,6 @@ jobs: test: runs-on: ubuntu-latest needs: lint - services: - visual-regression: - image: ghcr.io/${{ github.repository }}/visual-regression:baseline - ports: - - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -160,14 +158,41 @@ jobs: onlyChanged: true externals: '**/components/core/styles/**/*.scss' + visual-regression: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + needs: test + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8050:8080 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run visual regression baseline generation + run: yarn test:visual-regression + env: + NODE_ENV: production + - name: Store visual regression output + uses: actions/upload-artifact@v4 + with: + name: visual-regression-screenshots + path: dist/screenshots-artifact/ + visual-regression-baseline: runs-on: ubuntu-latest permissions: packages: write if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: test - env: - IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression services: visual-regression: image: ghcr.io/${{ github.repository }}/visual-regression:baseline diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 9b8f89edd4..9d800563ab 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -66,7 +66,7 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest . env: @@ -116,7 +116,7 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' run: docker build --tag $IMAGE_REPO_STORYBOOK:dev . env: diff --git a/Dockerfile b/Dockerfile index 2220166b4f..bd0cd09679 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM ghcr.io/nginxinc/nginx-unprivileged:stable +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + # Copy nginx configuration COPY ./.github/default.conf /etc/nginx/conf.d/default.conf diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts index c675869609..abe7497126 100644 --- a/src/components/button/button/button.snapshot.spec.ts +++ b/src/components/button/button/button.snapshot.spec.ts @@ -4,6 +4,7 @@ import { describeEach, describeViewports, fixture, + isVisualRegressionRun, visualRegressionSnapshot, } from '../../core/testing/private.js'; import type { SbbButtonSize } from '../common.js'; @@ -11,38 +12,40 @@ import type { SbbButtonSize } from '../common.js'; import './button.js'; describe(`sbb-button`, () => { - describe('visual-regression', () => { - const cases = { - size: ['s', 'm', 'l'] as SbbButtonSize[], - disabled: [false, true], - negative: [false, true], - iconName: [undefined, 'arrow-right-small'], - }; + if (isVisualRegressionRun()) { + describe('visual-regression', () => { + const cases = { + size: ['s', 'm', 'l'] as SbbButtonSize[], + disabled: [false, true], + negative: [false, true], + iconName: [undefined, 'arrow-right-small'], + }; - describeViewports(() => { - describeEach(cases, ({ size, disabled, negative, iconName }) => { - let root: HTMLElement; - beforeEach(async () => { - root = await fixture(html` -
- Button { + describeEach(cases, ({ size, disabled, negative, iconName }) => { + let root: HTMLElement; + beforeEach(async () => { + root = await fixture(html` +
-
- `); - }); + Button +
+ `); + }); - visualRegressionSnapshot(() => root); + visualRegressionSnapshot(() => root); + }); }); }); - }); + } }); diff --git a/src/components/core/testing.ts b/src/components/core/testing.ts index 032a69bed9..428ae3f973 100644 --- a/src/components/core/testing.ts +++ b/src/components/core/testing.ts @@ -1,6 +1,5 @@ export * from './testing/event-spy.js'; export * from './testing/mocha-extensions.js'; -export * from './testing/platform.js'; export * from './testing/scroll.js'; export * from './testing/wait-for-condition.js'; export * from './testing/wait-for-render.js'; diff --git a/src/components/core/testing/private.ts b/src/components/core/testing/private.ts index 45ecfb614c..ba1753f948 100644 --- a/src/components/core/testing/private.ts +++ b/src/components/core/testing/private.ts @@ -4,5 +4,6 @@ export * from './private/describe-viewports.js'; export * from './private/dispatch-events.js'; export * from './private/event-objects.js'; export * from './private/fixture.js'; +export * from './private/platform.js'; export * from './private/type-in-element.js'; export * from './private/visual-regression-snapshot.js'; diff --git a/src/components/core/testing/private/fixture.ts b/src/components/core/testing/private/fixture.ts index 7e3788ee48..db042554d3 100644 --- a/src/components/core/testing/private/fixture.ts +++ b/src/components/core/testing/private/fixture.ts @@ -1,8 +1,9 @@ import type { TemplateResult } from 'lit'; -import { isHydratedSsr, isNonHydratedSsr } from '../platform.js'; import { waitForLitRender } from '../wait-for-render.js'; +import { isHydratedSsr, isNonHydratedSsr } from './platform.js'; + // Copied from @lit-labs/testing/lib/fixtures/fixture-options.d.ts interface FixtureOptions { /** diff --git a/src/components/core/testing/platform.ts b/src/components/core/testing/private/platform.ts similarity index 82% rename from src/components/core/testing/platform.ts rename to src/components/core/testing/private/platform.ts index eb23176b41..28a308ec16 100644 --- a/src/components/core/testing/platform.ts +++ b/src/components/core/testing/private/platform.ts @@ -32,3 +32,10 @@ export const isNonHydratedSsr = (): boolean => * Returns true, if this is run in an SSR test group. */ export const isSsr = (): boolean => isHydratedSsr() || isNonHydratedSsr(); + +/** + * This is a custom implementation. + * Returns true, if this is run in the visual regression test group. + */ +export const isVisualRegressionRun = (): boolean => + !isServer && (globalThis as any).testGroup === 'visual-regression'; diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index 5f90f96f6d..a795c33515 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -2,7 +2,7 @@ import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -import { isHydratedSsr } from './platform.js'; +import { isHydratedSsr } from './private.js'; function setupIconConfig(): void { const testNamespaces = ['default', 'picto']; diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index f1b9ecc96b..4d49ebbcc7 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -1,18 +1,13 @@ FROM ghcr.io/nginxinc/nginx-unprivileged:stable +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# Copy screenshots +COPY ./dist/screenshots/Chromium/baseline/ /usr/share/nginx/html/Chromium/baseline/ +COPY ./dist/screenshots/Firefox/baseline/ /usr/share/nginx/html/Firefox/baseline/ +COPY ./dist/screenshots/Webkit/baseline/ /usr/share/nginx/html/Webkit/baseline/ + COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf -COPY ./dist/screenshots /usr/share/nginx/html +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh -USER root -# We calculate the sha1 hashes of the png files in order to use it as etag values. -# This allows us to use HTTP caching mechanisms, which should reduce network traffic -# for the baseline comparison. -RUN cd /usr/share/nginx/html && \ - find ./*/ -type f ! -iname "*.png" -delete && \ - find ./*/ -type d -name failed -prune -exec rm -rf {} \; && \ - find ./*/ -type d -name .cache -prune -exec rm -rf {} \; && \ - echo 'map_hash_bucket_size 16384;' > /etc/nginx/conf.d/1etags.conf && \ - echo 'map $uri $pngetag {' >> /etc/nginx/conf.d/1etags.conf && \ - find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha1sum $1 | cut -d " " -f 1);"' _ {} \; >> /etc/nginx/conf.d/1etags.conf && \ - echo '}' >> /etc/nginx/conf.d/1etags.conf -USER $UID +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/diff-app.Dockerfile b/tools/visual-regression-testing/diff-app.Dockerfile new file mode 100644 index 0000000000..1bd7ea0d99 --- /dev/null +++ b/tools/visual-regression-testing/diff-app.Dockerfile @@ -0,0 +1,11 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# This is currently the same config file as for baseline. Separate into separate configs, if this changes. +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY ./dist/diff-app /usr/share/nginx/html + +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh + +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts index 75d4501e46..981c615246 100644 --- a/tools/visual-regression-testing/diff-app/vite.config.ts +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -1,3 +1,4 @@ +import { createHash } from 'crypto'; import { existsSync, readdirSync, readFileSync } from 'fs'; import { relative } from 'path'; @@ -41,86 +42,101 @@ function prepareScreenshots(): PluginOption { }, load(id) { if (id === resolvedVirtualModuleId) { - const browsers = readdirSync(screenshotsDir, { withFileTypes: true }) - .filter((d) => d.name !== '.cache') - .map((d) => d.name); - - const screenshotsMeta = browsers - .filter((browserName) => existsSync(new URL(`./${browserName}/failed/`, screenshotsDir))) - .flatMap((browserName) => { - const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); - - return readdirSync(failedDir, { - withFileTypes: true, - }) - .filter((d) => !d.name.endsWith('-diff.png')) - .map((d) => { - const failedFilePath = new URL(`./${d.name}`, failedDir); - const diffFilePath = new URL( - `./${d.name.replace(/.png$/, '-diff.png')}`, - failedDir, - ); - const baselineFilePath = new URL( - `./${browserName}/baseline/${d.name}`, - screenshotsDir, - ); - const baselineCacheFilePath = new URL( - `./.cache/${browserName}/baseline/${d.name}`, - screenshotsDir, - ); - - const isNew = !existsSync(diffFilePath); - - const assetsScreenshots = 'assets/screenshots/'; - const failedRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, failedFilePath.pathname); - const diffRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); - const baselineRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, baselineFilePath.pathname); - - if (viteConfig.command !== 'serve') { - this.emitFile({ - type: 'asset', - fileName: failedRelativeFileName, - source: readFileSync(failedFilePath), + const failedScreenshotsHash = createHash('sha256'); + const screenshotsFailures = existsSync(screenshotsDir) + ? readdirSync(screenshotsDir, { withFileTypes: true }) + .map((d) => d.name) + .filter((browserName) => + existsSync(new URL(`./${browserName}/failed/`, screenshotsDir)), + ) + .flatMap((browserName) => { + const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); + + return readdirSync(failedDir, { + withFileTypes: true, + }) + .filter((d) => !d.name.endsWith('-diff.png')) + .map((d) => { + const failedFilePath = new URL(`./${d.name}`, failedDir); + const diffFilePath = new URL( + `./${d.name.replace(/.png$/, '-diff.png')}`, + failedDir, + ); + const baselineFilePath = new URL( + `./${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + + const isNew = !existsSync(diffFilePath); + + const assetsScreenshots = 'assets/screenshots/'; + const failedRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, failedFilePath.pathname); + const diffRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); + const baselineRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, baselineFilePath.pathname); + + if (viteConfig.command !== 'serve') { + const failedFileContent = readFileSync(failedFilePath); + // We only add the failed screenshot hashes, as the baseline and comparison (*-diff.png) + // are not relevant to detect whether it is a new difference. + failedScreenshotsHash.update(failedFileContent); + this.emitFile({ + type: 'asset', + fileName: failedRelativeFileName, + source: failedFileContent, + }); + + if (!isNew) { + this.emitFile({ + type: 'asset', + fileName: diffRelativeFileName, + source: readFileSync(diffFilePath), + }); + + this.emitFile({ + type: 'asset', + fileName: baselineRelativeFileName, + source: readFileSync(baselineFilePath), + }); + } + } + + return { + browserName, + name: d.name, + failedFile: failedRelativeFileName, + diffFile: diffRelativeFileName, + baselineFile: baselineRelativeFileName, + isNew, + }; }); - - if (!isNew) { - this.emitFile({ - type: 'asset', - fileName: diffRelativeFileName, - source: readFileSync(diffFilePath), - }); - - this.emitFile({ - type: 'asset', - fileName: baselineRelativeFileName, - source: readFileSync( - existsSync(baselineFilePath) ? baselineFilePath : baselineCacheFilePath, - ), - }); - } - } - - return { - browserName, - name: d.name, - failedFile: failedRelativeFileName, - diffFile: diffRelativeFileName, - baselineFile: baselineRelativeFileName, - isNew, - }; - }); - }) - .reduce( - (current, next) => - current.set( - next.name, - current.has(next.name) ? current.get(next.name)!.concat(next) : [next], - ), - new Map(), - ); + }) + : []; + + const screenshotsMeta = screenshotsFailures.reduce( + (current, next) => + current.set( + next.name, + current.has(next.name) ? current.get(next.name)!.concat(next) : [next], + ), + new Map(), + ); + + if (viteConfig.command !== 'serve') { + this.emitFile({ + type: 'asset', + fileName: 'diff.json', + source: JSON.stringify({ + changedAmount: screenshotsFailures.filter((f) => !f.isNew).length, + newAmount: screenshotsFailures.filter((f) => f.isNew).length, + hash: failedScreenshotsHash.digest('hex'), + }), + }); + } return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; } diff --git a/tools/visual-regression-testing/etag-map-generation.sh b/tools/visual-regression-testing/etag-map-generation.sh new file mode 100755 index 0000000000..5f88502c31 --- /dev/null +++ b/tools/visual-regression-testing/etag-map-generation.sh @@ -0,0 +1,10 @@ + +# We calculate the sha1 hashes of the png files in order to use it as etag values. +# This allows us to use HTTP caching mechanisms, which should reduce network traffic +# for the baseline comparison. +cd /usr/share/nginx/html + +echo 'map_hash_bucket_size 32768;' +echo 'map $uri $pngetag {' +find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha256sum $1 | cut -d " " -f 1);"' _ {} \; +echo '}' \ No newline at end of file diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index 8f493c5f55..13ebd9665b 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -2,14 +2,61 @@ // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; -import { mkdirSync } from 'fs'; +import { cpSync, existsSync, mkdirSync } from 'fs'; import { platform } from 'os'; import { startTestRunner } from '@web/test-runner'; +import * as glob from 'glob'; -const args = process.argv.slice(2); -if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { - startTestRunner(); +if (process.env.GITHUB_ACTIONS) { + // When being run on GitHub Actions we have two use cases. + // Baseline generation for which our expectation is to not fail. + // Diff generation if any test fails, for which we copy only the necessary + // files to dist/screenshots-artifact/ to reduce artifact size. + const runner = await startTestRunner({ autoExitProcess: false }); + if (!runner) { + throw new Error( + `Unexpected state. Test runner not available. Check tools/visual-regression-testing/exec.ts execution.`, + ); + } + await new Promise((r) => runner.on('stopped', r)); + + const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); + const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); + mkdirSync(artifactDir, { recursive: true }); + + if (runner.passed) { + // Tests passed. Do nothing. + process.exit(0); + } + + // When visual regression tests have failed, we only want to pack the relevant screenshots + // into the artifact transfered to the secure workflow, as uploading and downloading the full + // baseline would take far longer. + // Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will + // be moved to /dist/screenshots in the secure workflow. + const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); + for (const failedDir of failedDirs) { + cpSync(new URL(`./${failedDir}`, screenshotDir), new URL(`./${failedDir}`, artifactDir), { + force: true, + recursive: true, + }); + } + + const failedFiles = glob + .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) + .map((p) => p.replace('/failed/', '/baseline/')); + for (const failedFile of failedFiles) { + const baselineFile = new URL(`./${failedFile}`, screenshotDir); + if (existsSync(baselineFile)) { + cpSync(baselineFile, new URL(`./${failedFile}`, artifactDir), { + force: true, + recursive: true, + }); + } + } +} else if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { + await startTestRunner(); } else { function executableIsAvailable(name: string): string | null { try { @@ -26,6 +73,7 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { process.exit(1); } + const args = process.argv.slice(2); const cwd = new URL('../../', import.meta.url); const tag = 'lyne-vrt'; const execOptions: ExecSyncOptionsWithStringEncoding = { @@ -33,6 +81,10 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { stdio: 'inherit', cwd, }; + const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); execSync( `${containerCmd} build ` + '--file=tools/visual-regression-testing/testing.Dockerfile ' + @@ -43,6 +95,7 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + + `--env=BRANCH="${branch}"` + `--volume=./dist/screenshots:/dist/screenshots ` + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + tag, diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 9030e4d482..e127d69852 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -1,7 +1,13 @@ -import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; -import { dirname, extname, join } from 'path'; +import { execSync } from 'child_process'; +import { createHash } from 'crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, extname } from 'path'; -const baselineUrl = process.env.CI +const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); +const baselineUrl = process.env.GITHUB_ACTIONS ? 'http://localhost:8050/' : 'https://lyne-visual-regression-baseline.app.sbb.ch/'; @@ -10,51 +16,48 @@ export const visualRegressionConfig = (update) => ({ update, baseDir: 'dist/screenshots', - async getBaseline({ filePath, baseDir, name }) { - if (existsSync(filePath)) { - return readFileSync(filePath); - } - - const cacheFile = join(baseDir, '.cache', name + extname(filePath)); - const cacheFileDetails = cacheFile + '.json'; - mkdirSync(dirname(cacheFile), { recursive: true }); + async getBaseline({ filePath, name }) { const baselineFileUrl = baselineUrl + name + extname(filePath); - const downloadFile = async () => { - try { - const response = await fetch(baselineFileUrl); - - if (response.ok) { - writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); - writeFileSync( - cacheFileDetails, - JSON.stringify({ etag: response.headers.get('etag') }, null, 2), - 'utf8', - ); - - return readFileSync(cacheFile); - } - } catch { - /* empty */ - } - }; - - if (existsSync(cacheFileDetails)) { - const details = JSON.parse(readFileSync(cacheFileDetails)); + const infoFilePath = filePath + '.json'; + const info = existsSync(infoFilePath) ? JSON.parse(readFileSync(infoFilePath, 'utf8')) : {}; + if (existsSync(filePath) && info.branch === branch) { + return readFileSync(filePath); + } else if (existsSync(filePath) && info.etag) { const response = await fetch(baselineFileUrl, { method: 'HEAD', - headers: { 'if-none-match': details.etag }, + headers: { 'if-none-match': info.etag }, }); - if (response.status === 200) { - return await downloadFile(); + + if (response.status === 304) { + return readFileSync(filePath); } else if (response.status === 404) { - [cacheFile, cacheFileDetails].forEach(unlinkSync); - } else if (response.status === 304) { - return readFileSync(cacheFile); - } else { - console.error(`Unexpected response from baseline service: ${response.status} (${name})`); + return undefined; } - } else { - return await downloadFile(); } + + // If the image address is not reachable, fetch throws, so we wrap it in try/catch. + try { + const response = await fetch(baselineFileUrl); + if (response.ok) { + const etag = response.headers.get('etag'); + const buffer = Buffer.from(await response.arrayBuffer()); + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, buffer); + writeFileSync(infoFilePath, JSON.stringify({ etag }, null, 2), 'utf8'); + + return buffer; + } + } catch (e) { + return undefined; + } + }, + saveBaseline({ filePath, content }) { + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, content); + writeFileSync( + filePath + '.json', + JSON.stringify({ branch, etag: createHash('sha256').update(content).digest('hex') }), + 'utf8', + ); }, }); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index eb8101b0bf..209a2289e2 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -22,6 +22,11 @@ const webkit = process.argv.includes('--webkit'); const concurrency = process.argv.includes('--parallel') ? {} : { concurrency: 1 }; const updateBaseImages = process.argv.includes('--update-visual-baseline') || process.argv.includes('--uv'); +const visualRegressionRun = + process.argv.includes('--group=visual-regression') || + (process.argv.includes('--group') + ? process.argv[process.argv.indexOf('--group') + 1] === 'visual-regression' + : false); const stylesCompiler = new sass.initCompiler(); const renderStyles = () => @@ -86,15 +91,21 @@ const suppressedLogs = [ '[vite] connecting...', ]; +const groups = [ + // Disable ssr tests until stabilized. + // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, + // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, +]; + +// The visual regression test group is only added when expicitely set, as the tests are very expensive. +if (visualRegressionRun) { + groups.push({ name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }); +} + /** @type {import('@web/test-runner').TestRunnerConfig} */ export default { files: ['src/**/*.{e2e,spec,!snapshot.spec}.ts'], - groups: [ - // Disable ssr tests until stabilized. - // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, - // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, - { name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }, - ], + groups, nodeResolve: true, concurrency: resolveConcurrency(), reporters: From ba55143513c92bfcf6ba426046b0a900a8981579 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Fri, 26 Apr 2024 17:36:11 +0200 Subject: [PATCH 26/48] fix: lint --- src/components/accordion/accordion.e2e.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/accordion/accordion.e2e.ts b/src/components/accordion/accordion.e2e.ts index a01e2ff39c..be20fbb172 100644 --- a/src/components/accordion/accordion.e2e.ts +++ b/src/components/accordion/accordion.e2e.ts @@ -2,8 +2,8 @@ import { assert, expect } from '@open-wc/testing'; import { nothing } from 'lit'; import { html } from 'lit/static-html.js'; -import { fixture } from '../core/testing/private.js'; -import { waitForCondition, waitForLitRender, EventSpy, isSsr } from '../core/testing.js'; +import { fixture, isSsr } from '../core/testing/private.js'; +import { waitForCondition, waitForLitRender, EventSpy } from '../core/testing.js'; import { SbbExpansionPanelElement, type SbbExpansionPanelHeaderElement, From 07b2c92c9bc3e7717e03a48126e27816349d9a83 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Wed, 1 May 2024 17:01:22 +0200 Subject: [PATCH 27/48] fix: fix screenshot sizes --- .../testing/private/describe-viewports.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts index de764fe737..6c538dfab1 100644 --- a/src/components/core/testing/private/describe-viewports.ts +++ b/src/components/core/testing/private/describe-viewports.ts @@ -1,29 +1,29 @@ import { - SbbBreakpointLargeMax, - SbbBreakpointMediumMax, - SbbBreakpointMicroMax, - SbbBreakpointSmallMax, - SbbBreakpointUltraMax, - SbbBreakpointWideMax, - SbbBreakpointZeroMax, + SbbBreakpointLargeMin, + SbbBreakpointMediumMin, + SbbBreakpointMicroMin, + SbbBreakpointSmallMin, + SbbBreakpointUltraMin, + SbbBreakpointWideMin, + SbbBreakpointZeroMin, } from '@sbb-esta/lyne-design-tokens'; import { setViewport } from '@web/test-runner-commands'; const viewportSizes = { - zero: SbbBreakpointZeroMax, - micro: SbbBreakpointMicroMax, - small: SbbBreakpointSmallMax, - medium: SbbBreakpointMediumMax, - large: SbbBreakpointLargeMax, - wide: SbbBreakpointWideMax, - ultra: SbbBreakpointUltraMax, + zero: SbbBreakpointZeroMin, + micro: SbbBreakpointMicroMin, + small: SbbBreakpointSmallMin, + medium: SbbBreakpointMediumMin, + large: SbbBreakpointLargeMin, + wide: SbbBreakpointWideMin, + ultra: SbbBreakpointUltraMin, }; -export function describeViewports(fn: (this: Mocha.Suite) => void): void { +export function describeViewports(fn: (this: Mocha.Suite) => void, viewportHeight = 400): void { for (const [size, value] of Object.entries(viewportSizes)) { describe(`viewport=${size}`, function () { before(async () => { - await setViewport({ width: value, height: 400 }); + await setViewport({ width: value, height: viewportHeight }); }); fn.call(this); From 7ba55ff3ca2618b35e4ee1b36532333f233f93ed Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 11:47:28 +0200 Subject: [PATCH 28/48] feat: diff-app first version --- eslint.config.js | 7 + package.json | 1 + .../diff-app/index.html | 3 +- .../src/components/overview/overview.scss | 9 + .../src/components/overview/overview.ts | 72 ++++++ .../fullscreen-diff/fullscreen-diff.scss | 21 ++ .../fullscreen-diff/fullscreen-diff.ts | 58 +++++ .../test-case/image-diff/image-diff.scss | 69 ++++++ .../test-case/image-diff/image-diff.ts | 147 ++++++++++++ .../test-case-filter/test-case-filter.scss | 14 ++ .../test-case-filter/test-case-filter.ts | 84 +++++++ .../src/components/test-case/test-case.scss | 49 ++++ .../src/components/test-case/test-case.ts | 123 ++++++++++ .../diff-app/src/main.ts | 10 +- .../diff-app/src/routes.ts | 15 ++ .../diff-app/src/screenshots.ts | 218 ++++++++++++++++++ .../diff-app/src/vite-env.d.ts | 8 +- .../diff-app/vite.config.ts | 2 +- yarn.lock | 25 ++ 19 files changed, 927 insertions(+), 8 deletions(-) create mode 100644 tools/visual-regression-testing/diff-app/src/components/overview/overview.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/overview/overview.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss create mode 100644 tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts create mode 100644 tools/visual-regression-testing/diff-app/src/routes.ts create mode 100644 tools/visual-regression-testing/diff-app/src/screenshots.ts diff --git a/eslint.config.js b/eslint.config.js index 673978b295..1daf442a54 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -47,6 +47,13 @@ export default [ 'plugin:import-x/typescript', ), eslintPluginLyne.default.configs.recommended, + { + files: ['tools/visual-regression-testing/diff-app/**/*.ts'], + rules: { + 'lyne/custom-element-class-name-rule': 'off', + 'import-x/namespace': 'off', + }, + }, { files: ['**/*.ts'], rules: { diff --git a/package.json b/package.json index 0246fe0d98..02712ad005 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "@types/react-dom": "18.3.0", "@typescript-eslint/eslint-plugin": "7.8.0", "@typescript-eslint/parser": "7.8.0", + "@vaadin/router": "1.7.5", "@web/test-runner": "0.18.2", "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", diff --git a/tools/visual-regression-testing/diff-app/index.html b/tools/visual-regression-testing/diff-app/index.html index b6bc0adf39..54182ed5c0 100644 --- a/tools/visual-regression-testing/diff-app/index.html +++ b/tools/visual-regression-testing/diff-app/index.html @@ -2,6 +2,7 @@ Visual Regression Tests Comparison + - Hi +
diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.scss b/tools/visual-regression-testing/diff-app/src/components/overview/overview.scss new file mode 100644 index 0000000000..0458d025a8 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.scss @@ -0,0 +1,9 @@ +@use '../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +.app-overview { + display: flex; + gap: 1rem; + flex-direction: column; +} diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts new file mode 100644 index 0000000000..8e1a9032e7 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts @@ -0,0 +1,72 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement } from 'lit/decorators.js'; + +import { screenshotService } from '../../screenshots.js'; + +import style from './overview.scss?lit&inline'; + +import '../../../../../../src/components/accordion.js'; +import '../../../../../../src/components/button/secondary-button-link.js'; +import '../../../../../../src/components/card.js'; +import '../../../../../../src/components/container.js'; +import '../../../../../../src/components/expansion-panel.js'; +import '../../../../../../src/components/link-list.js'; +import '../../../../../../src/components/link/block-link.js'; +import '../../../../../../src/components/title.js'; + +/** + * Overview over all failed or new tests + */ +@customElement('app-overview') +export class Overview extends LitElement { + public static override styles: CSSResultGroup = style; + + public override render(): TemplateResult { + return html` + + Lyne visual regression comparison +
+ + ${screenshotService.screenshots.stats} + + Start comparing + + + + ${screenshotService.screenshots.components.map( + (screenshotComponent) => html` + + + ${screenshotComponent.name} (${screenshotComponent.stats}) + + + + ${screenshotComponent.testCases.map( + (entry) => + html` + ${entry.name} + `, + )} + + + + `, + )} + +
+ + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-overview': Overview; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss new file mode 100644 index 0000000000..3af3e700ae --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.scss @@ -0,0 +1,21 @@ +@use '../../../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +:host { + display: block; +} + +.app-labels { + text-transform: capitalize; +} + +.app-radio-button-group { + margin-block-end: 0.5rem; +} + +.app-scroll-container { + @include sbb.scrollbar; + + overflow: auto; +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts new file mode 100644 index 0000000000..5e6ef0208a --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/fullscreen-diff/fullscreen-diff.ts @@ -0,0 +1,58 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup, nothing } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; + +import type { SbbRadioButtonGroupElement } from '../../../../../../../../src/components/radio-button/radio-button-group/radio-button-group.js'; +import type { ScreenshotFailedFiles } from '../../../../screenshots.js'; + +import style from './fullscreen-diff.scss?lit&inline'; + +import '../../../../../../../../src/components/chip.js'; +import '../../../../../../../../src/components/radio-button.js'; + +/** + * Displays two images in fullscreen to overlay them. + */ +@customElement('app-fullscreen-diff') +export class FullscreenDiff extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public failedFile?: ScreenshotFailedFiles; + + @property() public selectedFile: 'baselineFile' | 'failedFile' | 'diffFile' = 'failedFile'; + + public override render(): TemplateResult { + if (!this.failedFile) { + return html``; + } + return html`
+ ${this.failedFile.browserName} + ${this.failedFile.viewport} +
+ + (this.selectedFile = (event.target as SbbRadioButtonGroupElement).value)} + > + ${!this.failedFile.isNew + ? html`Baseline` + : nothing} + ${this.failedFile.isNew ? 'New' : 'Failed'} + ${this.failedFile.diffFile + ? html`Diff` + : nothing} + +
+ +
`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-fullscreen-diff': FullscreenDiff; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss new file mode 100644 index 0000000000..9359cd30ff --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss @@ -0,0 +1,69 @@ +@use '../../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +:host { + display: block; +} + +.app-container { + display: flex; + flex-direction: column; + gap: var(--sbb-spacing-fixed-1x); +} + +.app-info-bar { + display: flex; + justify-content: space-between; +} + +.app-labels { + text-transform: capitalize; +} + +.app-diff-toggle { + white-space: nowrap; + align-self: center; +} + +.app-image-container { + width: 100%; + display: flex; + align-items: start; + justify-content: stretch; + flex-direction: column; + gap: var(--sbb-spacing-fixed-1x); + + @include sbb.mq($from: small) { + flex-direction: row; + } +} + +.app-image-baseline, +.app-image-failed { + display: flex; + flex: 1; + width: 100%; + background-color: var(--sbb-color-white); +} + +.app-image { + max-width: 100%; +} + +.app-new-test-case-info { + margin: 0.5rem; +} + +.app-image-button { + @include sbb.button-reset; + + display: block; + width: 100%; + text-align: left; + cursor: pointer; + + &:focus-visible { + @include sbb.focus-outline; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts new file mode 100644 index 0000000000..cb0797fa53 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts @@ -0,0 +1,147 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup, nothing } from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; + +import { SbbOverlayElement } from '../../../../../../../src/components/overlay/overlay.js'; +import type { SbbToggleCheckElement } from '../../../../../../../src/components/toggle-check/toggle-check.js'; +import type { ScreenshotFailedFiles } from '../../../screenshots.js'; + +import style from './image-diff.scss?lit&inline'; + +import '../../../../../../../src/components/chip.js'; +import '../../../../../../../src/components/status.js'; +import '../../../../../../../src/components/overlay.js'; +import '../../../../../../../src/components/toggle-check.js'; + +import './fullscreen-diff/fullscreen-diff.js'; + +const getImageDimension = (img: HTMLImageElement): string => + `${img.naturalWidth}x${img.naturalHeight}px`; + +/** + * Displays two images to compare them. + */ +@customElement('app-image-diff') +export class ImageDiff extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public failedFile?: ScreenshotFailedFiles; + + @state() private _baselineDimension?: string; + @state() private _failedDimension?: string; + + @state() private _showDiff: boolean = true; + + private _toggleDiff(event: Event): void { + this._showDiff = (event.target as SbbToggleCheckElement).checked; + } + + private _setFailedImageDimension(event: Event): void { + this._failedDimension = getImageDimension(event.target as HTMLImageElement); + } + + private _setBaselineImageDimension(event: Event): void { + this._baselineDimension = getImageDimension(event.target as HTMLImageElement); + } + + /** + * To avoid blown up DOM, we create the overlay only when it's needed. + */ + private _showFullscreen(selectedFile: 'baselineFile' | 'failedFile' | 'diffFile'): void { + const sbbOverlayElement: SbbOverlayElement = document.createElement('sbb-overlay'); + const appFullscreenDiff = document.createElement('app-fullscreen-diff'); + + appFullscreenDiff.selectedFile = selectedFile; + appFullscreenDiff.failedFile = this.failedFile; + + sbbOverlayElement.appendChild(appFullscreenDiff); + document.body.appendChild(sbbOverlayElement); + sbbOverlayElement.addEventListener(SbbOverlayElement.events.didClose, () => { + document.body.removeChild(sbbOverlayElement); + }); + + sbbOverlayElement.open(); + } + + public override render(): TemplateResult { + if (!this.failedFile) { + return html``; + } + return html`
+
+
+ ${this.failedFile.browserName} + ${this.failedFile.viewport} + ${this._baselineDimension + ? html` + Baseline: ${this._baselineDimension} + ` + : nothing} + + ${this.failedFile.isNew ? 'New' : 'Failed'}: ${this._failedDimension} + +
+ ${!this.failedFile.isNew + ? html` + Show Diff + ` + : nothing} +
+
+
+ ${!this.failedFile.isNew + ? html`` + : html` + New test case + `} +
+
+ ${this._showDiff && !this.failedFile.isNew + ? html`` + : html``} +
+
+
`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-image-diff': ImageDiff; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss new file mode 100644 index 0000000000..ecae9f0162 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.scss @@ -0,0 +1,14 @@ +@use '../../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +.app-test-case-filter { + display: flex; + gap: 0.5rem 4rem; + flex-direction: column; + text-transform: capitalize; +} + +sbb-title { + margin: 0; +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts new file mode 100644 index 0000000000..86057eeb67 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts @@ -0,0 +1,84 @@ +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; + +import type { SbbTagElement } from '../../../../../../../src/components/tag/tag/tag.js'; +import { type ScreenshotTestCase } from '../../../screenshots.js'; +import '../../../../../../../src/components/title.js'; +import '../../../../../../../src/components/tag.js'; + +import style from './test-case-filter.scss?lit&inline'; + +/** + * Shows filter for viewports and browsers + */ +@customElement('app-test-case-filter') +export class TestCaseFilter extends LitElement { + public static override styles: CSSResultGroup = style; + + @property() public testCase?: ScreenshotTestCase; + + private _handleViewportChange(event: CustomEvent): void { + this.dispatchEvent( + new CustomEvent('viewportFilterChange', { + bubbles: true, + composed: true, + detail: (event.target as SbbTagElement).value, + }), + ); + } + + private _handleBrowserChange(event: CustomEvent): void { + this.dispatchEvent( + new CustomEvent('browserFilterChange', { + bubbles: true, + composed: true, + detail: (event.target as SbbTagElement).value, + }), + ); + } + + public override render(): TemplateResult { + return html` +
+
+ Viewports + + + All + + ${this.testCase?.viewports?.map( + (viewport) => html` + + ${viewport.name} + + `, + )} + +
+
+ Browsers + + + All + + ${this.testCase?.availableBrowserNames?.map( + (browserName) => html`${browserName}`, + )} + +
+
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-test-case-filter': TestCaseFilter; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss new file mode 100644 index 0000000000..cd90c65dcd --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.scss @@ -0,0 +1,49 @@ +@use '../../../../../../src/components/core/styles/index' as sbb; + +@include sbb.box-sizing; + +sbb-chip { + width: fit-content; +} + +sbb-title { + margin: 0; +} + +.app-file-name-box { + display: flex; + flex-direction: column; +} + +.app-file-name-ellipsis { + display: block; + @include sbb.ellipsis; +} + +.app-navigation-block { + display: flex; + gap: 0.5rem; + align-items: center; +} + +.app-image-diffs { + display: flex; + flex-direction: column; + gap: 2rem; + padding-block: 1rem; +} + +.app-testcase { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.app-progress { + position: absolute; + inset-block-start: 0; + inset-inline-start: 0; + height: var(--sbb-border-radius-2x); + background-color: var(--sbb-color-black); + width: calc(var(--app-progress) * 100%); +} diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts new file mode 100644 index 0000000000..b665bdaf43 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts @@ -0,0 +1,123 @@ +import type { BeforeEnterObserver, RouterLocation } from '@vaadin/router'; +import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; + +import { screenshotService, type ScreenshotTestCase } from '../../screenshots.js'; + +import '../../../../../../src/components/button/secondary-button-link.js'; +import '../../../../../../src/components/chip.js'; +import '../../../../../../src/components/container.js'; +import '../../../../../../src/components/header.js'; +import '../../../../../../src/components/notification.js'; +import '../../../../../../src/components/title.js'; + +import style from './test-case.scss?lit&inline'; + +import './test-case-filter/test-case-filter.js'; +import './image-diff/image-diff.js'; + +interface Filter { + viewport?: string; + browser?: string; +} + +/** + * Displays a test case with its images. + * Provides filtering functions. + */ +@customElement('app-test-case') +export class TestCase extends LitElement implements BeforeEnterObserver { + public static override styles: CSSResultGroup = style; + + @state() private _testCaseName?: string; + @state() private _componentName?: string; + @state() private _testCase?: ScreenshotTestCase; + @state() private _filter: Filter = {}; + + // Called by router + public onBeforeEnter(location: RouterLocation): void { + this._testCaseName = location.params.testcase as string; + this._componentName = location.params.component as string; + + this._testCase = screenshotService.setCurrentTestCase(this._componentName, this._testCaseName); + } + + private _viewportFilterChanged(event: CustomEvent): void { + this._filter = { + ...this._filter, + viewport: event.detail && event.detail !== 'all' ? event.detail : undefined, + }; + } + + private _browserFilterChanged(event: CustomEvent): void { + this._filter = { + ...this._filter, + browser: event.detail && event.detail !== 'all' ? event.detail : undefined, + }; + } + + public override render(): TemplateResult { + return html` + +
+
+ ${this._componentName} + + ${this._testCaseName} + +
+
+
+ Overview + + +
+ + ${this._testCase + ? html`
+ + + + +
+ ${this._testCase + ?.filter(this._filter.viewport, this._filter.browser) + .map( + (failedFile) => + html``, + )} +
+ +
` + : html` + + No screenshots found. Please check component and test case name. + + `} + `; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-test-case': TestCase; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts index efabec6e06..3cf7b340e6 100644 --- a/tools/visual-regression-testing/diff-app/src/main.ts +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -1,8 +1,8 @@ -import '../../../../src/components/title.js'; +import { Router } from '@vaadin/router'; -import '../../../../src/components/core/styles/global.scss'; +import { routes } from './routes.js'; -// eslint-disable-next-line import-x/no-unresolved -import { screenshots } from 'virtual:screenshots'; +import '../../../../src/components/core/styles/standard-theme.scss'; -console.log(screenshots); +export const router = new Router(document.querySelector('#outlet')); +router.setRoutes(routes); diff --git a/tools/visual-regression-testing/diff-app/src/routes.ts b/tools/visual-regression-testing/diff-app/src/routes.ts new file mode 100644 index 0000000000..5fa9bfe2fb --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/routes.ts @@ -0,0 +1,15 @@ +import type { Route } from '@vaadin/router'; + +import './components/overview/overview.js'; +import './components/test-case/test-case.js'; + +export const routes: Route[] = [ + { + path: '/', + component: 'app-overview', + }, + { + path: '/compare/:component/:testcase', + component: 'app-test-case', + }, +]; diff --git a/tools/visual-regression-testing/diff-app/src/screenshots.ts b/tools/visual-regression-testing/diff-app/src/screenshots.ts new file mode 100644 index 0000000000..71c49eb205 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/src/screenshots.ts @@ -0,0 +1,218 @@ +// eslint-disable-next-line import-x/no-unresolved +import { screenshotsRaw } from 'virtual:screenshots'; + +import type { FailedFiles } from '../vite.config.js'; + +const viewportOrder = ['zero', 'micro', 'small', 'medium', 'large', 'wide', 'ultra']; + +// TODO: discuss whether to include it in creation of screenshotsRaw +const extractHierarchicalMap = ( + screenshots: Record, +): Map>> => { + const map = new Map>>(); + + Object.entries(screenshots).forEach(([fileName, failedFiles]) => { + const component = fileName.match(/^(.*?)_/)![1]; + const name = fileName.match(/_viewport=.*?_(.*?).png$/)![1]; + const viewport = fileName.match(/viewport=(.*?)_/)![1]; + + if (!map.has(component)) { + map.set(component, new Map()); + } + + const componentsMap = map.get(component)!; + + if (!componentsMap.has(name)) { + componentsMap.set(name, new Map()); + } + + const testCaseMap = componentsMap.get(name)!; + + testCaseMap.set( + viewport, + failedFiles.map((failedFile) => ({ ...failedFile, viewport })), + ); + }); + return map; +}; + +export interface ScreenshotFailedFiles extends FailedFiles { + viewport: string; +} + +export class ScreenshotStatistics { + public static fromFailedFiles(failedFiles: ScreenshotFailedFiles[]): ScreenshotStatistics { + return failedFiles.reduce( + (current, next) => + current.sum(new ScreenshotStatistics(next.isNew ? 0 : 1, next.isNew ? 1 : 0)), + new ScreenshotStatistics(0, 0), + ); + } + + public static fromList(list: { stats: ScreenshotStatistics }[]): ScreenshotStatistics { + return list.reduce((current, next) => current.sum(next.stats), new ScreenshotStatistics(0, 0)); + } + + public constructor( + public readonly failedTests: number, + public readonly newTests: number, + ) {} + + public sum(other: ScreenshotStatistics): ScreenshotStatistics { + return new ScreenshotStatistics( + this.failedTests + other.failedTests, + this.newTests + other.newTests, + ); + } + + public toString(): string { + return `${this.failedTests} failed, ${this.newTests} new`; + } +} + +export class ScreenshotViewport { + public readonly stats: ScreenshotStatistics; + public readonly browserNames: string[]; + + public constructor( + public readonly name: string, + public readonly browsers: ScreenshotFailedFiles[], + ) { + this.stats = ScreenshotStatistics.fromFailedFiles(this.browsers); + + this.browserNames = this.browsers.map((browser) => browser.browserName); + } + + /** Compare by respecting defined viewport order. */ + public compare(other: ScreenshotViewport): number { + return viewportOrder.indexOf(this.name) - viewportOrder.indexOf(other.name); + } +} + +export class ScreenshotTestCase { + public readonly stats: ScreenshotStatistics; + public readonly availableBrowserNames: string[]; + public readonly path: string; + + public constructor( + public readonly component: string, + public readonly name: string, + public readonly viewports: ScreenshotViewport[], + ) { + this.stats = ScreenshotStatistics.fromList(this.viewports); + this.path = `${this.component}/${this.name}`; + + this.availableBrowserNames = Array.from( + this.viewports.reduce((current, next) => { + next.browserNames.forEach((browserName) => current.add(browserName)); + return current; + }, new Set()), + ); + } + + public filter(viewport?: string, browser?: string): ScreenshotFailedFiles[] { + return this.viewports + .filter((entry) => !viewport || entry.name === viewport) + .flatMap((entry) => + entry.browsers.filter((failedFiles) => !browser || failedFiles.browserName === browser), + ); + } +} + +export class ScreenshotComponent { + public readonly stats: ScreenshotStatistics; + + public constructor( + public readonly name: string, + public readonly testCases: ScreenshotTestCase[], + ) { + this.stats = ScreenshotStatistics.fromList(this.testCases); + } +} + +export class Screenshots { + public readonly components: ScreenshotComponent[]; + public readonly stats: ScreenshotStatistics; + public readonly testCaseCount: number; + public readonly flatTestCases: ScreenshotTestCase[]; + + public constructor(screenshots: Record) { + const flatTestCases: ScreenshotTestCase[] = []; + + // Convert hierarchical screenshot map to classes + this.components = Array.from(extractHierarchicalMap(screenshots).entries()).map( + ([componentName, testCases]) => + new ScreenshotComponent( + componentName, + Array.from(testCases.entries()).map(([testCase, viewports]) => { + const screenshotTestCase = new ScreenshotTestCase( + componentName, + testCase, + Array.from(viewports.entries()) + .map(([viewport, entries]) => new ScreenshotViewport(viewport, entries)) + .sort((a: ScreenshotViewport, b: ScreenshotViewport) => a.compare(b)), + ); + flatTestCases.push(screenshotTestCase); + return screenshotTestCase; + }), + ), + ); + + this.flatTestCases = flatTestCases; + this.testCaseCount = this.flatTestCases.length; + this.stats = ScreenshotStatistics.fromList(this.components); + } + + public indexOfTestCase(componentName: string, testCaseName: string): number { + return this.flatTestCases.findIndex( + (component) => component.component === componentName && component.name === testCaseName, + ); + } + + public getByTestCaseIndex(index: number): ScreenshotTestCase | undefined { + return this.flatTestCases[index]; + } +} + +export class ScreenshotService { + public readonly screenshots; + private _currentIndex: number = -1; + + public get progressFraction(): number { + return (this._currentIndex + 1) / this.screenshots.testCaseCount; + } + + public get current(): ScreenshotTestCase | undefined { + return this._current; + } + private _current?: ScreenshotTestCase; + + public constructor(screenshots: Screenshots) { + this.screenshots = screenshots; + this._currentIndex = 0; + this._current = screenshots.getByTestCaseIndex(this._currentIndex); + } + + public get next(): ScreenshotTestCase | undefined { + return this.screenshots.getByTestCaseIndex(this._currentIndex + 1); + } + + public get previous(): ScreenshotTestCase | undefined { + return this.screenshots.getByTestCaseIndex(this._currentIndex - 1); + } + + public setCurrentTestCase( + componentName: string, + testCaseName: string, + ): ScreenshotTestCase | undefined { + const testCaseIndex = this.screenshots.indexOfTestCase(componentName, testCaseName); + + this._currentIndex = testCaseIndex; + if (testCaseIndex >= 0) { + this._current = this.screenshots.getByTestCaseIndex(testCaseIndex); + return this._current; + } + } +} + +export const screenshotService = new ScreenshotService(new Screenshots(screenshotsRaw)); diff --git a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts index 8b67858995..d6c5e3c360 100644 --- a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts +++ b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts @@ -2,5 +2,11 @@ declare module 'virtual:screenshots' { // eslint-disable-next-line @typescript-eslint/consistent-type-imports - export const screenshots: Record; + export const screenshotsRaw: Record; +} + +declare module '*?lit&inline' { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const src: import('lit').CSSResultGroup; + export default src; } diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts index 981c615246..3cadf5763d 100644 --- a/tools/visual-regression-testing/diff-app/vite.config.ts +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -138,7 +138,7 @@ function prepareScreenshots(): PluginOption { }); } - return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; + return `export const screenshotsRaw = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; } }, configureServer(server) { diff --git a/yarn.lock b/yarn.lock index e080350c88..740ae870e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3137,6 +3137,26 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== +"@vaadin/router@1.7.5": + version "1.7.5" + resolved "https://registry.yarnpkg.com/@vaadin/router/-/router-1.7.5.tgz#af4b8f05a86e2890ea875f9bca9b406d74f9bae4" + integrity sha512-uRN3vd1ihgd596bF/NMZqpgxau0nlvIc0/JDd1EwStFNbZID/xIVse5LXdQhIyUKLmSl4T0GeCQK505xerWX0w== + dependencies: + "@vaadin/vaadin-usage-statistics" "^2.1.0" + path-to-regexp "2.4.0" + +"@vaadin/vaadin-development-mode-detector@^2.0.0": + version "2.0.6" + resolved "https://registry.yarnpkg.com/@vaadin/vaadin-development-mode-detector/-/vaadin-development-mode-detector-2.0.6.tgz#2acb568975c18b7965e56ed765638dcf183d8dd1" + integrity sha512-N6a5nLT/ytEUlpPo+nvdCKIGoyNjPsj3rzPGvGYK8x9Ceg76OTe1xI/GtN71mRW9e2HUScR0kCNOkl1Z63YDjw== + +"@vaadin/vaadin-usage-statistics@^2.1.0": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@vaadin/vaadin-usage-statistics/-/vaadin-usage-statistics-2.1.2.tgz#a0a211b298e647742b60c6d4d1d8834381e3c77f" + integrity sha512-xKs1PvRfTXsG0eWWcImLXWjv7D+f1vfoIvovppv6pZ5QX8xgcxWUdNgERlOOdGt3CTuxQXukTBW3+Qfva+OXSg== + dependencies: + "@vaadin/vaadin-development-mode-detector" "^2.0.0" + "@vitest/expect@1.3.1": version "1.3.1" resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.3.1.tgz#d4c14b89c43a25fd400a6b941f51ba27fe0cb918" @@ -8868,6 +8888,11 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== +path-to-regexp@2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" + integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== + path-to-regexp@^6.2.1: version "6.2.2" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.2.tgz#324377a83e5049cbecadc5554d6a63a9a4866b36" From 3c028ffa0d94b6ae97d5d001045293be2d3c373c Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 14:42:45 +0200 Subject: [PATCH 29/48] fix: a11y cleanup --- src/components/core/testing/private/a11y-tree-snapshot.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/core/testing/private/a11y-tree-snapshot.ts b/src/components/core/testing/private/a11y-tree-snapshot.ts index db975d0875..57e4c576ba 100644 --- a/src/components/core/testing/private/a11y-tree-snapshot.ts +++ b/src/components/core/testing/private/a11y-tree-snapshot.ts @@ -1,4 +1,4 @@ -import { aTimeout, expect, fixture } from '@open-wc/testing'; +import { aTimeout, expect, fixture, fixtureCleanup } from '@open-wc/testing'; import { a11ySnapshot } from '@web/test-runner-commands'; import type { TemplateResult } from 'lit'; import { html } from 'lit/static-html.js'; @@ -18,6 +18,7 @@ async function a11yTreeEqualSnapshot(): Promise { const htmlWrapper = await fixture(html`

${JSON.stringify(snapshot, null, 2)}

`); await expect(htmlWrapper).to.be.equalSnapshot(); + fixtureCleanup(); } /** From e5339e58c5568777621fe22ab8c351074cb989cc Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 15:37:16 +0200 Subject: [PATCH 30/48] fix: fix viewport --- src/components/core/testing/private/describe-viewports.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts index 6c538dfab1..09364e05dd 100644 --- a/src/components/core/testing/private/describe-viewports.ts +++ b/src/components/core/testing/private/describe-viewports.ts @@ -5,12 +5,11 @@ import { SbbBreakpointSmallMin, SbbBreakpointUltraMin, SbbBreakpointWideMin, - SbbBreakpointZeroMin, } from '@sbb-esta/lyne-design-tokens'; import { setViewport } from '@web/test-runner-commands'; const viewportSizes = { - zero: SbbBreakpointZeroMin, + zero: 320, micro: SbbBreakpointMicroMin, small: SbbBreakpointSmallMin, medium: SbbBreakpointMediumMin, From 886c82d369c7adeff3398692cf304e2a1cca9c9c Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 15:59:16 +0200 Subject: [PATCH 31/48] fix: fix diff app image size --- .../test-case/image-diff/image-diff.scss | 6 ++- .../test-case/image-diff/image-diff.ts | 43 +++++++++---------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss index 9359cd30ff..bf876ea77d 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.scss @@ -58,7 +58,7 @@ .app-image-button { @include sbb.button-reset; - display: block; + display: flex; width: 100%; text-align: left; cursor: pointer; @@ -66,4 +66,8 @@ &:focus-visible { @include sbb.focus-outline; } + + &[hidden] { + display: none; + } } diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts index cb0797fa53..411bb43fcd 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/image-diff/image-diff.ts @@ -50,6 +50,7 @@ export class ImageDiff extends LitElement { const sbbOverlayElement: SbbOverlayElement = document.createElement('sbb-overlay'); const appFullscreenDiff = document.createElement('app-fullscreen-diff'); + sbbOverlayElement.expanded = true; appFullscreenDiff.selectedFile = selectedFile; appFullscreenDiff.failedFile = this.failedFile; @@ -110,29 +111,25 @@ export class ImageDiff extends LitElement { `}
- ${this._showDiff && !this.failedFile.isNew - ? html`` - : html``} + +
`; From 5ad91a5aaa40c2d3e6c74554d3851b0cfce0982a Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 16:28:14 +0200 Subject: [PATCH 32/48] fix: activate sbb-icons in tests --- src/components/core/testing/test-setup.ts | 4 +++- web-test-runner.config.js | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index a795c33515..c1bf4bd5e1 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -27,7 +27,9 @@ function setupIconConfig(): void { mergeConfig({ icon }); } -setupIconConfig(); +if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { + setupIconConfig(); +} if (isHydratedSsr()) { await import('@lit-labs/ssr-client/lit-element-hydrate-support.js'); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 209a2289e2..5e4ee272ca 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -69,6 +69,7 @@ const testRunnerHtml = (testFramework, _config, group) => ` From 5ffacf302c6b71a658329cb37eb1986cc0ff2c03 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Thu, 2 May 2024 17:25:17 +0200 Subject: [PATCH 33/48] fix: activate sbb-icons in tests 2 --- src/components/core/testing/test-setup-ssr.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/core/testing/test-setup-ssr.ts b/src/components/core/testing/test-setup-ssr.ts index 7d08fd62df..035fcef3f4 100644 --- a/src/components/core/testing/test-setup-ssr.ts +++ b/src/components/core/testing/test-setup-ssr.ts @@ -26,4 +26,6 @@ function setupIconConfig(): void { }); } -setupIconConfig(); +if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { + setupIconConfig(); +} From 11fbe2a5c37a13758349a6ee9aeea63547115627 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Mon, 6 May 2024 12:06:23 +0200 Subject: [PATCH 34/48] fix: icon handling --- src/components/core/testing/test-setup-ssr.ts | 38 ++++++++++--------- src/components/core/testing/test-setup.ts | 4 +- web-test-runner.config.js | 1 - 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/components/core/testing/test-setup-ssr.ts b/src/components/core/testing/test-setup-ssr.ts index 035fcef3f4..76313d001f 100644 --- a/src/components/core/testing/test-setup-ssr.ts +++ b/src/components/core/testing/test-setup-ssr.ts @@ -1,31 +1,33 @@ +import { isServer } from 'lit'; + import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -function setupIconConfig(): void { - const testNamespaces = ['default', 'picto']; - const icon: SbbIconConfig = { - interceptor: ({ namespace, name, request }) => { - if (testNamespaces.includes(namespace)) { - const dimension = name.endsWith('-large') ? 48 : name.endsWith('-medium') ? 36 : 24; - return Promise.resolve( - ` { + if (testNamespaces.includes(namespace)) { + const dimension = name.endsWith('-large') ? 48 : name.endsWith('-medium') ? 36 : 24; + return Promise.resolve( + ` `, - ); - } - return request(); - }, - }; + ); + } + return request(); + }, + }; - mergeConfig({ - icon, - }); -} + mergeConfig({ + icon, + }); + } -if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { setupIconConfig(); } diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index c1bf4bd5e1..8489618e69 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -2,7 +2,7 @@ import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -import { isHydratedSsr } from './private.js'; +import { isHydratedSsr, isVisualRegressionRun } from './private.js'; function setupIconConfig(): void { const testNamespaces = ['default', 'picto']; @@ -27,7 +27,7 @@ function setupIconConfig(): void { mergeConfig({ icon }); } -if (!(globalThis as unknown as { isVisualRegressionRun: boolean }).isVisualRegressionRun) { +if (!isVisualRegressionRun()) { setupIconConfig(); } diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 5e4ee272ca..209a2289e2 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -69,7 +69,6 @@ const testRunnerHtml = (testFramework, _config, group) => ` From fd8a5d31558e2c71142412931d5e6a9236983507 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Mon, 6 May 2024 16:47:57 +0200 Subject: [PATCH 35/48] refactor: replace vaadin router by lit labs router --- package.json | 2 +- .../diff-app/index.html | 3 +- .../test-case-filter/test-case-filter.ts | 9 ++++ .../src/components/test-case/test-case.ts | 37 ++++++++++----- .../diff-app/src/main.ts | 47 +++++++++++++++++-- .../diff-app/src/routes.ts | 15 ------ yarn.lock | 32 +++---------- 7 files changed, 84 insertions(+), 61 deletions(-) delete mode 100644 tools/visual-regression-testing/diff-app/src/routes.ts diff --git a/package.json b/package.json index 02712ad005..486ab3a708 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "@custom-elements-manifest/to-markdown": "0.1.0", "@eslint/eslintrc": "3.0.2", "@eslint/js": "9.2.0", + "@lit-labs/router": "^0.1.3", "@lit-labs/testing": "0.2.3", "@lit/react": "1.0.5", "@open-wc/lit-helpers": "0.7.0", @@ -89,7 +90,6 @@ "@types/react-dom": "18.3.0", "@typescript-eslint/eslint-plugin": "7.8.0", "@typescript-eslint/parser": "7.8.0", - "@vaadin/router": "1.7.5", "@web/test-runner": "0.18.2", "@web/test-runner-commands": "0.9.0", "@web/test-runner-playwright": "0.11.0", diff --git a/tools/visual-regression-testing/diff-app/index.html b/tools/visual-regression-testing/diff-app/index.html index 54182ed5c0..2a80312eb1 100644 --- a/tools/visual-regression-testing/diff-app/index.html +++ b/tools/visual-regression-testing/diff-app/index.html @@ -26,8 +26,7 @@ /> -
- + diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts index 86057eeb67..c7509cae00 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case-filter/test-case-filter.ts @@ -17,6 +17,15 @@ export class TestCaseFilter extends LitElement { @property() public testCase?: ScreenshotTestCase; + /** + * Activate `all`-tag of viewports and browsers. + */ + public reset(): void { + this.shadowRoot!.querySelectorAll(`sbb-tag[value='all']`).forEach( + (tag) => (tag.checked = true), + ); + } + private _handleViewportChange(event: CustomEvent): void { this.dispatchEvent( new CustomEvent('viewportFilterChange', { diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts index b665bdaf43..79ae973a20 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts @@ -1,6 +1,11 @@ -import type { BeforeEnterObserver, RouterLocation } from '@vaadin/router'; -import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; -import { customElement, state } from 'lit/decorators.js'; +import { + LitElement, + html, + type TemplateResult, + type CSSResultGroup, + type PropertyValues, +} from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; import { screenshotService, type ScreenshotTestCase } from '../../screenshots.js'; @@ -11,6 +16,7 @@ import '../../../../../../src/components/header.js'; import '../../../../../../src/components/notification.js'; import '../../../../../../src/components/title.js'; +import type { TestCaseFilter } from './test-case-filter/test-case-filter.js'; import style from './test-case.scss?lit&inline'; import './test-case-filter/test-case-filter.js'; @@ -26,20 +32,25 @@ interface Filter { * Provides filtering functions. */ @customElement('app-test-case') -export class TestCase extends LitElement implements BeforeEnterObserver { +export class TestCase extends LitElement { public static override styles: CSSResultGroup = style; - @state() private _testCaseName?: string; - @state() private _componentName?: string; + @property() public params?: { componentName: string; testCaseName: string }; + @state() private _testCase?: ScreenshotTestCase; @state() private _filter: Filter = {}; - // Called by router - public onBeforeEnter(location: RouterLocation): void { - this._testCaseName = location.params.testcase as string; - this._componentName = location.params.component as string; + protected override willUpdate(changedProperties: PropertyValues): void { + super.willUpdate(changedProperties); - this._testCase = screenshotService.setCurrentTestCase(this._componentName, this._testCaseName); + if (changedProperties.has('params')) { + this._filter = {}; + this.shadowRoot!.querySelector('app-test-case-filter')?.reset(); + this._testCase = screenshotService.setCurrentTestCase( + this.params!.componentName!, + this.params!.testCaseName!, + ); + } } private _viewportFilterChanged(event: CustomEvent): void { @@ -64,9 +75,9 @@ export class TestCase extends LitElement implements BeforeEnterObserver { style="--app-progress: ${screenshotService.progressFraction}" >
- ${this._componentName} + ${this.params?.componentName} - ${this._testCaseName} + ${this.params?.testCaseName}
diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts index 3cf7b340e6..d89e69c28e 100644 --- a/tools/visual-regression-testing/diff-app/src/main.ts +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -1,8 +1,45 @@ -import { Router } from '@vaadin/router'; - -import { routes } from './routes.js'; +import { Router } from '@lit-labs/router'; +import { html, LitElement, type TemplateResult } from 'lit'; +import { customElement } from 'lit/decorators.js'; import '../../../../src/components/core/styles/standard-theme.scss'; -export const router = new Router(document.querySelector('#outlet')); -router.setRoutes(routes); +/** + * Main app containing the router outlet. + */ +@customElement('app-main') +export class Main extends LitElement { + private _router = new Router(this, [ + { + path: '/', + render: () => html``, + enter: async () => { + await import('./components/overview/overview.js'); + return true; + }, + }, + { + path: '/compare/:component/:testcase', + render: ({ component, testcase }) => + html``, + + enter: async () => { + await import('./components/test-case/test-case.js'); + return true; + }, + }, + ]); + + public override render(): TemplateResult { + return html`${this._router.outlet()}`; + } +} + +declare global { + interface HTMLElementTagNameMap { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'app-main': Main; + } +} diff --git a/tools/visual-regression-testing/diff-app/src/routes.ts b/tools/visual-regression-testing/diff-app/src/routes.ts deleted file mode 100644 index 5fa9bfe2fb..0000000000 --- a/tools/visual-regression-testing/diff-app/src/routes.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Route } from '@vaadin/router'; - -import './components/overview/overview.js'; -import './components/test-case/test-case.js'; - -export const routes: Route[] = [ - { - path: '/', - component: 'app-overview', - }, - { - path: '/compare/:component/:testcase', - component: 'app-test-case', - }, -]; diff --git a/yarn.lock b/yarn.lock index 740ae870e7..4120daad97 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1508,6 +1508,13 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@lit-labs/router@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@lit-labs/router/-/router-0.1.3.tgz#6be268eec6bbcbf0d28ee66688440cc46f587882" + integrity sha512-G+HHo57KsArG58LOI8DLtipFfC9tVV4lGaDy2I8hYQvS2P/pV5wQObrpFYPZswse8D47y8VuHNXNdVPQOVc5MA== + dependencies: + lit "^2.0.0 || ^3.0.0" + "@lit-labs/ssr-client@^1.1.4", "@lit-labs/ssr-client@^1.1.7": version "1.1.7" resolved "https://registry.yarnpkg.com/@lit-labs/ssr-client/-/ssr-client-1.1.7.tgz#fd0cf4ce986ee903dc67ad647193f2323ac15a6d" @@ -3137,26 +3144,6 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@vaadin/router@1.7.5": - version "1.7.5" - resolved "https://registry.yarnpkg.com/@vaadin/router/-/router-1.7.5.tgz#af4b8f05a86e2890ea875f9bca9b406d74f9bae4" - integrity sha512-uRN3vd1ihgd596bF/NMZqpgxau0nlvIc0/JDd1EwStFNbZID/xIVse5LXdQhIyUKLmSl4T0GeCQK505xerWX0w== - dependencies: - "@vaadin/vaadin-usage-statistics" "^2.1.0" - path-to-regexp "2.4.0" - -"@vaadin/vaadin-development-mode-detector@^2.0.0": - version "2.0.6" - resolved "https://registry.yarnpkg.com/@vaadin/vaadin-development-mode-detector/-/vaadin-development-mode-detector-2.0.6.tgz#2acb568975c18b7965e56ed765638dcf183d8dd1" - integrity sha512-N6a5nLT/ytEUlpPo+nvdCKIGoyNjPsj3rzPGvGYK8x9Ceg76OTe1xI/GtN71mRW9e2HUScR0kCNOkl1Z63YDjw== - -"@vaadin/vaadin-usage-statistics@^2.1.0": - version "2.1.2" - resolved "https://registry.yarnpkg.com/@vaadin/vaadin-usage-statistics/-/vaadin-usage-statistics-2.1.2.tgz#a0a211b298e647742b60c6d4d1d8834381e3c77f" - integrity sha512-xKs1PvRfTXsG0eWWcImLXWjv7D+f1vfoIvovppv6pZ5QX8xgcxWUdNgERlOOdGt3CTuxQXukTBW3+Qfva+OXSg== - dependencies: - "@vaadin/vaadin-development-mode-detector" "^2.0.0" - "@vitest/expect@1.3.1": version "1.3.1" resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.3.1.tgz#d4c14b89c43a25fd400a6b941f51ba27fe0cb918" @@ -8888,11 +8875,6 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== -path-to-regexp@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-to-regexp@^6.2.1: version "6.2.2" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.2.tgz#324377a83e5049cbecadc5554d6a63a9a4866b36" From 2414324355cfe1673a0582ea1ffcdede002a7b20 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Mon, 6 May 2024 21:10:16 +0200 Subject: [PATCH 36/48] refactor: remove service because obsolete since router refactoring Signed-off-by: Jeremias Peier --- .../src/components/overview/overview.ts | 12 +++--- .../src/components/test-case/test-case.ts | 42 ++++++++++++------ .../diff-app/src/screenshots.ts | 43 +------------------ 3 files changed, 35 insertions(+), 62 deletions(-) diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts index 8e1a9032e7..cf835c5219 100644 --- a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts @@ -1,7 +1,7 @@ import { LitElement, html, type TemplateResult, type CSSResultGroup } from 'lit'; import { customElement } from 'lit/decorators.js'; -import { screenshotService } from '../../screenshots.js'; +import { screenshots } from '../../screenshots.js'; import style from './overview.scss?lit&inline'; @@ -27,20 +27,18 @@ export class Overview extends LitElement { Lyne visual regression comparison
- ${screenshotService.screenshots.stats} + ${screenshots.stats} Start comparing - ${screenshotService.screenshots.components.map( + ${screenshots.components.map( (screenshotComponent) => html` - + ${screenshotComponent.name} (${screenshotComponent.stats}) diff --git a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts index 79ae973a20..000ecb9984 100644 --- a/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts +++ b/tools/visual-regression-testing/diff-app/src/components/test-case/test-case.ts @@ -1,13 +1,13 @@ import { - LitElement, - html, - type TemplateResult, type CSSResultGroup, + html, + LitElement, type PropertyValues, + type TemplateResult, } from 'lit'; import { customElement, property, state } from 'lit/decorators.js'; -import { screenshotService, type ScreenshotTestCase } from '../../screenshots.js'; +import { screenshots, type ScreenshotTestCase } from '../../screenshots.js'; import '../../../../../../src/components/button/secondary-button-link.js'; import '../../../../../../src/components/chip.js'; @@ -38,21 +38,40 @@ export class TestCase extends LitElement { @property() public params?: { componentName: string; testCaseName: string }; @state() private _testCase?: ScreenshotTestCase; + @state() private _testCaseIndex: number = -1; @state() private _filter: Filter = {}; protected override willUpdate(changedProperties: PropertyValues): void { super.willUpdate(changedProperties); if (changedProperties.has('params')) { + // Reset this._filter = {}; this.shadowRoot!.querySelector('app-test-case-filter')?.reset(); - this._testCase = screenshotService.setCurrentTestCase( + + // Get test case + this._testCaseIndex = screenshots.indexOfTestCase( this.params!.componentName!, this.params!.testCaseName!, ); + if (this._testCaseIndex >= 0) { + this._testCase = screenshots.getByTestCaseIndex(this._testCaseIndex); + } } } + private _progressFraction(): number { + return (this._testCaseIndex + 1) / screenshots.testCaseCount; + } + + private _next(): ScreenshotTestCase | undefined { + return screenshots.getByTestCaseIndex(this._testCaseIndex + 1); + } + + private _previous(): ScreenshotTestCase | undefined { + return screenshots.getByTestCaseIndex(this._testCaseIndex - 1); + } + private _viewportFilterChanged(event: CustomEvent): void { this._filter = { ...this._filter, @@ -70,10 +89,7 @@ export class TestCase extends LitElement { public override render(): TemplateResult { return html` -
+
${this.params?.componentName} @@ -84,16 +100,16 @@ export class TestCase extends LitElement {
Overview
diff --git a/tools/visual-regression-testing/diff-app/src/screenshots.ts b/tools/visual-regression-testing/diff-app/src/screenshots.ts index 71c49eb205..f14f1415b5 100644 --- a/tools/visual-regression-testing/diff-app/src/screenshots.ts +++ b/tools/visual-regression-testing/diff-app/src/screenshots.ts @@ -174,45 +174,4 @@ export class Screenshots { } } -export class ScreenshotService { - public readonly screenshots; - private _currentIndex: number = -1; - - public get progressFraction(): number { - return (this._currentIndex + 1) / this.screenshots.testCaseCount; - } - - public get current(): ScreenshotTestCase | undefined { - return this._current; - } - private _current?: ScreenshotTestCase; - - public constructor(screenshots: Screenshots) { - this.screenshots = screenshots; - this._currentIndex = 0; - this._current = screenshots.getByTestCaseIndex(this._currentIndex); - } - - public get next(): ScreenshotTestCase | undefined { - return this.screenshots.getByTestCaseIndex(this._currentIndex + 1); - } - - public get previous(): ScreenshotTestCase | undefined { - return this.screenshots.getByTestCaseIndex(this._currentIndex - 1); - } - - public setCurrentTestCase( - componentName: string, - testCaseName: string, - ): ScreenshotTestCase | undefined { - const testCaseIndex = this.screenshots.indexOfTestCase(componentName, testCaseName); - - this._currentIndex = testCaseIndex; - if (testCaseIndex >= 0) { - this._current = this.screenshots.getByTestCaseIndex(testCaseIndex); - return this._current; - } - } -} - -export const screenshotService = new ScreenshotService(new Screenshots(screenshotsRaw)); +export const screenshots = new Screenshots(screenshotsRaw); From 42cf6c201fb226557a6e8884a4faad2ccc9dc694 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Tue, 14 May 2024 10:13:03 +0200 Subject: [PATCH 37/48] refactor: reduce tests --- .../button/button/button.snapshot.spec.ts | 17 ++--- .../core/testing/private/describe-each.ts | 20 +++--- .../testing/private/describe-viewports.ts | 30 +++++++-- .../private/visual-regression-snapshot.ts | 64 +++++++++++++------ src/components/core/testing/test-setup.ts | 19 ++++-- 5 files changed, 106 insertions(+), 44 deletions(-) diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts index abe7497126..e632e4a594 100644 --- a/src/components/button/button/button.snapshot.spec.ts +++ b/src/components/button/button/button.snapshot.spec.ts @@ -7,7 +7,6 @@ import { isVisualRegressionRun, visualRegressionSnapshot, } from '../../core/testing/private.js'; -import type { SbbButtonSize } from '../common.js'; import './button.js'; @@ -15,14 +14,17 @@ describe(`sbb-button`, () => { if (isVisualRegressionRun()) { describe('visual-regression', () => { const cases = { - size: ['s', 'm', 'l'] as SbbButtonSize[], disabled: [false, true], negative: [false, true], - iconName: [undefined, 'arrow-right-small'], + state: [ + { icon: undefined, text: 'Button' }, + { icon: 'arrow-right-small', text: 'Button' }, + { icon: 'arrow-right-small', text: '' }, + ], }; - describeViewports(() => { - describeEach(cases, ({ size, disabled, negative, iconName }) => { + describeViewports({ viewports: ['zero', 'medium'] }, () => { + describeEach(cases, ({ disabled, negative, state }) => { let root: HTMLElement; beforeEach(async () => { root = await fixture(html` @@ -33,11 +35,10 @@ describe(`sbb-button`, () => { > Button${state.text}
`); diff --git a/src/components/core/testing/private/describe-each.ts b/src/components/core/testing/private/describe-each.ts index bfa5d2c1ef..c9889bc6dd 100644 --- a/src/components/core/testing/private/describe-each.ts +++ b/src/components/core/testing/private/describe-each.ts @@ -1,3 +1,12 @@ +function generateDescribeName(payload: Record): string { + return Object.entries(payload) + .map( + ([key, value]) => + `${key}=${typeof value === 'object' && value ? `(${generateDescribeName(value as Record)})` : value}`, + ) + .join(', '); +} + function partialDescribeEach>( cases: T, payload: Record, @@ -16,14 +25,9 @@ function partialDescribeEach>( } else { for (const value of values) { const finalPayload = { ...payload, [key]: value }; - describe( - Object.entries(finalPayload) - .map(([key, value]) => `${key}=${value}`) - .join(', '), - function () { - suiteRun.call(this, finalPayload); - }, - ); + describe(generateDescribeName(finalPayload), function () { + suiteRun.call(this, finalPayload); + }); } } } diff --git a/src/components/core/testing/private/describe-viewports.ts b/src/components/core/testing/private/describe-viewports.ts index 09364e05dd..4e9d33e868 100644 --- a/src/components/core/testing/private/describe-viewports.ts +++ b/src/components/core/testing/private/describe-viewports.ts @@ -16,13 +16,35 @@ const viewportSizes = { large: SbbBreakpointLargeMin, wide: SbbBreakpointWideMin, ultra: SbbBreakpointUltraMin, -}; +} as const; -export function describeViewports(fn: (this: Mocha.Suite) => void, viewportHeight = 400): void { - for (const [size, value] of Object.entries(viewportSizes)) { +export interface DescribeViewportOptions { + viewports?: (keyof typeof viewportSizes)[]; + viewportHeight?: number; +} + +export function describeViewports( + options: DescribeViewportOptions, + fn: (this: Mocha.Suite) => void, +): void; +export function describeViewports(fn: (this: Mocha.Suite) => void): void; +export function describeViewports( + optionsOrFn: DescribeViewportOptions | ((this: Mocha.Suite) => void), + fn?: (this: Mocha.Suite) => void, +): void { + const options = typeof optionsOrFn === 'object' ? optionsOrFn : {}; + fn ??= optionsOrFn as (this: Mocha.Suite) => void; + let viewportSizeTests = Object.entries(viewportSizes); + if (options.viewports?.length) { + viewportSizeTests = viewportSizeTests.filter(([key, _value]) => + options.viewports?.includes(key as keyof typeof viewportSizes), + ); + } + + for (const [size, value] of viewportSizeTests) { describe(`viewport=${size}`, function () { before(async () => { - await setViewport({ width: value, height: viewportHeight }); + await setViewport({ width: value, height: options.viewportHeight ?? 400 }); }); fn.call(this); diff --git a/src/components/core/testing/private/visual-regression-snapshot.ts b/src/components/core/testing/private/visual-regression-snapshot.ts index 8c63501b1e..db75c0dfc1 100644 --- a/src/components/core/testing/private/visual-regression-snapshot.ts +++ b/src/components/core/testing/private/visual-regression-snapshot.ts @@ -1,41 +1,67 @@ -import { sendKeys, sendMouse } from '@web/test-runner-commands'; +import { resetMouse, sendKeys, sendMouse } from '@web/test-runner-commands'; import { visualDiff } from '@web/test-runner-visual-regression'; export function imageName(test: Mocha.Runnable): string { return test!.fullTitle().replaceAll(', ', '-').replaceAll(' ', '_'); } -export function visualRegressionSnapshot(snapshotElement: () => HTMLElement): void { +function findElementCenter(snapshotElement: () => HTMLElement): [number, number] { + const element = snapshotElement(); + // Look for the first sbb-* element and get center of the element to + // move the mouse cursor over it. + const positionElement = element.localName.startsWith('sbb-') + ? element + : element.firstElementChild!; + const position = positionElement.getBoundingClientRect(); + return [ + Math.round(position.x + position.width / 2), + Math.round(position.y + position.height / 2), + ]; +} + +export function testVisualDiff(snapshotElement: () => HTMLElement): void { it('default', async function () { await visualDiff(snapshotElement(), imageName(this.test!)); }); +} +export function testVisualDiffFocus(snapshotElement: () => HTMLElement): void { it('focus', async function () { await sendKeys({ press: 'Tab' }); await visualDiff(snapshotElement(), imageName(this.test!)); }); +} +export function testVisualDiffHover(snapshotElement: () => HTMLElement): void { it('hover', async function () { - const element = snapshotElement(); - const positionElement = element.localName.startsWith('sbb-') - ? element - : element.firstElementChild!; - const position = positionElement.getBoundingClientRect(); - await sendMouse({ - type: 'move', - position: [ - Math.round(position.x + position.width / 2), - Math.round(position.y + position.height / 2), - ], - }); + const position = findElementCenter(snapshotElement); try { - await visualDiff(element, imageName(this.test!)); + await sendMouse({ type: 'move', position }); + await visualDiff(snapshotElement(), imageName(this.test!)); } finally { - await sendMouse({ - type: 'move', - position: [0, 0], - }); + await resetMouse(); } }); } + +export function testVisualDiffActive(snapshotElement: () => HTMLElement): void { + it('active', async function () { + const position = findElementCenter(snapshotElement); + + try { + await sendMouse({ type: 'move', position }); + await sendMouse({ type: 'down' }); + await visualDiff(snapshotElement(), imageName(this.test!)); + } finally { + await resetMouse(); + } + }); +} + +export function visualRegressionSnapshot(snapshotElement: () => HTMLElement): void { + testVisualDiff(snapshotElement); + testVisualDiffFocus(snapshotElement); + testVisualDiffHover(snapshotElement); + testVisualDiffActive(snapshotElement); +} diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index 8489618e69..bcccbc8319 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -1,10 +1,23 @@ +import { getSvgContent } from '../../icon.js'; import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; import { isHydratedSsr, isVisualRegressionRun } from './private.js'; -function setupIconConfig(): void { +if (isVisualRegressionRun()) { + const preloadedIcons = ['arrow-right-small']; + await Promise.all(preloadedIcons.map((icon) => getSvgContent('default', icon, true))); + + mergeConfig({ + icon: { + interceptor({ namespace, name }) { + throw new Error(`Icon ${namespace}:${name} must be preloaded in test-setup.ts!`); + }, + }, + }); +} else { + // Setup mock configuration for icons const testNamespaces = ['default', 'picto']; const icon: SbbIconConfig = { interceptor: ({ namespace, name, request }) => { @@ -27,10 +40,6 @@ function setupIconConfig(): void { mergeConfig({ icon }); } -if (!isVisualRegressionRun()) { - setupIconConfig(); -} - if (isHydratedSsr()) { await import('@lit-labs/ssr-client/lit-element-hydrate-support.js'); } From 048690f2c2a2af4764c4c6076c02570f82eb7e83 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Tue, 14 May 2024 10:39:03 +0200 Subject: [PATCH 38/48] fix: review diff-app Signed-off-by: Jeremias Peier --- .../diff-app/interfaces.ts | 11 ++++ .../src/components/overview/overview.ts | 2 +- .../diff-app/src/main.ts | 1 - .../diff-app/src/screenshots.ts | 51 +++---------------- .../diff-app/src/vite-env.d.ts | 2 +- .../diff-app/vite.config.ts | 50 ++++++++++++++---- 6 files changed, 62 insertions(+), 55 deletions(-) create mode 100644 tools/visual-regression-testing/diff-app/interfaces.ts diff --git a/tools/visual-regression-testing/diff-app/interfaces.ts b/tools/visual-regression-testing/diff-app/interfaces.ts new file mode 100644 index 0000000000..f1db57ea56 --- /dev/null +++ b/tools/visual-regression-testing/diff-app/interfaces.ts @@ -0,0 +1,11 @@ +export interface FailedFiles { + browserName: string; + name: string; + failedFile: string; + diffFile: string; + baselineFile: string; + isNew: boolean; + vierports: string; +} + +export type ScreenshotMap = Record>>; diff --git a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts index cf835c5219..5e9e1cf8a4 100644 --- a/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts +++ b/tools/visual-regression-testing/diff-app/src/components/overview/overview.ts @@ -29,7 +29,7 @@ export class Overview extends LitElement { ${screenshots.stats} Start comparing diff --git a/tools/visual-regression-testing/diff-app/src/main.ts b/tools/visual-regression-testing/diff-app/src/main.ts index d89e69c28e..b02fe47ce0 100644 --- a/tools/visual-regression-testing/diff-app/src/main.ts +++ b/tools/visual-regression-testing/diff-app/src/main.ts @@ -24,7 +24,6 @@ export class Main extends LitElement { html``, - enter: async () => { await import('./components/test-case/test-case.js'); return true; diff --git a/tools/visual-regression-testing/diff-app/src/screenshots.ts b/tools/visual-regression-testing/diff-app/src/screenshots.ts index f14f1415b5..a431a6e47b 100644 --- a/tools/visual-regression-testing/diff-app/src/screenshots.ts +++ b/tools/visual-regression-testing/diff-app/src/screenshots.ts @@ -1,47 +1,12 @@ // eslint-disable-next-line import-x/no-unresolved import { screenshotsRaw } from 'virtual:screenshots'; -import type { FailedFiles } from '../vite.config.js'; +import type { FailedFiles, ScreenshotMap } from '../interfaces.js'; const viewportOrder = ['zero', 'micro', 'small', 'medium', 'large', 'wide', 'ultra']; -// TODO: discuss whether to include it in creation of screenshotsRaw -const extractHierarchicalMap = ( - screenshots: Record, -): Map>> => { - const map = new Map>>(); - - Object.entries(screenshots).forEach(([fileName, failedFiles]) => { - const component = fileName.match(/^(.*?)_/)![1]; - const name = fileName.match(/_viewport=.*?_(.*?).png$/)![1]; - const viewport = fileName.match(/viewport=(.*?)_/)![1]; - - if (!map.has(component)) { - map.set(component, new Map()); - } - - const componentsMap = map.get(component)!; - - if (!componentsMap.has(name)) { - componentsMap.set(name, new Map()); - } - - const testCaseMap = componentsMap.get(name)!; - - testCaseMap.set( - viewport, - failedFiles.map((failedFile) => ({ ...failedFile, viewport })), - ); - }); - return map; -}; - -export interface ScreenshotFailedFiles extends FailedFiles { - viewport: string; -} - export class ScreenshotStatistics { - public static fromFailedFiles(failedFiles: ScreenshotFailedFiles[]): ScreenshotStatistics { + public static fromFailedFiles(failedFiles: FailedFiles[]): ScreenshotStatistics { return failedFiles.reduce( (current, next) => current.sum(new ScreenshotStatistics(next.isNew ? 0 : 1, next.isNew ? 1 : 0)), @@ -76,7 +41,7 @@ export class ScreenshotViewport { public constructor( public readonly name: string, - public readonly browsers: ScreenshotFailedFiles[], + public readonly browsers: FailedFiles[], ) { this.stats = ScreenshotStatistics.fromFailedFiles(this.browsers); @@ -110,7 +75,7 @@ export class ScreenshotTestCase { ); } - public filter(viewport?: string, browser?: string): ScreenshotFailedFiles[] { + public filter(viewport?: string, browser?: string): FailedFiles[] { return this.viewports .filter((entry) => !viewport || entry.name === viewport) .flatMap((entry) => @@ -136,19 +101,19 @@ export class Screenshots { public readonly testCaseCount: number; public readonly flatTestCases: ScreenshotTestCase[]; - public constructor(screenshots: Record) { + public constructor(screenshotsRaw: ScreenshotMap) { const flatTestCases: ScreenshotTestCase[] = []; // Convert hierarchical screenshot map to classes - this.components = Array.from(extractHierarchicalMap(screenshots).entries()).map( + this.components = Object.entries(screenshotsRaw).map( ([componentName, testCases]) => new ScreenshotComponent( componentName, - Array.from(testCases.entries()).map(([testCase, viewports]) => { + Object.entries(testCases).map(([testCase, viewports]) => { const screenshotTestCase = new ScreenshotTestCase( componentName, testCase, - Array.from(viewports.entries()) + Object.entries(viewports) .map(([viewport, entries]) => new ScreenshotViewport(viewport, entries)) .sort((a: ScreenshotViewport, b: ScreenshotViewport) => a.compare(b)), ); diff --git a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts index d6c5e3c360..acd9038143 100644 --- a/tools/visual-regression-testing/diff-app/src/vite-env.d.ts +++ b/tools/visual-regression-testing/diff-app/src/vite-env.d.ts @@ -2,7 +2,7 @@ declare module 'virtual:screenshots' { // eslint-disable-next-line @typescript-eslint/consistent-type-imports - export const screenshotsRaw: Record; + export const screenshotsRaw: import('../interfaces.js').ScreenshotMap; } declare module '*?lit&inline' { diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts index 3cadf5763d..20f757e4dc 100644 --- a/tools/visual-regression-testing/diff-app/vite.config.ts +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -13,17 +13,40 @@ import { import rootConfig from '../../../vite.config.js'; import { distDir } from '../../vite/index.js'; +import type { FailedFiles } from './interfaces.js'; + const packageRoot = new URL('.', import.meta.url); const screenshotsDir = new URL(`./screenshots/`, distDir); -export interface FailedFiles { - browserName: string; - name: string; - failedFile: string; - diffFile: string; - baselineFile: string; - isNew: boolean; -} +const extractHierarchicalMap = ( + screenshots: Map, +): Map>> => { + const map = new Map>>(); + + screenshots.forEach((failedFiles, fileName) => { + const component = fileName.match(/^(.*?)_/)![1]; + const name = fileName.match(/_viewport=.*?_(.*?).png$/)![1]; + const viewport = fileName.match(/viewport=(.*?)_/)![1]; + + if (!map.has(component)) { + map.set(component, new Map()); + } + + const componentsMap = map.get(component)!; + + if (!componentsMap.has(name)) { + componentsMap.set(name, new Map()); + } + + const testCaseMap = componentsMap.get(name)!; + + testCaseMap.set( + viewport, + failedFiles.map((failedFile) => ({ ...failedFile, viewport })), + ); + }); + return map; +}; function prepareScreenshots(): PluginOption { let viteConfig: ResolvedConfig; @@ -138,7 +161,16 @@ function prepareScreenshots(): PluginOption { }); } - return `export const screenshotsRaw = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; + return `export const screenshotsRaw = ${JSON.stringify( + extractHierarchicalMap(screenshotsMeta), + (_key, value) => { + if (value instanceof Map) { + return Object.fromEntries(Array.from(value)); + } else { + return value; + } + }, + )}`; } }, configureServer(server) { From 38fc14636e3c9a76952aac53f7f8210e3c1914d0 Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Tue, 14 May 2024 10:39:38 +0200 Subject: [PATCH 39/48] fix: preload fonts Signed-off-by: Jeremias Peier --- web-test-runner.config.js | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 209a2289e2..fd6ee4c304 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -63,8 +63,29 @@ const groupNameOverride = process.argv.includes('--ssr-hydrated') const testRunnerHtml = (testFramework, _config, group) => ` - + + + +