diff --git a/.github/workflows/container-image-cleanup.yml b/.github/workflows/container-image-cleanup.yml index 15fb21b859e..0788b6f5538 100644 --- a/.github/workflows/container-image-cleanup.yml +++ b/.github/workflows/container-image-cleanup.yml @@ -3,7 +3,7 @@ name: Container Image Cleanup on: workflow_dispatch: {} schedule: - - cron: '0 5 * * *' + - cron: '0 3 * * *' permissions: packages: write @@ -12,9 +12,8 @@ jobs: container-image-cleanup: runs-on: ubuntu-latest env: - CLOSED_PR_RETENTION_DAYS: 14 - PACKAGE_NAME: storybook-preview - PR_TAG_PREFIX: preview-pr + CLOSED_PR_RETENTION_DAYS: 5 + PACKAGE_NAMES: storybook-preview,visual-regression steps: - uses: actions/github-script@v7 with: @@ -23,31 +22,34 @@ jobs: const pullRequests = await github.paginate( github.rest.pulls.list.endpoint.merge({ owner, repo, state: 'all' }) ); - const twoWeeksAgo = + const retentionPivot = new Date(Date.now() - (+process.env.CLOSED_PR_RETENTION_DAYS * 24 * 60 * 60 * 1000)); - const olderThanTwoWeeks = (date) => new Date(date) < twoWeeksAgo; + const olderThanTwoWeeks = (date) => new Date(date) < retentionPivot; const isExpiredPrTag = (version) => { const prNumber = +version.metadata?.container?.tags - ?.find((t) => t.startsWith(process.env.PR_TAG_PREFIX))?.split(process.env.PR_TAG_PREFIX)[1]; + ?.find((t) => t.match(/(preview-pr|pr)(\d+)/))?.match(/(preview-pr|pr)(\d+)/)[2]; const pr = pullRequests.find((p) => p.number === prNumber); return !!prNumber && pr?.state === 'closed' && olderThanTwoWeeks(pr.closed_at); }; - const params = { - package_type: 'container', - package_name: `${repo}/${process.env.PACKAGE_NAME}`, - username: owner - }; - const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + const packageNames = process.env.PACKAGE_NAME.split(',').map((n) => n.trim()); let packageDeletionFailed = false; - for (const version of versions.filter(isExpiredPrTag)) { - try { - await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); - console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); - } catch(e) { - console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); - console.error(e); - packageDeletionFailed = true; + for (const packageName of packageNames) { + const params = { + package_type: 'container', + package_name: `${repo}/${packageNames}`, + username: owner + }; + const { data: versions } = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser(params); + for (const version of versions.filter(isExpiredPrTag)) { + try { + await github.rest.packages.deletePackageVersionForUser({ ...params, package_version_id: version.id }); + console.log(`Deleted ${version.name} (${version.metadata.container.tags.join(', ')})`); + } catch(e) { + console.error(`Failed to delete ${version.name} (${version.metadata.container.tags.join(', ')})`); + console.error(e); + packageDeletionFailed = true; + } } } diff --git a/.github/workflows/continuous-integration-secure.yml b/.github/workflows/continuous-integration-secure.yml index 13262d420b5..be560293868 100644 --- a/.github/workflows/continuous-integration-secure.yml +++ b/.github/workflows/continuous-integration-secure.yml @@ -8,10 +8,13 @@ on: workflows: [Continuous Integration] types: [completed] -permissions: - deployments: write - packages: write - pull-requests: write +env: + IMAGE_REPO_PREVIEW: ghcr.io/${{ github.repository }}/storybook-preview + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0] != null && github.event.workflow_run.pull_requests[0].number || '' }} + VISUAL_REQUIRED: 'pr: visual review required' + VISUAL_APPROVED: 'pr: visual review approved' + DIFF_URL: https://lyne-visual-regression-diff-pr{}.app.sbb.ch jobs: preview-image: @@ -23,9 +26,10 @@ jobs: github.event.workflow_run.head_branch == 'main' ) ) - env: - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0] != null && github.event.workflow_run.pull_requests[0].number || '' }} - IMAGE_REPO: ghcr.io/${{ github.repository }}/storybook-preview + permissions: + deployments: write + packages: write + pull-requests: write steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 @@ -42,7 +46,7 @@ jobs: uses: actions/github-script@v7 with: script: | - const environment = process.env.PR_NUMBER ? `preview-pr${process.env.PR_NUMBER}` : 'preview-main'; + const environment = process.env.PR_NUMBER ? `pr${process.env.PR_NUMBER}` : 'main'; const payload = { owner: context.repo.owner, repo: context.repo.repo, environment }; const { data: deployment } = await github.rest.repos.createDeployment({ ...payload, @@ -59,30 +63,26 @@ jobs: return environment; result-encoding: string - - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin - - name: 'Container: Build image' - run: docker build -t $IMAGE_REPO:${{ steps.tag-name.outputs.result }} . + - name: 'Container: Build and preview image' + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + docker build --tag $IMAGE_REPO_PREVIEW:${{ steps.tag-name.outputs.result }} . + docker push $IMAGE_REPO_PREVIEW:${{ steps.tag-name.outputs.result }} env: DOCKER_BUILDKIT: 1 - - name: 'Container: Publish image' - run: docker push $IMAGE_REPO:${{ steps.tag-name.outputs.result }} - name: "Add 'preview-available' label" if: env.PR_NUMBER != '' # This label is used for filtering deployments in ArgoCD - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: preview-available - number: ${{ env.PR_NUMBER }} + run: gh issue edit ${{ env.PR_NUMBER }} --repo ${{ github.repository }} --add-label "preview-available" codecov: runs-on: ubuntu-latest if: > github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' - env: - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0] != null && github.event.workflow_run.pull_requests[0].number || '' }} + permissions: + pull-requests: write steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 @@ -100,3 +100,91 @@ jobs: override_pr: ${{ env.PR_NUMBER }} fail_ci_if_error: true verbose: true + + visual-regression: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + permissions: + checks: write + packages: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - uses: actions/download-artifact@v4 + with: + name: visual-regression-screenshots + path: dist/screenshots/ + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GH_ACTIONS_ARTIFACT_DOWNLOAD }} + + - name: Build diff-app + run: yarn build:diff-app + + - name: Create check if changed + uses: actions/github-script@v7 + id: screenshot-check + with: + script: | + const { readdirSync, readFileSync } = await import('fs'); + + // If we have no screenshots, we do not need to create a check or containers. + if (!readdirSync('dist/screenshots').length) { + return 'empty'; + } + + const diffUrl = process.env.DIFF_URL.replace('{}', process.env.PR_NUMBER); + const diffInfo = JSON.parse(readFileSync('dist/diff.json', 'utf8')); + let previousDiffInfo = {}; + try { + const response = await fetch(diffUrl + 'diff.json'); + if (response.ok) { + previousDiffInfo = await response.json(); + } + } catch {} + + // If the diff hash is the same as previously, we do not need to update the state or containers. + if (diffInfo.hash === previousDiffInfo.hash) { + return 'no-change'; + } + + const { data: deployment } = await github.rest.checks.create(({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'Visual Regression Check', + head_sha: context.payload.workflow_run.head_sha, + status: 'in_progress', + details_url: diffUrl, + output: { + title: 'Visual Regression Check', + summary: diffUrl, + text: `Changes: ${diffInfo.changedAmount}\nNew: ${diffInfo.newAmount}`, + } + }); + + return 'changed'; + result-encoding: string + + - name: Remove labels when no failed screenshots exist + if: steps.screenshot-check.outputs.result == 'empty' + run: gh issue edit $PR_NUMBER --remove-label "$VISUAL_REQUIRED" + + - name: Add label and create container + if: steps.screenshot-check.outputs.result == 'changed' + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + docker build --tag $IMAGE_REPO_VISUAL_REGRESSION:pr$PR_NUMBER . + docker push $IMAGE_REPO_VISUAL_REGRESSION:pr$PR_NUMBER + + gh issue edit $PR_NUMBER --remove-label "$VISUAL_APPROVED" + gh issue edit $PR_NUMBER --add-label "$VISUAL_REQUIRED" + gh issue edit $PR_NUMBER --add-label "visual-regression-diff-available" + env: + DOCKER_BUILDKIT: 1 diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d0e48d26037..9050f779fc7 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -7,6 +7,9 @@ concurrency: permissions: read-all +env: + IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression + jobs: lint: runs-on: ubuntu-latest @@ -40,11 +43,6 @@ jobs: test: runs-on: ubuntu-latest needs: lint - services: - visual-regression: - image: ghcr.io/${{ github.repository }}/visual-regression:baseline - ports: - - 8080:8050 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -160,14 +158,41 @@ jobs: onlyChanged: true externals: '**/components/core/styles/**/*.scss' + visual-regression: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + needs: test + services: + visual-regression: + image: ghcr.io/${{ github.repository }}/visual-regression:baseline + ports: + - 8050:8080 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn install --frozen-lockfile --non-interactive + + - name: Install browser dependencies + run: yarn playwright install-deps + - name: Run visual regression baseline generation + run: yarn test:visual-regression + env: + NODE_ENV: production + - name: Store visual regression output + uses: actions/upload-artifact@v4 + with: + name: visual-regression-screenshots + path: dist/screenshots-artifact/ + visual-regression-baseline: runs-on: ubuntu-latest permissions: packages: write if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: test - env: - IMAGE_REPO_VISUAL_REGRESSION: ghcr.io/${{ github.repository }}/visual-regression services: visual-regression: image: ghcr.io/${{ github.repository }}/visual-regression:baseline diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 9b8f89edd47..9d800563ab6 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -66,7 +66,7 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' run: docker build --tag $IMAGE_REPO_STORYBOOK:$VERSION --tag $IMAGE_REPO_STORYBOOK:latest . env: @@ -116,7 +116,7 @@ jobs: - name: Remove files with forbidden extensions run: node ./scripts/clean-storybook-files.cjs - name: 'Container: Login to GitHub Container Repository' - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io --username ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - name: 'Container: Build image' run: docker build --tag $IMAGE_REPO_STORYBOOK:dev . env: diff --git a/Dockerfile b/Dockerfile index 2220166b4f8..bd0cd09679d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM ghcr.io/nginxinc/nginx-unprivileged:stable +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + # Copy nginx configuration COPY ./.github/default.conf /etc/nginx/conf.d/default.conf diff --git a/src/components/button/button/button.snapshot.spec.ts b/src/components/button/button/button.snapshot.spec.ts index c6758696097..abe74971265 100644 --- a/src/components/button/button/button.snapshot.spec.ts +++ b/src/components/button/button/button.snapshot.spec.ts @@ -4,6 +4,7 @@ import { describeEach, describeViewports, fixture, + isVisualRegressionRun, visualRegressionSnapshot, } from '../../core/testing/private.js'; import type { SbbButtonSize } from '../common.js'; @@ -11,38 +12,40 @@ import type { SbbButtonSize } from '../common.js'; import './button.js'; describe(`sbb-button`, () => { - describe('visual-regression', () => { - const cases = { - size: ['s', 'm', 'l'] as SbbButtonSize[], - disabled: [false, true], - negative: [false, true], - iconName: [undefined, 'arrow-right-small'], - }; + if (isVisualRegressionRun()) { + describe('visual-regression', () => { + const cases = { + size: ['s', 'm', 'l'] as SbbButtonSize[], + disabled: [false, true], + negative: [false, true], + iconName: [undefined, 'arrow-right-small'], + }; - describeViewports(() => { - describeEach(cases, ({ size, disabled, negative, iconName }) => { - let root: HTMLElement; - beforeEach(async () => { - root = await fixture(html` -
- Button { + describeEach(cases, ({ size, disabled, negative, iconName }) => { + let root: HTMLElement; + beforeEach(async () => { + root = await fixture(html` +
-
- `); - }); + Button +
+ `); + }); - visualRegressionSnapshot(() => root); + visualRegressionSnapshot(() => root); + }); }); }); - }); + } }); diff --git a/src/components/core/testing.ts b/src/components/core/testing.ts index 032a69bed9b..428ae3f9736 100644 --- a/src/components/core/testing.ts +++ b/src/components/core/testing.ts @@ -1,6 +1,5 @@ export * from './testing/event-spy.js'; export * from './testing/mocha-extensions.js'; -export * from './testing/platform.js'; export * from './testing/scroll.js'; export * from './testing/wait-for-condition.js'; export * from './testing/wait-for-render.js'; diff --git a/src/components/core/testing/private.ts b/src/components/core/testing/private.ts index 45ecfb614c9..ba1753f9481 100644 --- a/src/components/core/testing/private.ts +++ b/src/components/core/testing/private.ts @@ -4,5 +4,6 @@ export * from './private/describe-viewports.js'; export * from './private/dispatch-events.js'; export * from './private/event-objects.js'; export * from './private/fixture.js'; +export * from './private/platform.js'; export * from './private/type-in-element.js'; export * from './private/visual-regression-snapshot.js'; diff --git a/src/components/core/testing/private/fixture.ts b/src/components/core/testing/private/fixture.ts index 7e3788ee48f..db042554d3e 100644 --- a/src/components/core/testing/private/fixture.ts +++ b/src/components/core/testing/private/fixture.ts @@ -1,8 +1,9 @@ import type { TemplateResult } from 'lit'; -import { isHydratedSsr, isNonHydratedSsr } from '../platform.js'; import { waitForLitRender } from '../wait-for-render.js'; +import { isHydratedSsr, isNonHydratedSsr } from './platform.js'; + // Copied from @lit-labs/testing/lib/fixtures/fixture-options.d.ts interface FixtureOptions { /** diff --git a/src/components/core/testing/platform.ts b/src/components/core/testing/private/platform.ts similarity index 82% rename from src/components/core/testing/platform.ts rename to src/components/core/testing/private/platform.ts index eb23176b417..28a308ec16e 100644 --- a/src/components/core/testing/platform.ts +++ b/src/components/core/testing/private/platform.ts @@ -32,3 +32,10 @@ export const isNonHydratedSsr = (): boolean => * Returns true, if this is run in an SSR test group. */ export const isSsr = (): boolean => isHydratedSsr() || isNonHydratedSsr(); + +/** + * This is a custom implementation. + * Returns true, if this is run in the visual regression test group. + */ +export const isVisualRegressionRun = (): boolean => + !isServer && (globalThis as any).testGroup === 'visual-regression'; diff --git a/src/components/core/testing/test-setup.ts b/src/components/core/testing/test-setup.ts index 5f90f96f6d0..a795c33515c 100644 --- a/src/components/core/testing/test-setup.ts +++ b/src/components/core/testing/test-setup.ts @@ -2,7 +2,7 @@ import { sbbInputModalityDetector } from '../a11y.js'; import type { SbbIconConfig } from '../config.js'; import { mergeConfig } from '../config.js'; -import { isHydratedSsr } from './platform.js'; +import { isHydratedSsr } from './private.js'; function setupIconConfig(): void { const testNamespaces = ['default', 'picto']; diff --git a/tools/visual-regression-testing/baseline.Dockerfile b/tools/visual-regression-testing/baseline.Dockerfile index f1b9ecc96b4..4d49ebbcc78 100644 --- a/tools/visual-regression-testing/baseline.Dockerfile +++ b/tools/visual-regression-testing/baseline.Dockerfile @@ -1,18 +1,13 @@ FROM ghcr.io/nginxinc/nginx-unprivileged:stable +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# Copy screenshots +COPY ./dist/screenshots/Chromium/baseline/ /usr/share/nginx/html/Chromium/baseline/ +COPY ./dist/screenshots/Firefox/baseline/ /usr/share/nginx/html/Firefox/baseline/ +COPY ./dist/screenshots/Webkit/baseline/ /usr/share/nginx/html/Webkit/baseline/ + COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf -COPY ./dist/screenshots /usr/share/nginx/html +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh -USER root -# We calculate the sha1 hashes of the png files in order to use it as etag values. -# This allows us to use HTTP caching mechanisms, which should reduce network traffic -# for the baseline comparison. -RUN cd /usr/share/nginx/html && \ - find ./*/ -type f ! -iname "*.png" -delete && \ - find ./*/ -type d -name failed -prune -exec rm -rf {} \; && \ - find ./*/ -type d -name .cache -prune -exec rm -rf {} \; && \ - echo 'map_hash_bucket_size 16384;' > /etc/nginx/conf.d/1etags.conf && \ - echo 'map $uri $pngetag {' >> /etc/nginx/conf.d/1etags.conf && \ - find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha1sum $1 | cut -d " " -f 1);"' _ {} \; >> /etc/nginx/conf.d/1etags.conf && \ - echo '}' >> /etc/nginx/conf.d/1etags.conf -USER $UID +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/diff-app.Dockerfile b/tools/visual-regression-testing/diff-app.Dockerfile new file mode 100644 index 00000000000..1bd7ea0d99d --- /dev/null +++ b/tools/visual-regression-testing/diff-app.Dockerfile @@ -0,0 +1,11 @@ +FROM ghcr.io/nginxinc/nginx-unprivileged:stable + +LABEL org.opencontainers.image.source=https://github.com/lyne-design-system/lyne-components + +# This is currently the same config file as for baseline. Separate into separate configs, if this changes. +COPY ./tools/visual-regression-testing/baseline.nginx.conf /etc/nginx/conf.d/default.conf +COPY ./dist/diff-app /usr/share/nginx/html + +COPY --chmod=555 ./tools/visual-regression-testing/etag-map-generation.sh /usr/share/nginx/etag-map-generation.sh + +RUN /usr/share/nginx/etag-map-generation.sh > /etc/nginx/conf.d/1etags.conf diff --git a/tools/visual-regression-testing/diff-app/vite.config.ts b/tools/visual-regression-testing/diff-app/vite.config.ts index 75d4501e464..981c6152461 100644 --- a/tools/visual-regression-testing/diff-app/vite.config.ts +++ b/tools/visual-regression-testing/diff-app/vite.config.ts @@ -1,3 +1,4 @@ +import { createHash } from 'crypto'; import { existsSync, readdirSync, readFileSync } from 'fs'; import { relative } from 'path'; @@ -41,86 +42,101 @@ function prepareScreenshots(): PluginOption { }, load(id) { if (id === resolvedVirtualModuleId) { - const browsers = readdirSync(screenshotsDir, { withFileTypes: true }) - .filter((d) => d.name !== '.cache') - .map((d) => d.name); - - const screenshotsMeta = browsers - .filter((browserName) => existsSync(new URL(`./${browserName}/failed/`, screenshotsDir))) - .flatMap((browserName) => { - const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); - - return readdirSync(failedDir, { - withFileTypes: true, - }) - .filter((d) => !d.name.endsWith('-diff.png')) - .map((d) => { - const failedFilePath = new URL(`./${d.name}`, failedDir); - const diffFilePath = new URL( - `./${d.name.replace(/.png$/, '-diff.png')}`, - failedDir, - ); - const baselineFilePath = new URL( - `./${browserName}/baseline/${d.name}`, - screenshotsDir, - ); - const baselineCacheFilePath = new URL( - `./.cache/${browserName}/baseline/${d.name}`, - screenshotsDir, - ); - - const isNew = !existsSync(diffFilePath); - - const assetsScreenshots = 'assets/screenshots/'; - const failedRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, failedFilePath.pathname); - const diffRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); - const baselineRelativeFileName = - assetsScreenshots + relative(screenshotsDir.pathname, baselineFilePath.pathname); - - if (viteConfig.command !== 'serve') { - this.emitFile({ - type: 'asset', - fileName: failedRelativeFileName, - source: readFileSync(failedFilePath), + const failedScreenshotsHash = createHash('sha256'); + const screenshotsFailures = existsSync(screenshotsDir) + ? readdirSync(screenshotsDir, { withFileTypes: true }) + .map((d) => d.name) + .filter((browserName) => + existsSync(new URL(`./${browserName}/failed/`, screenshotsDir)), + ) + .flatMap((browserName) => { + const failedDir = new URL(`./${browserName}/failed/`, screenshotsDir); + + return readdirSync(failedDir, { + withFileTypes: true, + }) + .filter((d) => !d.name.endsWith('-diff.png')) + .map((d) => { + const failedFilePath = new URL(`./${d.name}`, failedDir); + const diffFilePath = new URL( + `./${d.name.replace(/.png$/, '-diff.png')}`, + failedDir, + ); + const baselineFilePath = new URL( + `./${browserName}/baseline/${d.name}`, + screenshotsDir, + ); + + const isNew = !existsSync(diffFilePath); + + const assetsScreenshots = 'assets/screenshots/'; + const failedRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, failedFilePath.pathname); + const diffRelativeFileName = + assetsScreenshots + relative(screenshotsDir.pathname, diffFilePath.pathname); + const baselineRelativeFileName = + assetsScreenshots + + relative(screenshotsDir.pathname, baselineFilePath.pathname); + + if (viteConfig.command !== 'serve') { + const failedFileContent = readFileSync(failedFilePath); + // We only add the failed screenshot hashes, as the baseline and comparison (*-diff.png) + // are not relevant to detect whether it is a new difference. + failedScreenshotsHash.update(failedFileContent); + this.emitFile({ + type: 'asset', + fileName: failedRelativeFileName, + source: failedFileContent, + }); + + if (!isNew) { + this.emitFile({ + type: 'asset', + fileName: diffRelativeFileName, + source: readFileSync(diffFilePath), + }); + + this.emitFile({ + type: 'asset', + fileName: baselineRelativeFileName, + source: readFileSync(baselineFilePath), + }); + } + } + + return { + browserName, + name: d.name, + failedFile: failedRelativeFileName, + diffFile: diffRelativeFileName, + baselineFile: baselineRelativeFileName, + isNew, + }; }); - - if (!isNew) { - this.emitFile({ - type: 'asset', - fileName: diffRelativeFileName, - source: readFileSync(diffFilePath), - }); - - this.emitFile({ - type: 'asset', - fileName: baselineRelativeFileName, - source: readFileSync( - existsSync(baselineFilePath) ? baselineFilePath : baselineCacheFilePath, - ), - }); - } - } - - return { - browserName, - name: d.name, - failedFile: failedRelativeFileName, - diffFile: diffRelativeFileName, - baselineFile: baselineRelativeFileName, - isNew, - }; - }); - }) - .reduce( - (current, next) => - current.set( - next.name, - current.has(next.name) ? current.get(next.name)!.concat(next) : [next], - ), - new Map(), - ); + }) + : []; + + const screenshotsMeta = screenshotsFailures.reduce( + (current, next) => + current.set( + next.name, + current.has(next.name) ? current.get(next.name)!.concat(next) : [next], + ), + new Map(), + ); + + if (viteConfig.command !== 'serve') { + this.emitFile({ + type: 'asset', + fileName: 'diff.json', + source: JSON.stringify({ + changedAmount: screenshotsFailures.filter((f) => !f.isNew).length, + newAmount: screenshotsFailures.filter((f) => f.isNew).length, + hash: failedScreenshotsHash.digest('hex'), + }), + }); + } return `export const screenshots = ${JSON.stringify(Object.fromEntries(screenshotsMeta))}`; } diff --git a/tools/visual-regression-testing/etag-map-generation.sh b/tools/visual-regression-testing/etag-map-generation.sh new file mode 100755 index 00000000000..5f88502c319 --- /dev/null +++ b/tools/visual-regression-testing/etag-map-generation.sh @@ -0,0 +1,10 @@ + +# We calculate the sha1 hashes of the png files in order to use it as etag values. +# This allows us to use HTTP caching mechanisms, which should reduce network traffic +# for the baseline comparison. +cd /usr/share/nginx/html + +echo 'map_hash_bucket_size 32768;' +echo 'map $uri $pngetag {' +find . -type f -name '*.png' -exec bash -c 'echo " \"${1:1}\" $(sha256sum $1 | cut -d " " -f 1);"' _ {} \; +echo '}' \ No newline at end of file diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts index 8f493c5f554..13ebd9665b9 100644 --- a/tools/visual-regression-testing/exec.ts +++ b/tools/visual-regression-testing/exec.ts @@ -2,14 +2,61 @@ // and if it is not Linux, runs it in a container. import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; -import { mkdirSync } from 'fs'; +import { cpSync, existsSync, mkdirSync } from 'fs'; import { platform } from 'os'; import { startTestRunner } from '@web/test-runner'; +import * as glob from 'glob'; -const args = process.argv.slice(2); -if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { - startTestRunner(); +if (process.env.GITHUB_ACTIONS) { + // When being run on GitHub Actions we have two use cases. + // Baseline generation for which our expectation is to not fail. + // Diff generation if any test fails, for which we copy only the necessary + // files to dist/screenshots-artifact/ to reduce artifact size. + const runner = await startTestRunner({ autoExitProcess: false }); + if (!runner) { + throw new Error( + `Unexpected state. Test runner not available. Check tools/visual-regression-testing/exec.ts execution.`, + ); + } + await new Promise((r) => runner.on('stopped', r)); + + const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); + const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); + mkdirSync(artifactDir, { recursive: true }); + + if (runner.passed) { + // Tests passed. Do nothing. + process.exit(0); + } + + // When visual regression tests have failed, we only want to pack the relevant screenshots + // into the artifact transfered to the secure workflow, as uploading and downloading the full + // baseline would take far longer. + // Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will + // be moved to /dist/screenshots in the secure workflow. + const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); + for (const failedDir of failedDirs) { + cpSync(new URL(`./${failedDir}`, screenshotDir), new URL(`./${failedDir}`, artifactDir), { + force: true, + recursive: true, + }); + } + + const failedFiles = glob + .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) + .map((p) => p.replace('/failed/', '/baseline/')); + for (const failedFile of failedFiles) { + const baselineFile = new URL(`./${failedFile}`, screenshotDir); + if (existsSync(baselineFile)) { + cpSync(baselineFile, new URL(`./${failedFile}`, artifactDir), { + force: true, + recursive: true, + }); + } + } +} else if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { + await startTestRunner(); } else { function executableIsAvailable(name: string): string | null { try { @@ -26,6 +73,7 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { process.exit(1); } + const args = process.argv.slice(2); const cwd = new URL('../../', import.meta.url); const tag = 'lyne-vrt'; const execOptions: ExecSyncOptionsWithStringEncoding = { @@ -33,6 +81,10 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { stdio: 'inherit', cwd, }; + const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); execSync( `${containerCmd} build ` + '--file=tools/visual-regression-testing/testing.Dockerfile ' + @@ -43,6 +95,7 @@ if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); execSync( `${containerCmd} run -it --rm --ipc=host ` + + `--env=BRANCH="${branch}"` + `--volume=./dist/screenshots:/dist/screenshots ` + `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + tag, diff --git a/tools/web-test-runner/visual-regression-plugin-config.js b/tools/web-test-runner/visual-regression-plugin-config.js index 9030e4d4827..e127d69852a 100644 --- a/tools/web-test-runner/visual-regression-plugin-config.js +++ b/tools/web-test-runner/visual-regression-plugin-config.js @@ -1,7 +1,13 @@ -import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; -import { dirname, extname, join } from 'path'; +import { execSync } from 'child_process'; +import { createHash } from 'crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, extname } from 'path'; -const baselineUrl = process.env.CI +const branch = + process.env.GITHUB_REF_NAME ?? + process.env.BRANCH ?? + execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); +const baselineUrl = process.env.GITHUB_ACTIONS ? 'http://localhost:8050/' : 'https://lyne-visual-regression-baseline.app.sbb.ch/'; @@ -10,51 +16,48 @@ export const visualRegressionConfig = (update) => ({ update, baseDir: 'dist/screenshots', - async getBaseline({ filePath, baseDir, name }) { - if (existsSync(filePath)) { - return readFileSync(filePath); - } - - const cacheFile = join(baseDir, '.cache', name + extname(filePath)); - const cacheFileDetails = cacheFile + '.json'; - mkdirSync(dirname(cacheFile), { recursive: true }); + async getBaseline({ filePath, name }) { const baselineFileUrl = baselineUrl + name + extname(filePath); - const downloadFile = async () => { - try { - const response = await fetch(baselineFileUrl); - - if (response.ok) { - writeFileSync(cacheFile, Buffer.from(new Uint8Array(await response.arrayBuffer()))); - writeFileSync( - cacheFileDetails, - JSON.stringify({ etag: response.headers.get('etag') }, null, 2), - 'utf8', - ); - - return readFileSync(cacheFile); - } - } catch { - /* empty */ - } - }; - - if (existsSync(cacheFileDetails)) { - const details = JSON.parse(readFileSync(cacheFileDetails)); + const infoFilePath = filePath + '.json'; + const info = existsSync(infoFilePath) ? JSON.parse(readFileSync(infoFilePath, 'utf8')) : {}; + if (existsSync(filePath) && info.branch === branch) { + return readFileSync(filePath); + } else if (existsSync(filePath) && info.etag) { const response = await fetch(baselineFileUrl, { method: 'HEAD', - headers: { 'if-none-match': details.etag }, + headers: { 'if-none-match': info.etag }, }); - if (response.status === 200) { - return await downloadFile(); + + if (response.status === 304) { + return readFileSync(filePath); } else if (response.status === 404) { - [cacheFile, cacheFileDetails].forEach(unlinkSync); - } else if (response.status === 304) { - return readFileSync(cacheFile); - } else { - console.error(`Unexpected response from baseline service: ${response.status} (${name})`); + return undefined; } - } else { - return await downloadFile(); } + + // If the image address is not reachable, fetch throws, so we wrap it in try/catch. + try { + const response = await fetch(baselineFileUrl); + if (response.ok) { + const etag = response.headers.get('etag'); + const buffer = Buffer.from(await response.arrayBuffer()); + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, buffer); + writeFileSync(infoFilePath, JSON.stringify({ etag }, null, 2), 'utf8'); + + return buffer; + } + } catch (e) { + return undefined; + } + }, + saveBaseline({ filePath, content }) { + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, content); + writeFileSync( + filePath + '.json', + JSON.stringify({ branch, etag: createHash('sha256').update(content).digest('hex') }), + 'utf8', + ); }, }); diff --git a/web-test-runner.config.js b/web-test-runner.config.js index 0bdb2a51950..d257457f6e5 100644 --- a/web-test-runner.config.js +++ b/web-test-runner.config.js @@ -22,6 +22,11 @@ const webkit = process.argv.includes('--webkit'); const concurrency = process.argv.includes('--parallel') ? {} : { concurrency: 1 }; const updateBaseImages = process.argv.includes('--update-visual-baseline') || process.argv.includes('--uv'); +const visualRegressionRun = + process.argv.includes('--group=visual-regression') || + (process.argv.includes('--group') + ? process.argv[process.argv.indexOf('--group') + 1] === 'visual-regression' + : false); const stylesCompiler = new sass.initCompiler(); const renderStyles = () => @@ -86,15 +91,21 @@ const suppressedLogs = [ '[vite] connecting...', ]; +const groups = [ + // Disable ssr tests until stabilized. + // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, + // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, +]; + +// The visual regression test group is only added when expicitely set, as the tests are very expensive. +if (visualRegressionRun) { + groups.push({ name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }); +} + /** @type {import('@web/test-runner').TestRunnerConfig} */ export default { files: ['src/**/*.{e2e,spec,!snapshot.spec}.ts'], - groups: [ - // Disable ssr tests until stabilized. - // { name: 'e2e-ssr-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, - // { name: 'e2e-ssr-non-hydrated', files: 'src/**/*.e2e.ts', testRunnerHtml }, - { name: 'visual-regression', files: 'src/**/*.snapshot.spec.ts', testRunnerHtml }, - ], + groups, nodeResolve: true, concurrency: resolveConcurrency(), reporters: