From f3a7ea9d5af207f20aac0fcf491fe4c4d63d98bf Mon Sep 17 00:00:00 2001 From: Hector Alfaro Date: Tue, 19 Sep 2023 14:40:53 -0400 Subject: [PATCH] Add playwright-axe test suite (#41814) --- .github/workflows/headless-tests.yml | 13 ++- package-lock.json | 13 +++ package.json | 1 + src/landings/components/SidebarProduct.tsx | 7 +- src/landings/components/TableOfContents.tsx | 42 +++++----- src/rest/components/RestCodeSamples.tsx | 11 ++- tests/axe.md | 84 +++++++++++++++++++ .../content/get-started/liquid/index.md | 1 + .../get-started/liquid/table-row-headers.md | 6 +- .../liquid/tool-platform-switcher.md | 80 ++++++++++++++++++ .../get-started/markdown/code-annotations.md | 34 ++++++++ .../content/get-started/markdown/index.md | 1 + tests/rendering-fixtures/liquid.js | 2 +- .../playwright-a11y.spec.ts | 35 ++++++++ 14 files changed, 301 insertions(+), 29 deletions(-) create mode 100644 tests/axe.md create mode 100644 tests/fixtures/content/get-started/liquid/tool-platform-switcher.md create mode 100644 tests/fixtures/content/get-started/markdown/code-annotations.md create mode 100644 tests/rendering-fixtures/playwright-a11y.spec.ts diff --git a/.github/workflows/headless-tests.yml b/.github/workflows/headless-tests.yml index 8a9953daa58c..d57fb0b45a4c 100644 --- a/.github/workflows/headless-tests.yml +++ b/.github/workflows/headless-tests.yml @@ -26,6 +26,14 @@ jobs: playwright-tests: if: github.repository == 'github/docs-internal' || github.repository == 'github/docs' runs-on: ${{ fromJSON('["ubuntu-latest", "ubuntu-20.04-xl"]')[github.repository == 'github/docs-internal'] }} + strategy: + # When we're comfortable a11y tests aren't generating false positives and helping, + # let's remove the matrix and just run playwright in a single job. + matrix: + node: + - playwright-rendering + - playwright-a11y + fail-fast: false timeout-minutes: 60 steps: - name: Check out repo @@ -46,4 +54,7 @@ jobs: - name: Run Playwright tests env: PLAYWRIGHT_WORKERS: ${{ fromJSON('[1, 4]')[github.repository == 'github/docs-internal'] }} - run: npm run playwright-test -- --reporter list + # Run playwright rendering tests and a11y tests (axe scans) as distinct checks + # so that we can run them without blocking merges until we can be confident + # results for a11y tests are meaningul and scenarios we're testing are correct. + run: npm run playwright-test -- ${{ matrix.node }} --reporter list diff --git a/package-lock.json b/package-lock.json index fc575c1000ea..22c5f1137001 100644 --- a/package-lock.json +++ b/package-lock.json @@ -95,6 +95,7 @@ "devDependencies": { "@actions/core": "^1.10.0", "@actions/github": "^5.0.3", + "@axe-core/playwright": "^4.7.3", "@graphql-inspector/core": "^5.0.0", "@graphql-tools/load": "^8.0.0", "@jest/globals": "29.7.0", @@ -222,6 +223,18 @@ "node": ">=6.0.0" } }, + "node_modules/@axe-core/playwright": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.7.3.tgz", + "integrity": "sha512-v2PRgAyGvop7bamrTpNJtc5b1R7giAPnMzZXrS/VDZBCY5+uwVYtCNgDvBsqp5P1QMZxUMoBN+CERJUTMjFN0A==", + "dev": true, + "dependencies": { + "axe-core": "^4.7.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.18.6", "license": "MIT", diff --git a/package.json b/package.json index b11ca5d3f658..0f1904876934 100644 --- a/package.json +++ b/package.json @@ -144,6 +144,7 @@ "devDependencies": { "@actions/core": "^1.10.0", "@actions/github": "^5.0.3", + "@axe-core/playwright": "^4.7.3", "@graphql-inspector/core": "^5.0.0", "@graphql-tools/load": "^8.0.0", "@jest/globals": "29.7.0", diff --git a/src/landings/components/SidebarProduct.tsx b/src/landings/components/SidebarProduct.tsx index 28422a85d6f7..17efbc8a4ddb 100644 --- a/src/landings/components/SidebarProduct.tsx +++ b/src/landings/components/SidebarProduct.tsx @@ -56,12 +56,15 @@ export const SidebarProduct = () => { return ( <>
- + {conceptualPages.map((childPage) => ( ))} + + +
-
+ {restPages.map((category) => ( ))} diff --git a/src/landings/components/TableOfContents.tsx b/src/landings/components/TableOfContents.tsx index 2abd3e1c99f0..bc7640db15eb 100644 --- a/src/landings/components/TableOfContents.tsx +++ b/src/landings/components/TableOfContents.tsx @@ -13,7 +13,7 @@ export const TableOfContents = (props: Props) => { const { items, variant = 'expanded' } = props return ( -
    @@ -22,7 +22,7 @@ export const TableOfContents = (props: Props) => { const { fullPath: href, title, intro } = item return ( -
  • { {intro && (
    )} -
    • +
) })} @@ -52,28 +52,30 @@ export const TableOfContents = (props: Props) => { {title} {(childTocItems || []).length > 0 && ( - +
  • + + {(childTocItems || []).filter(Boolean).map((childItem) => { + return ( + + {childItem.title} + + ) + })} + +
    • )} ) })} )} - + ) } diff --git a/src/rest/components/RestCodeSamples.tsx b/src/rest/components/RestCodeSamples.tsx index 130122970170..c294609ee099 100644 --- a/src/rest/components/RestCodeSamples.tsx +++ b/src/rest/components/RestCodeSamples.tsx @@ -248,7 +248,7 @@ export function RestCodeSamples({ operation, slug, heading }: Props) {
    - + {languageSelectOptions.map((optionKey) => ( {displayedExample[selectedLanguage]}
    @@ -314,7 +316,10 @@ export function RestCodeSamples({ operation, slug, heading }: Props) { >
    {displayedExample.response.schema ? ( - + {responseSelectOptions.map((optionKey) => ( {selectedResponse === ResponseKeys.example diff --git a/tests/axe.md b/tests/axe.md new file mode 100644 index 000000000000..af00c30d417b --- /dev/null +++ b/tests/axe.md @@ -0,0 +1,84 @@ +# Axe test suite +We're using Playwright with the Axe Accessibility Engine to automatically flag accessibility issues with our site. This document summarizes some of the behavior we're enabling, and provides you with techniques and supporting documentation for working with this tooling. + +**Please note**: accessibility testing requires both automated and manual testing. The results from this tool are not comprehensive, and they need to be validated against best practices and our own accessibility standards. Furthermore, at the time of this writing, we're not testing every single page or interactive element on our site, so there are plenty of opportunities to improve upon this implementation and we've got lots of room to mature. + +## What we test +In this first iteration of these tests, we're testing some [high-usage pages, layouts, and elements](https://github.com/github/docs-internal/blob/30ff2bd725d3dfc326da3ed2f90fe8e87a065c93/tests/rendering-fixtures/playwright-a11y.spec.ts#L5-L21) that are at risk of becoming accessibility concerns. We understand the risk relatively well because we've fixed these types of accessibility issues in the past, but the test suite attempts to avoid making the same mistakes. + +The Axe tests aren't comprehensive; for example, when elements are hidden in the DOM they may be missed. Over time, we should create more fine-grained tests that manipulate interactive elements we know can result in accessibility issues. + +## How to interpret test results +A results from the Axe test suite will be verbose. We've highlighted some fields you'll find in the results below. + +```shell +# Example 1 + + "description": "Ensures table headers have discernible text", + + "help": "Table header text should not be empty", + + "helpUrl": "https://dequeuniversity.com/rules/axe/4.7/empty-table-header?application=playwright", + + "id": "empty-table-header", + + "impact": "minor", + +# Example 2 + + "description": "Ensures that lists are structured correctly", + + "help": "
      and
        must only directly contain
      1. ,