diff --git a/docs/architectural-decision-records/005-frontend-a11y-toolset.md b/docs/architectural-decision-records/005-frontend-a11y-toolset.md index 40035553a9..d3e55271fa 100644 --- a/docs/architectural-decision-records/005-frontend-a11y-toolset.md +++ b/docs/architectural-decision-records/005-frontend-a11y-toolset.md @@ -1,5 +1,7 @@ # Define Frontend Toolset for Accessibility Standards +## This ADR has been superseded by [028-updated-frontend-a11y-toolset](028-updated-frontend-a11y-toolset.md) + Decide on a toolset to help ensure MCRRS frontend development follows accessibility standards (specifically WCAG AA guidelines). Tooling is not a replacement for engineers/designers with strong expertise in HTML or manual testing with assistive technology. Thus, the toolset will focus on the types of accessibility issues where tooling is useful. This includes validating HTML attributes, checking basic page structure, ensuring media content has descriptive tags, ensuring that buttons are used appropriately, and checking for basic color contrast. diff --git a/docs/architectural-decision-records/028-updated-frontend-a11y-toolset.md b/docs/architectural-decision-records/028-updated-frontend-a11y-toolset.md new file mode 100644 index 0000000000..01d39ab389 --- /dev/null +++ b/docs/architectural-decision-records/028-updated-frontend-a11y-toolset.md @@ -0,0 +1,60 @@ +# Updated Frontend Toolset for Accessibility Standards + +Decide on a toolset to help ensure MCRRS frontend development follows accessibility standards (specifically WCAG 2.2 AA guidelines). + +Tooling is not a replacement for engineers/designers with strong expertise in HTML or manual testing with assistive technology. Thus, the toolset will focus on the types of accessibility issues where tooling is useful. This includes validating HTML attributes, checking basic page structure, ensuring media content has descriptive tags, ensuring that buttons are used appropriately, and checking for basic color contrast. + +### Considerations + +- Must check React component design for accessibility standards +- Must validate raw HTML markup for accessibility standards +- Must include end to end testing for accessibility standards rather than only unit tests +- Preference for easy options to run in CI + +### Decision Changes + +Updated toolset for accessibility testing and supersedes [005-frontend-a11y-toolset](005-frontend-a11y-toolset.md) + +[storybook addon-a11y][storybook-addon-a11y] - Provides integration with Storybook GUI which allows the team to surface a11y concerns to non-developer stakeholders. This helps make a11y issues visible beyond the code editor or command line. + +[eslint jsx-a11y][eslint-a11y] Provides React-friendly code linting which allows developers to find accessibility issues during local development. + +[cypress-axe][cypress-axe] and [axe-core][axe-core] - Accessibility test runner for end to end testing in the command line or Node.js. End to end testing can catch concerns that are not noticeable in linting or unit tests since tests run on browser. Our previous choice [pa11y][pa11y] and [pa11y-ci][pa11y-ci] seems to not be maintained anymore and no longer works with Cypress end to end testing. + +[jest-axe][jest-axe] - We are no longer using this tool and has been removed `package.json` + +### Pro/Cons + +#### [@axe-core/react][axe-core-react], [@axe-core/cli][axe-core-cli] + +- `+` This react plugin outputs accessibility warnings in Chrome Devtools when the app is running. The cli plugin surfaces a command for running in CI. +- `+` Base standard for most accessibility tools. Customizable for different levels of accessibility compliance. +- `-` Documentation. Couldn’t get it set up quickly, they are in progress of moving their docs and consolidating their npm modules. Minimal documentation and example on best use in CI. +- `-` Heard from other teams that have struggled customization - when used with standard flags it is more strict than needed. + +#### [cypress-axe][cypress-axe] and [axe-core][axe-core] +- `+` `axe-core` is a well known and maintained a11y testing engine +- `+` `axe-core` has great documentation +- `+` `cypress-axe` can run tests with specific a11y standards like `WCAG 2.2 AA`. +- `+` `cypress-axe` is a Cypress plugin for axe-core and most of the configuration for this plugin follows `axe-core` which has great documentation. +- `-` `cypress-axe` itself has sparse documentation +- `-` `cypress-axe` is very minimal in features and if we wanted specific things like reporting, we would have to implement that ourselves. + +#### [Cypress Accessibility][cypress-accessibility] +- `+` Built in accessibility testing by Cypress. +- `-` Still in early access and not much is known about implementation or documentation. +- `-` As of now it looks like it would cost money in addition to what we already pay for Cypress. +- `-` To get access we have to sign up for early access as a trial. + +[storybook-addon-a11y]: https://storybook.js.org/addons/@storybook/addon-a11y +[eslint-a11y]: https://github.com/jsx-eslint/eslint-plugin-jsx-a11y +[pa11y]: https://github.com/pa11y/pa11y +[pa11y-ci]: https://github.com/pa11y/pa11y-ci +[jest-axe]: https://github.com/nickcolley/jest-axe +[axe-core-react]: https://www.npmjs.com/package/@axe-core/react +[axe-core-cli]: https://www.npmjs.com/package/@axe-core/cli +[cypress-audit]: https://github.com/mfrachet/cypress-audit +[lighthouse-ci]: https://github.com/GoogleChrome/lighthouse-ci +[cypress-axe]: https://github.com/component-driven/cypress-axe +[axe-core]: https://github.com/dequelabs/axe-core +[cypress-accessibility]: https://www.cypress.io/blog/introducing-cypress-accessibility diff --git a/package.json b/package.json index 688da72bab..cb6af428ab 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,8 @@ "devDependencies": { "c8": "^10.1.2", "cypress": "^13.13.2", + "axe-core": "^4.10.0", + "cypress-axe": "^1.5.0", "danger": "^11.2.6", "husky": "^9.1.5", "lint-staged": "^15.2.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2a5a617c8..7d59c95aa2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,18 @@ importers: .: devDependencies: + axe-core: + specifier: ^4.10.0 + version: 4.10.0 c8: specifier: ^10.1.2 version: 10.1.2 cypress: specifier: ^13.13.2 version: 13.13.2 + cypress-axe: + specifier: ^1.5.0 + version: 1.5.0(axe-core@4.10.0)(cypress@13.13.2) danger: specifier: ^11.2.6 version: 11.3.1(encoding@0.1.13) @@ -707,9 +713,6 @@ importers: graphql: specifier: ^16.9.0 version: 16.9.0 - protobufjs: - specifier: ^7.3.0 - version: 7.3.2 devDependencies: '@bahmutov/cypress-esbuild-preprocessor': specifier: ^2.2.0 @@ -723,12 +726,18 @@ importers: '@testing-library/cypress': specifier: ^10.0.1 version: 10.0.1(cypress@13.13.2) + axe-core: + specifier: ^4.10.0 + version: 4.10.0 c8: specifier: ^10.1.2 version: 10.1.2 cypress: specifier: ^13.13.2 version: 13.13.2 + cypress-axe: + specifier: ^1.5.0 + version: 1.5.0(axe-core@4.10.0)(cypress@13.13.2) husky: specifier: ^9.1.5 version: 9.1.5 @@ -6296,6 +6305,10 @@ packages: aws4@1.11.0: resolution: {integrity: sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==} + axe-core@4.10.0: + resolution: {integrity: sha512-Mr2ZakwQ7XUAjp7pAwQWRhhK8mQQ6JAaNWSjmjxil0R8BPioMtQsTLOolGYkji1rcL++3dCqZA3zWqpT+9Ew6g==} + engines: {node: '>=4'} + axe-core@4.2.4: resolution: {integrity: sha512-9AiDKFKUCWEQm1Kj4lcq7KFavLqSXdf2m/zJo+NVh4VXlW5iwXRJ6alkKmipCyYorsRnqsICH9XLubP1jBF+Og==} engines: {node: '>=4'} @@ -6304,10 +6317,6 @@ packages: resolution: {integrity: sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==} engines: {node: '>=4'} - axe-core@4.8.1: - resolution: {integrity: sha512-9l850jDDPnKq48nbad8SiEelCv4OrUWrKab/cPj0GScVg6cb6NbCCt/Ulk26QEq5jP9NnGr04Bit1BHyV6r5CQ==} - engines: {node: '>=4'} - axios@0.26.0: resolution: {integrity: sha512-lKoGLMYtHvFrPVt3r+RBMp9nh34N0M8zEfCWqdWZx6phynIEhQqAdydpyBAAG211zlhX9Rgu08cOamy6XjE5Og==} @@ -7163,6 +7172,13 @@ packages: engines: {node: '>= 10'} hasBin: true + cypress-axe@1.5.0: + resolution: {integrity: sha512-Hy/owCjfj+25KMsecvDgo4fC/781ccL+e8p+UUYoadGVM2ogZF9XIKbiM6KI8Y3cEaSreymdD6ZzccbI2bY0lQ==} + engines: {node: '>=10'} + peerDependencies: + axe-core: ^3 || ^4 + cypress: ^10 || ^11 || ^12 || ^13 + cypress-file-upload@5.0.8: resolution: {integrity: sha512-+8VzNabRk3zG6x8f8BWArF/xA/W0VK4IZNx3MV0jFWrJS/qKn8eHfa5nU73P9fOQAgwHFJx7zjg4lwOnljMO8g==} engines: {node: '>=8.2.1'} @@ -13740,10 +13756,10 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.637.0) + '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.609.0) '@aws-sdk/client-sts': 3.609.0 '@aws-sdk/core': 3.609.0 - '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0) '@aws-sdk/middleware-host-header': 3.609.0 '@aws-sdk/middleware-logger': 3.609.0 '@aws-sdk/middleware-recursion-detection': 3.609.0 @@ -14053,10 +14069,10 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.637.0) '@aws-sdk/client-sts': 3.609.0 '@aws-sdk/core': 3.609.0 - '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0) '@aws-sdk/middleware-host-header': 3.609.0 '@aws-sdk/middleware-logger': 3.609.0 '@aws-sdk/middleware-recursion-detection': 3.609.0 @@ -14138,7 +14154,7 @@ snapshots: '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.609.0) '@aws-sdk/client-sts': 3.609.0 '@aws-sdk/core': 3.609.0 - '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0) '@aws-sdk/middleware-host-header': 3.609.0 '@aws-sdk/middleware-logger': 3.609.0 '@aws-sdk/middleware-recursion-detection': 3.609.0 @@ -14759,7 +14775,7 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/client-sts': 3.609.0 '@aws-sdk/core': 3.609.0 - '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0) '@aws-sdk/middleware-host-header': 3.609.0 '@aws-sdk/middleware-logger': 3.609.0 '@aws-sdk/middleware-recursion-detection': 3.609.0 @@ -15057,7 +15073,7 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.609.0) '@aws-sdk/core': 3.609.0 - '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/credential-provider-node': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0) '@aws-sdk/middleware-host-header': 3.609.0 '@aws-sdk/middleware-logger': 3.609.0 '@aws-sdk/middleware-recursion-detection': 3.609.0 @@ -15335,24 +15351,6 @@ snapshots: '@aws-sdk/types': 3.6.1 tslib: 1.14.1 - '@aws-sdk/credential-provider-ini@3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0)': - dependencies: - '@aws-sdk/client-sts': 3.609.0 - '@aws-sdk/credential-provider-env': 3.609.0 - '@aws-sdk/credential-provider-http': 3.609.0 - '@aws-sdk/credential-provider-process': 3.609.0 - '@aws-sdk/credential-provider-sso': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0)) - '@aws-sdk/credential-provider-web-identity': 3.609.0(@aws-sdk/client-sts@3.609.0) - '@aws-sdk/types': 3.609.0 - '@smithy/credential-provider-imds': 3.2.0 - '@smithy/property-provider': 3.1.3 - '@smithy/shared-ini-file-loader': 3.1.4 - '@smithy/types': 3.3.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - aws-crt - '@aws-sdk/credential-provider-ini@3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0)': dependencies: '@aws-sdk/client-sts': 3.637.0 @@ -15415,25 +15413,6 @@ snapshots: '@aws-sdk/types': 3.6.1 tslib: 1.14.1 - '@aws-sdk/credential-provider-node@3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0)': - dependencies: - '@aws-sdk/credential-provider-env': 3.609.0 - '@aws-sdk/credential-provider-http': 3.609.0 - '@aws-sdk/credential-provider-ini': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.609.0) - '@aws-sdk/credential-provider-process': 3.609.0 - '@aws-sdk/credential-provider-sso': 3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0)) - '@aws-sdk/credential-provider-web-identity': 3.609.0(@aws-sdk/client-sts@3.609.0) - '@aws-sdk/types': 3.609.0 - '@smithy/credential-provider-imds': 3.2.0 - '@smithy/property-provider': 3.1.3 - '@smithy/shared-ini-file-loader': 3.1.4 - '@smithy/types': 3.3.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - '@aws-sdk/client-sts' - - aws-crt - '@aws-sdk/credential-provider-node@3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))(@aws-sdk/client-sts@3.637.0)': dependencies: '@aws-sdk/credential-provider-env': 3.609.0 @@ -15545,14 +15524,6 @@ snapshots: '@aws-sdk/types': 3.186.0 tslib: 2.6.3 - '@aws-sdk/credential-provider-web-identity@3.609.0(@aws-sdk/client-sts@3.609.0)': - dependencies: - '@aws-sdk/client-sts': 3.609.0 - '@aws-sdk/types': 3.609.0 - '@smithy/property-provider': 3.1.3 - '@smithy/types': 3.3.0 - tslib: 2.6.3 - '@aws-sdk/credential-provider-web-identity@3.609.0(@aws-sdk/client-sts@3.637.0)': dependencies: '@aws-sdk/client-sts': 3.637.0 @@ -16145,7 +16116,7 @@ snapshots: '@aws-sdk/token-providers@3.609.0(@aws-sdk/client-sso-oidc@3.609.0(@aws-sdk/client-sts@3.609.0))': dependencies: - '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.609.0) + '@aws-sdk/client-sso-oidc': 3.609.0(@aws-sdk/client-sts@3.637.0) '@aws-sdk/types': 3.609.0 '@smithy/property-provider': 3.1.3 '@smithy/shared-ini-file-loader': 3.1.4 @@ -20741,7 +20712,7 @@ snapshots: '@storybook/addon-a11y@8.2.9(storybook@8.2.9(@babel/preset-env@7.24.8(@babel/core@7.24.7)))': dependencies: '@storybook/addon-highlight': 8.2.9(storybook@8.2.9(@babel/preset-env@7.24.8(@babel/core@7.24.7))) - axe-core: 4.8.1 + axe-core: 4.10.0 storybook: 8.2.9(@babel/preset-env@7.24.8(@babel/core@7.24.7)) '@storybook/addon-actions@8.2.9(storybook@8.2.9(@babel/preset-env@7.24.8(@babel/core@7.24.7)))': @@ -22375,12 +22346,12 @@ snapshots: aws4@1.11.0: {} + axe-core@4.10.0: {} + axe-core@4.2.4: {} axe-core@4.7.0: {} - axe-core@4.8.1: {} - axios@0.26.0: dependencies: follow-redirects: 1.15.6 @@ -23523,6 +23494,11 @@ snapshots: dependencies: minimist: 1.2.8 + cypress-axe@1.5.0(axe-core@4.10.0)(cypress@13.13.2): + dependencies: + axe-core: 4.10.0 + cypress: 13.13.2 + cypress-file-upload@5.0.8(cypress@13.13.2): dependencies: cypress: 13.13.2 diff --git a/services/cypress/README.md b/services/cypress/README.md index 1c81928c5e..71c1994349 100644 --- a/services/cypress/README.md +++ b/services/cypress/README.md @@ -68,6 +68,107 @@ CI deployment is congifured in `.github/workflows/deploy.yml`. LaunchDarkly integration docs can be found in [launch-darkly-testing-approach.md](../../docs/technical-design/launch-darkly-testing-approach.md#feature-flag-cypress-testing) +### Accessibility Testing + +We are using the `cypress-axe` plugin to run our a11y tests. `cypress-axe` uses the `axe-core` testing engine. + +Our implementation is almost exactly like the [documentation](https://github.com/component-driven/cypress-axe) for `cypress-axe`. The only difference is our custom Cypress command `cy.checkA11yWithWcag22aa()` which is configured to only use `WCAG 2.2 AA` standard for tests. This follows our to follow our ADR [frontend-a11y-toolset](/docs/architectural-decision-records/005-frontend-a11y-toolset.md). + +To run axe inject the `axe-core` runtime using `cy.injectAxe()` after `cy.visit()`. Then you can call the custom command `cy.checkA11yWithWcag22aa()` to check a11y against the page. Each time you use `cy.visit()` you will need to inject the runtime again. + +The `checkA11yWithWcag22aa` command configures `cy.checkA11y()` with the WCAG 2.2 AA standard we use for the MC-Review app. In the `cy.checkA11y()` there are more options we can configure the check with. The options are specified in the [axe-core documentation](https://www.deque.com/axe/core-documentation/api-documentation). + +In the example below, you will notice that there are more than the WCAG 2.2 AA standard being used in the test. This is because the rules for each standard are not cumulative. So in order to test all rules, all the standards must be included. If we only include `'wcag22aa'` then only the specific rules added in for that standard will be tested. +```typescript +Cypress.Commands.add( + 'checkA11yWithWcag22aa', + () => { + cy.checkA11y('', { + runOnly: { + type: 'tag', + values: ['wcag2a','wcag2aa', 'wcag21a', 'wcag21aa','wcag22aa'] + }, + }, terminalLog) + } +) +``` + +In the example below, `cy.logInAsStateUser()` calls `cy.visit()` so we can inject the runtime after and then run the a11y checks. +Using the app to navigate allows us to check a11y without injecting the runtime again because it is still available + +```typescript + it('has not a11y errors with submission form and and form erros', () => { + // 438-attestation still needs to go through design, there is an a11y violation for links and spacing + cy.interceptFeatureFlags({'438-attestation': false}) + cy.logInAsStateUser() + + // Inject the axe run-time + cy.injectAxe() + + // Start a base contract only submissions + cy.findByTestId('state-dashboard-page').should('exist') + cy.findByRole('link', { name: 'Start new submission' }).click() + + // Check accessibility on Submission type page + cy.findByRole('heading', { level: 1, name: /New submission/ }) + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() + + cy.fillOutContractActionAndRateCertification() + cy.deprecatedNavigateV1Form('CONTINUE_FROM_START_NEW') + + cy.findByRole('heading', { level: 2, name: /Contract details/ }) + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() + ... +``` + +In the next portion of the same test, we are then navigating by direct links using `cy.visit()` wrapped by our custom command `navigateFormByDirectLink()`. Now that we called `cy.visit()` the last injection of the runtime is unavailable and will need to be re-injected. + +```typescript + ... + // Check accessibility on rate details page + cy.navigateFormByDirectLink(`${submissionURL}edit/rate-details`) + cy.findByRole('radiogroup', { + name: /Was this rate certification included with another submission?/, + }) + .should('exist') + .within(() => { + cy.findByText('No, this rate certification was not included with any other submissions').click() + }) + cy.injectAxe() + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() + + //Check accessibility on contacts page + cy.navigateFormByDirectLink(`${submissionURL}edit/contacts`) + cy.findByRole('heading', { level: 2, name: /Contacts/ }) + cy.injectAxe() + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() + ... +``` + ### Direct API Requests The reason for making direct API request through Cypress was to speed up testing by quickly making a new submission to test against. For example, the `questionResponse.spec` test only tests Q&A features but needed a submission in order to get to Q&A, so a new submission was needed and without API request Cypress had to manually go through the state submission form. Now we make a new submission using direct API request which is much faster. @@ -82,7 +183,6 @@ We mimicked the same setup in the application in Cypress to make graphql request - In `cypress.config` under `setupNodeEvents` there is a task, `readGraphQLSchema`, that can be called in Cypress to load our `GraphQL` schema and convert it to `gql`. ```ts on('task', { - pa11y: pa11y(), readGraphQLSchema() { const gqlSchema = fs.readFileSync( path.resolve(__dirname, './gen/schema.graphql'), diff --git a/services/cypress/cypress.config.ts b/services/cypress/cypress.config.ts index ec00c676e0..ad7d2585bb 100644 --- a/services/cypress/cypress.config.ts +++ b/services/cypress/cypress.config.ts @@ -1,6 +1,5 @@ const { defineConfig } = require('cypress') const { gql } = require('@apollo/client') -const { pa11y, prepareAudit } = require('@cypress-audit/pa11y') const fs = require('fs') const path = require('path') const createBundler = require('@bahmutov/cypress-esbuild-preprocessor') @@ -49,13 +48,9 @@ module.exports = defineConfig({ process.env.COGNITO_IDENTITY_POOL_ID newConfig.env.COGNITO_USER_POOL_WEB_CLIENT_ID = process.env.COGNITO_USER_POOL_WEB_CLIENT_ID - on('before:browser:launch', (browser, launchOptions) => { - prepareAudit(launchOptions) - }) // Reads graphql schema and converts it to gql for apollo client. on('task', { - pa11y: pa11y(), readGraphQLSchema() { const gqlSchema = fs.readFileSync( path.resolve(__dirname, './gen/schema.graphql'), @@ -63,6 +58,16 @@ module.exports = defineConfig({ ) return gql(`${gqlSchema}`) }, + log(message) { + console.log(message) + + return null + }, + table(message) { + console.table(message) + + return null + } }) return newConfig }, diff --git a/services/cypress/integration/stateWorkflow/stateSubmissionForm/accessiblility.spec.ts b/services/cypress/integration/stateWorkflow/stateSubmissionForm/accessiblility.spec.ts index f151ea0eec..fa41f1d2ef 100644 --- a/services/cypress/integration/stateWorkflow/stateSubmissionForm/accessiblility.spec.ts +++ b/services/cypress/integration/stateWorkflow/stateSubmissionForm/accessiblility.spec.ts @@ -1,73 +1,114 @@ +import { stateUser } from '../../../utils/apollo-test-utils'; + describe('state user in state submission form', () => { beforeEach(() => { cy.stubFeatureFlags() cy.interceptGraphQL() }) - // SKIPPED because we currently have a bug that looks related to https://github.com/mfrachet/cypress-audit/issues/136#issuecomment-1311236777 - // this started when we upgraded node and serverless and started default to ipv6 which prevented tests - // current this keeps pa11y from running - it.skip('can fill out contract only submission with no a11y errors', () => { - // goal of this test is to check every single form page and navigation (going backwards, forwards or save as draft with new info) - + it.only('has no a11y violations on submission form with form input errors', () => { + // 438-attestation still needs to go through design, there is an a11y violation for links and spacing + cy.interceptFeatureFlags({'438-attestation': false}) cy.logInAsStateUser() - // Start a base contract only submissions - cy.startNewContractOnlySubmissionWithBaseContract() - + // Inject the axe run-time + cy.injectAxe() - cy.findByRole('heading', { - level: 2, - name: /Contract details/, - timeout: 10_000, - }) + // Start a base contract only submissions + cy.findByTestId('state-dashboard-page').should('exist') + cy.findByRole('link', { name: 'Start new submission' }).click() - // check contract details a11y - cy.pa11y({ - hideElements: '.usa-step-indicator', - actions: ['wait for element #form-guidance to be visible'], - threshold: 6, - hideElements: '.usa-step-indicator', - }) - cy.deprecatedNavigateV1Form('BACK') - // check submissionType a11y - cy.pa11y({ - hideElements: '.usa-step-indicator', - actions: ['wait for element #form-guidance to be visible'], - threshold: 6, - hideElements: '.usa-step-indicator', - }) + // Check accessibility on Submission type page + cy.findByRole('heading', { level: 1, name: /New submission/ }) + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() - cy.deprecatedNavigateV1Form('CONTINUE') - cy.deprecatedNavigateV1Form('CONTINUE') + cy.fillOutContractActionAndRateCertification() + cy.navigateContractForm('CONTINUE_FROM_START_NEW') - cy.findByRole('heading', { level: 2, name: /Contacts/ }) - cy.fillOutStateContact() + cy.findByRole('heading', { level: 2, name: /Contract details/ }) + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() - // check contacts a11y - cy.pa11y({ - hideElements: '.usa-step-indicator', - actions: ['wait for element #form-guidance to be visible'], - threshold: 6, - hideElements: '.usa-step-indicator', - }) + cy.location().then((fullUrl) => { + const submissionURL = fullUrl.toString().replace( + 'edit/contract-details', + '' + ) - cy.deprecatedNavigateV1Form('SAVE_DRAFT') - cy.findByRole('heading', { level: 1, name: /Submissions dashboard/ }) + // Check accessibility on rate details page + cy.navigateFormByDirectLink(`${submissionURL}edit/rate-details`) + cy.findByRole('radiogroup', { + name: /Was this rate certification included with another submission?/, + }) + .should('exist') + .within(() => { + cy.findByText('No, this rate certification was not included with any other submissions').click() + }) + cy.injectAxe() + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() + //Check accessibility on contacts page + cy.navigateFormByDirectLink(`${submissionURL}edit/contacts`) + cy.findByRole('heading', { level: 2, name: /Contacts/ }) + cy.injectAxe() + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() - cy.deprecatedNavigateV1Form('CONTINUE') - // skip documents page - that will be deleted soon - cy.deprecatedNavigateV1Form('CONTINUE') + //Check accessibility on documents page + cy.navigateFormByDirectLink(`${submissionURL}edit/documents`) + cy.findByRole('heading', { level: 2, name: /Supporting documents/ }) + cy.injectAxe() + cy.findByRole('button', { + name: 'Continue', + }).should('not.have.attr', 'aria-disabled') + cy.findByRole('button', { + name: 'Continue', + }).safeClick() + cy.checkA11yWithWcag22aa() - // Check that we end up on Review and Submit + //Check accessibility on review and submit page + cy.navigateFormByDirectLink(`${submissionURL}edit/review-and-submit`) cy.findByRole('heading', { level: 2, name: /Review and submit/ }) + cy.injectAxe() + cy.checkA11yWithWcag22aa() - // check review and submit a11y - cy.pa11y({ - hideElements: '.usa-step-indicator', - actions: ['wait for element #form-guidance to be visible'], - threshold: 6, - hideElements: '.usa-step-indicator', + //Check the dashboard + cy.navigateContractRatesForm('SAVE_DRAFT', false) + cy.checkA11yWithWcag22aa() }) + }) + + it('has no a11y violations on CMS dashboards', () => { + cy.apiCreateAndSubmitContractWithRates(stateUser()).then(() => { + cy.logInAsCMSUser() + cy.injectAxe() + //check submissions tab + cy.checkA11yWithWcag22aa() + + //check rate reviews tab + cy.findByRole('tab', { name: 'Rate reviews' }).should('exist').click() + cy.checkA11yWithWcag22aa() }) }) +}) diff --git a/services/cypress/package.json b/services/cypress/package.json index 38f0f842bb..19bf728ebf 100644 --- a/services/cypress/package.json +++ b/services/cypress/package.json @@ -43,8 +43,10 @@ "@cypress-audit/pa11y": "^1.3.0", "@cypress/code-coverage": "^3.10.0", "@testing-library/cypress": "^10.0.1", + "axe-core": "^4.10.0", "c8": "^10.1.2", "cypress": "^13.13.2", + "cypress-axe": "^1.5.0", "husky": "^9.1.5", "lint-staged": "^15.2.2", "prettier": "^3.3.3" @@ -55,7 +57,6 @@ "cypress-pipe": "^2.0.0", "aws-amplify": "^5.0.10", "axios": "^1.7.4", - "graphql": "^16.9.0", - "protobufjs": "^7.3.0" + "graphql": "^16.9.0" } } diff --git a/services/cypress/support/accessibilityCommands.ts b/services/cypress/support/accessibilityCommands.ts new file mode 100644 index 0000000000..b875a260f2 --- /dev/null +++ b/services/cypress/support/accessibilityCommands.ts @@ -0,0 +1,43 @@ +import {getRules} from 'axe-core' + +function terminalLog(violations: Record[]) { + console.log(violations) + cy.task( + 'log', + `${violations.length} accessibility violation${ + violations.length === 1 ? '' : 's' + } ${violations.length === 1 ? 'was' : 'were'} detected` + ) + // pluck specific keys to keep the table readable + const violationData: Record[] = violations.map( + ({ id, impact, description, nodes }) => ({ + id, + impact, + description, + nodes: nodes.length + }) + ) + + cy.task('table', violationData) +} + +Cypress.Commands.add( + 'checkA11yWithWcag22aa', + () => { + cy.checkA11y('', { + runOnly: { + type: 'tags', + values: ['wcag2a','wcag2aa', 'wcag21a', 'wcag21aa','wcag22aa'] + }, + rules: { + // Rule skipped. It can be removed from config when https://jiraent.cms.gov/browse/MCR-4421 has been + // completed + 'aria-allowed-attr': { enabled: false }, + // Both of these rules are skipped. They can be removed from config when + // https://jiraent.cms.gov/browse/MCR-4420 has been completed + 'dlitem': { enabled: false }, + 'definition-list': { enabled: false } + } + }, terminalLog) + } +) diff --git a/services/cypress/support/e2e.ts b/services/cypress/support/e2e.ts index 3cd559d4c7..321230abbd 100644 --- a/services/cypress/support/e2e.ts +++ b/services/cypress/support/e2e.ts @@ -1,4 +1,5 @@ import '@cypress/code-coverage/support' +import 'cypress-axe' Cypress.on('uncaught:exception', (err, runnable) => { // returning false here prevents Cypress from diff --git a/services/cypress/support/index.ts b/services/cypress/support/index.ts index 5ce7b5799a..d303605185 100644 --- a/services/cypress/support/index.ts +++ b/services/cypress/support/index.ts @@ -26,6 +26,7 @@ import { FeatureFlagSettings, } from '../../app-web/src/common-code/featureFlags' import './apiCommands' +import './accessibilityCommands' import { Contract, HealthPlanPackage } from '../gen/gqlClient'; import { CMSUserType, DivisionType } from '../utils/apollo-test-utils'; import { StateUserType } from '../../app-api/src/domain-models'; @@ -110,6 +111,7 @@ declare global { division: DivisionType }): void + // Direct API commands apiCreateAndSubmitContractOnlySubmission(stateUser: StateUserType): Cypress.Chainable apiCreateAndSubmitContractWithRates(stateUser: StateUserType): Cypress.Chainable apiDeprecatedCreateSubmitHPP(stateUser: StateUserType, formData?: Partial): Cypress.Chainable @@ -117,7 +119,11 @@ declare global { apiAssignDivisionToCMSUser(cmsUser: CMSUserType, division: DivisionType): Cypress.Chainable apiCreateAndSubmitContractWithRates(stateUser: StateUserType): Cypress.Chainable + // GraphQL intercept commands interceptGraphQL(): void + + // Accessibility Commands + checkA11yWithWcag22aa(): void } } } diff --git a/services/cypress/support/launchDarklyCommands.ts b/services/cypress/support/launchDarklyCommands.ts index 4c2cce6c57..f0dc718627 100644 --- a/services/cypress/support/launchDarklyCommands.ts +++ b/services/cypress/support/launchDarklyCommands.ts @@ -41,10 +41,14 @@ Cypress.Commands.add( JSON.stringify(featureFlagObject) ) + const clientSDKMatchers = Cypress.env('AUTH_MODE') === 'LOCAL' ? + { method: 'GET', pathname: /^\/ld-clientsdk(\/.*)?$/ } : + { method: 'GET', hostname: /\.*clientsdk\.launchdarkly\.us/ } + // Intercepts LD request and returns with our own feature flags and values. return cy .intercept( - { method: 'GET', hostname: /\.*clientsdk\.launchdarkly\.us/ }, + clientSDKMatchers, { body: featureFlagObject } ) .as('LDApp') @@ -54,24 +58,32 @@ Cypress.Commands.add( // Intercepting feature flag api calls and returns some response. This should stop the app from calling making requests to LD. Cypress.Commands.add('stubFeatureFlags', () => { // ignore api calls to events endpoint + const eventMatchers = Cypress.env('AUTH_MODE') === 'LOCAL' ? + { method: 'POST', pathname: /^\/ld-events(\/.*)?$/ } : + { method: 'POST', hostname: /\.*events\.launchdarkly\.us/ } + cy.intercept( - { method: 'POST', hostname: /\.*events\.launchdarkly\.us/ }, + eventMatchers, // { body: {} } (req) => { req.on('response', (res) => { - res.setDelay(60000) + res.setDelay(15000) }) req.reply({ body: {} }) } ).as('LDEvents') + const clientStreamMatchers = Cypress.env('AUTH_MODE') === 'LOCAL' ? + { method: 'GET', pathname: /^\/ld-clientstream(\/.*)?$/ } : + { method: 'GET', hostname: /\.*clientstream\.launchdarkly\.us/ } + // turn off push updates from LaunchDarkly (EventSource) cy.intercept( - { method: 'GET', hostname: /\.*clientstream\.launchdarkly\.us/ }, + clientStreamMatchers, // access the request handler and stub a response (req) => { req.on('response', (res) => { - res.setDelay(60000) + res.setDelay(15000) }) req.reply('data: no streaming feature flag data here\n\n', { 'content-type': 'text/event-stream; charset=utf-8', diff --git a/services/cypress/support/loginCommands.ts b/services/cypress/support/loginCommands.ts index 12429d32e8..94c12ef753 100644 --- a/services/cypress/support/loginCommands.ts +++ b/services/cypress/support/loginCommands.ts @@ -1,5 +1,3 @@ -import { aliasQuery } from '../utils/graphql-test-utils' - Cypress.Commands.add('logInAsStateUser', () => { // Set up gql intercept for requests on app load diff --git a/services/cypress/support/questionResponseCommands.ts b/services/cypress/support/questionResponseCommands.ts index e25f16ffcb..18dcbeef66 100644 --- a/services/cypress/support/questionResponseCommands.ts +++ b/services/cypress/support/questionResponseCommands.ts @@ -1,5 +1,3 @@ -import {aliasMutation, aliasQuery} from '../utils/graphql-test-utils'; - Cypress.Commands.add( 'addQuestion', ({ documentPath }: { documentPath: string }) => { diff --git a/services/cypress/tsconfig.json b/services/cypress/tsconfig.json index 16c6f54b0d..4db3fa5dfc 100644 --- a/services/cypress/tsconfig.json +++ b/services/cypress/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "target": "ES2017", "lib": ["ES2017", "dom"], - "types": ["cypress", "@testing-library/cypress", "cypress-file-upload", "node"] + "types": ["cypress", "cypress-axe", "@testing-library/cypress", "cypress-file-upload", "node"] }, "include": ["**/*.ts"] }