diff --git a/README.md b/README.md index 9eec31c9..a7ed5acc 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Licensed under the MIT License. [![npm](https://img.shields.io/npm/v/axe-sarif-converter.svg)](https://www.npmjs.com/package/axe-sarif-converter) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) -Convert [axe-core](https://github.com/dequelabs/axe-core) accessibility scan results to the [SARIF format](http://sarifweb.azurewebsites.net/). +Convert [axe-core](https://github.com/dequelabs/axe-core) accessibility scan results to the [SARIF format](http://sarifweb.azurewebsites.net/). Provides both a TypeScript API and a CLI tool. Use this with the [Sarif Viewer Build Tab Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=sariftools.sarif-viewer-build-tab) to visualize accessibility scan results in the build results of an [Azure Pipelines](https://azure.microsoft.com/en-us/services/devops/pipelines/) build. @@ -46,6 +46,18 @@ test('my accessibility test', async () => { } ``` +You can also use axe-sarif-converter as a command line tool: + +```bash +# axe-cli is used here for example purposes only; you could also run axe-core +# using your library of choice and JSON.stringify the results. +npx axe-cli https://accessibilityinsights.io --save ./sample-axe-results.json + +npx axe-sarif-converter --input-files ./sample-axe-results.json --output-file ./sample-axe-results.sarif +``` + +See `npx axe-sarif-converter --help` for full command line option details. + ## Samples The [microsoft/axe-pipelines-samples](https://github.com/microsoft/axe-pipelines-samples) project contains full sample code that walks you through integrating this library into your project, from writing a test to seeing results in Azure Pipelines. @@ -61,6 +73,26 @@ Note that the SARIF format _does not use semantic versioning_, and there are bre ## Contributing +To get started working on the project: + +1. Install dependencies: + + - Install [Node.js](https://nodejs.org/en/download/) (LTS version) + - `npm install -g yarn` + - `yarn install` + +1. Run all build, lint, and test steps: + + - `yarn precheckin` + +1. Run the CLI tool with your changes: + + - `yarn build` + - `node dist/cli.js` + - Alternately, register a linked global `axe-sarif-converter` command with `npm install && npm link` (yarn doesn't work for this; see [yarnpkg/yarn#1585](https://github.com/yarnpkg/yarn/issues/1585)) + +### Contributor License Agreement + This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. @@ -69,6 +101,8 @@ When you submit a pull request, a CLA-bot will automatically determine whether y a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. +### Code of Conduct + This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f4a83be0..5d7733e9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,9 @@ jobs: - script: yarn copyrightheaders displayName: yarn copyrightheaders + - script: yarn build + displayName: yarn build + - script: yarn test -- --ci displayName: yarn test @@ -48,9 +51,6 @@ jobs: condition: always() displayName: publish sarif results - - script: yarn build - displayName: yarn build - - script: yarn semantic-release displayName: yarn semantic-release (master branch only) env: diff --git a/jest.config.js b/jest.config.js index 6da868c8..b70ced75 100644 --- a/jest.config.js +++ b/jest.config.js @@ -15,7 +15,13 @@ module.exports = { moduleFileExtensions: ['ts', 'js'], rootDir: rootDir, collectCoverage: true, - collectCoverageFrom: ['./**/*.ts', '!./**/*.test.ts'], + collectCoverageFrom: [ + '/**/*.ts', + '!/**/*.test.ts', + // The CLI is tested via integration tests that spawn separate node + // processes, so coverage information on this file isn't accurate + '!/cli.ts', + ], coverageReporters: ['json', 'lcov', 'text', 'cobertura'], testMatch: [`${currentDir}/**/*.test.(ts|js)`], reporters: [ diff --git a/package.json b/package.json index 54532c6d..b0a5eabc 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.0.0-managed-by-semantic-release", "description": "Convert axe-core accessibility scan results to the SARIF format", "main": "dist/index.js", + "bin": "dist/cli.js", "types": "dist/index.d.js", "files": [ "dist/", @@ -14,12 +15,14 @@ }, "dependencies": { "@types/sarif": ">=2.1.1 <=2.1.2", - "axe-core": "^3.2.2" + "axe-core": "^3.2.2", + "yargs": "^14.0.0" }, "devDependencies": { "@types/jest": "^24.0.15", "@types/lodash": "^4.14.136", "@types/node": "^12.6.8", + "@types/yargs": "^13.0.2", "jest": "^24.8.0", "jest-circus": "^24.8.0", "jest-junit": "^8.0.0", diff --git a/src/__snapshots__/cli.test.ts.snap b/src/__snapshots__/cli.test.ts.snap new file mode 100644 index 00000000..eacf8be5 --- /dev/null +++ b/src/__snapshots__/cli.test.ts.snap @@ -0,0 +1,28 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`axe-sarif-converter CLI prints help info with --help: help_stdout 1`] = ` +"axe-sarif-converter: Converts JSON files containing axe-core Result object(s) +into SARIF files + +Options: + --help Show help [boolean] + --version Show version number [boolean] + --input-files, -i Input JSON file(s) containing axe-core Result object(s). + Does not support globs. Each input file may consist of + either a single root-level axe-core Results object or a + root-level array of axe-core Results objects. + [array] [required] + --output-file, -o Output SARIF file. Multiple input files (or input files + containing multiple Result objects) will be combined into + one output file with a SARIF Run per axe-core Result. + [string] [required] + --verbose, -v Enables verbose console output. [boolean] [default: false] + --pretty, -p Includes line breaks and indentation in the output. + [boolean] [default: false] + --force, -f Overwrites the output file if it already exists. + [boolean] [default: false] + +Examples: + axe-sarif-converter -i axe-results.json -o axe-results.sarif +" +`; diff --git a/src/cli.test.ts b/src/cli.test.ts new file mode 100644 index 00000000..e5cba66b --- /dev/null +++ b/src/cli.test.ts @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import * as child_process from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { promisify } from 'util'; + +// tslint:disable: mocha-no-side-effect-code + +describe('axe-sarif-converter CLI', () => { + beforeAll(async () => { + await ensureDirectoryExists(testResultsDir); + }); + + it('prints help info with --help', async () => { + const output = await invokeCliWith('--help'); + expect(output.stderr).toBe(''); + expect(output.stdout).toMatchSnapshot('help_stdout'); + }); + + it('prints version number from package.json with --version', async () => { + const output = await invokeCliWith('--version'); + expect(output.stderr).toBe(''); + expect(output.stdout).toBe('0.0.0-managed-by-semantic-release\n'); + }); + + it('requires the -i parameter', async () => { + try { + await invokeCliWith(`-o irrelevant.sarif`); + fail('Should have returned non-zero exit code'); + } catch (e) { + expect(e.stderr).toMatch('Missing required argument: input-files'); + } + }); + + it('requires the -o parameter', async () => { + try { + await invokeCliWith(`-i irrelevant.json`); + fail('Should have returned non-zero exit code'); + } catch (e) { + expect(e.stderr).toMatch('Missing required argument: output-file'); + } + }); + + it('supports conversion from axe-cli style list of results', async () => { + const outputFile = path.join(testResultsDir, 'axe-cli.sarif'); + await deleteIfExists(outputFile); + + const output = await invokeCliWith(`-i ${axeCliFile} -o ${outputFile}`); + + expect(output.stderr).toBe(''); + expect(output.stdout).toBe(''); + + const outputJson = JSON.parse((await readFile(outputFile)).toString()); + expect(outputJson.runs.length).toBe(1); + }); + + it('supports basic conversion with short-form i/o args', async () => { + const outputFile = path.join(testResultsDir, 'basic_short.sarif'); + await deleteIfExists(outputFile); + + const output = await invokeCliWith( + `-i ${basicAxeV2File} -o ${outputFile}`, + ); + + expect(output.stderr).toBe(''); + expect(output.stdout).toBe(''); + expectSameJSONContent(outputFile, basicSarifFile); + }); + + it('supports basic conversion with long-form i/o args', async () => { + const outputFile = path.join(testResultsDir, 'basic_long.sarif'); + await deleteIfExists(outputFile); + + const output = await invokeCliWith( + `--input-files ${basicAxeV2File} --output-file ${outputFile}`, + ); + + expect(output.stderr).toBe(''); + expect(output.stdout).toBe(''); + expectSameJSONContent(outputFile, basicSarifFile); + }); + + it("doesn't overwrite existing files by default", async () => { + const outputFile = path.join( + testResultsDir, + 'overwrite_no_force.sarif', + ); + await writeFile(outputFile, 'preexisting content'); + + try { + await invokeCliWith(`-i ${basicAxeV2File} -o ${outputFile}`); + fail('Should have returned non-zero exit code'); + } catch (e) { + expect(e.code).toBeGreaterThan(0); + expect(e.stderr).toMatch('Did you mean to use --force?'); + } + + const outputFileContent = (await readFile(outputFile)).toString(); + expect(outputFileContent).toBe('preexisting content'); + }); + + it.each(['-f', '--force'])('overwrites files with %s', async arg => { + const outputFile = path.join(testResultsDir, `overwrite_${arg}.sarif`); + await writeFile(outputFile, 'preexisting content'); + + await invokeCliWith(`-i ${basicAxeV2File} -o ${outputFile} ${arg}`); + + expectSameJSONContent(outputFile, basicSarifFile); + }); + + it.each(['-v', '--verbose'])('emits verbose output', async verboseArg => { + const outputFile = path.join( + testResultsDir, + 'emits_verbose_output.sarif', + ); + await deleteIfExists(outputFile); + + const output = await invokeCliWith( + `-i ${basicAxeV2File} -o ${outputFile} ${verboseArg}`, + ); + + expect(output.stderr).toBe(''); + expect(output.stdout).toContain(basicAxeV2File); + expect(output.stdout).toContain(outputFile); + + expectSameJSONContent(outputFile, basicSarifFile); + }); + + it.each(['-p', '--pretty'])('pretty-prints', async prettyArg => { + const outputFile = path.join(testResultsDir, 'pretty-prints.sarif'); + await deleteIfExists(outputFile); + + await invokeCliWith( + `-i ${basicAxeV2File} -o ${outputFile} ${prettyArg}`, + ); + + const outputContents = (await readFile(outputFile)).toString(); + + const lines = outputContents.split('\n'); + expect(lines.length).toBeGreaterThan(100); + expect(lines.every(line => line.length < 200)).toBe(true); + }); + + async function invokeCliWith( + args: string, + ): Promise<{ stderr: string; stdout: string }> { + return await exec(`node ${__dirname}/../dist/cli.js ${args}`); + } + + const testResourcesDir = path.join(__dirname, 'test-resources'); + const testResultsDir = path.join(__dirname, '..', 'test-results'); + const basicAxeV2File = path.join( + testResourcesDir, + 'basic-axe-v3.2.2-reporter-v2.json', + ); + const basicSarifFile = path.join( + testResourcesDir, + 'basic-axe-v3.2.2-sarif-v2.1.2.sarif', + ); + const axeCliFile = path.join(testResourcesDir, 'axe-cli-v3.1.1.json'); + + const mkdir = promisify(fs.mkdir); + const writeFile = promisify(fs.writeFile); + const readFile = promisify(fs.readFile); + const unlink = promisify(fs.unlink); + const exec = promisify(child_process.exec); + + async function deleteIfExists(path: string): Promise { + try { + await unlink(path); + } catch (e) { + if (e.code != 'ENOENT') { + throw e; + } + } + } + + async function ensureDirectoryExists(path: string): Promise { + try { + await mkdir(path); + } catch (e) { + if (e.code !== 'EEXIST') { + throw e; + } + } + } + + async function expectSameJSONContent( + actualFile: string, + expectedFile: string, + ) { + const actualContentBuffer = await readFile(actualFile); + const actualJSONContent = JSON.parse(actualContentBuffer.toString()); + + const expectedContentBuffer = await readFile(expectedFile); + const expectedJSONContent = JSON.parse( + expectedContentBuffer.toString(), + ); + + expect(actualJSONContent).toStrictEqual(expectedJSONContent); + } +}); diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 00000000..c3f3356c --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,121 @@ +#!/usr/bin/env node +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as fs from 'fs'; +import { Log } from 'sarif'; +import * as yargs from 'yargs'; +import { convertAxeToSarif } from '.'; + +type Arguments = { + 'input-files': string[]; + 'output-file': string; + verbose: boolean; + pretty: boolean; + force: boolean; +}; + +const argv: Arguments = yargs + .scriptName('axe-sarif-converter') + .version() // inferred from package.json + .usage( + '$0: Converts JSON files containing axe-core Result object(s) into SARIF files', + ) + .example('$0 -i axe-results.json -o axe-results.sarif', '') + .option('input-files', { + alias: 'i', + describe: + 'Input JSON file(s) containing axe-core Result object(s). Does not support globs. Each input file may consist of either a single root-level axe-core Results object or a root-level array of axe-core Results objects.', + demandOption: true, + type: 'string', + }) + .array('input-files') + .option('output-file', { + alias: 'o', + describe: + 'Output SARIF file. Multiple input files (or input files containing multiple Result objects) will be combined into one output file with a SARIF Run per axe-core Result.', + demandOption: true, + type: 'string', + }) + .option('verbose', { + alias: 'v', + describe: 'Enables verbose console output.', + default: false, + type: 'boolean', + }) + .option('pretty', { + alias: 'p', + describe: 'Includes line breaks and indentation in the output.', + default: false, + type: 'boolean', + }) + .option('force', { + alias: 'f', + describe: 'Overwrites the output file if it already exists.', + default: false, + type: 'boolean', + }).argv; + +const verboseLog = argv.verbose ? console.log : () => {}; + +function exitWithErrorMessage(message: string) { + console.error(message); + process.exit(1); +} + +function flatten(nestedArray: T[][]): T[] { + return nestedArray.reduce( + (accumulator, next) => accumulator.concat(next), + [], + ); +} + +const sarifLogs: Log[] = flatten( + argv['input-files'].map((inputFilePath, index) => { + verboseLog( + `Reading input file ${index + 1}/${ + argv['input-files'].length + } ${inputFilePath}`, + ); + + // tslint:disable-next-line: non-literal-fs-path + const rawInputFileContents = fs.readFileSync(inputFilePath); + const inputFileJson = JSON.parse(rawInputFileContents.toString()); + if (Array.isArray(inputFileJson)) { + // Treating as array of axe results, like axe-cli produces + return inputFileJson.map(convertAxeToSarif); + } else { + // Treating as a single axe results object, like + // JSON.stringify(await axe.run(...)) would produce + return [convertAxeToSarif(inputFileJson)]; + } + }), +); + +verboseLog(`Aggregating converted input file(s) into one SARIF log`); +const combinedLog: Log = { + ...sarifLogs[0], + runs: flatten(sarifLogs.map(log => log.runs)), +}; + +verboseLog(`Formatting SARIF data into file contents`); +const jsonSpacing = argv.pretty ? 2 : undefined; +const outputFileContent = JSON.stringify(combinedLog, null, jsonSpacing); + +verboseLog(`Writing output file ${argv['output-file']}`); +try { + // tslint:disable-next-line: non-literal-fs-path + fs.writeFileSync(argv['output-file'], outputFileContent, { + flag: argv.force ? 'w' : 'wx', + }); +} catch (e) { + if (e.code == 'EEXIST') { + exitWithErrorMessage( + `Error: EEXIST: Output file ${argv['output-file']} already exists. Did you mean to use --force?`, + ); + } else { + throw e; + } +} + +verboseLog(`Done`); diff --git a/src/test-resources/axe-cli-v3.1.1.json b/src/test-resources/axe-cli-v3.1.1.json new file mode 100644 index 00000000..bc8f2b95 --- /dev/null +++ b/src/test-resources/axe-cli-v3.1.1.json @@ -0,0 +1,10760 @@ +[ + { + "inapplicable": [ + { + "description": "Ensures every accesskey attribute value is unique", + "help": "accesskey attribute value must be unique", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/accesskeys?application=webdriverjs", + "id": "accesskeys", + "impact": null, + "nodes": [], + "tags": [ + "best-practice", + "cat.keyboard" + ] + }, + { + "description": "Ensures elements of image maps have alternate text", + "help": "Active elements must have alternate text", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/area-alt?application=webdriverjs", + "id": "area-alt", + "impact": null, + "nodes": [], + "tags": [ + "cat.text-alternatives", + "wcag2a", + "wcag111", + "section508", + "section508.22.a" + ] + }, + { + "description": "Ensures ARIA attributes are allowed for an element's role", + "help": "Elements must only use allowed ARIA attributes", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-allowed-attr?application=webdriverjs", + "id": "aria-allowed-attr", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag412" + ] + }, + { + "description": "Ensures role attribute has an appropriate value for the element", + "help": "ARIA role must be appropriate for the element", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-allowed-role?application=webdriverjs", + "id": "aria-allowed-role", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "best-practice" + ] + }, + { + "description": "Ensures unsupported DPUB roles are only used on elements with implicit fallback roles", + "help": "Unsupported DPUB ARIA roles should be used on elements with implicit fallback roles", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-dpub-role-fallback?application=webdriverjs", + "id": "aria-dpub-role-fallback", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag131" + ] + }, + { + "description": "Ensures aria-hidden elements do not contain focusable elements", + "help": "ARIA hidden element must not contain focusable elements", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-hidden-focus?application=webdriverjs", + "id": "aria-hidden-focus", + "impact": null, + "nodes": [], + "tags": [ + "cat.name-role-value", + "wcag2a", + "wcag412" + ] + }, + { + "description": "Ensures elements with ARIA roles have all required ARIA attributes", + "help": "Required ARIA attributes must be provided", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-required-attr?application=webdriverjs", + "id": "aria-required-attr", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag412" + ] + }, + { + "description": "Ensures elements with an ARIA role that require child roles contain them", + "help": "Certain ARIA roles must contain particular children", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-required-children?application=webdriverjs", + "id": "aria-required-children", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag131" + ] + }, + { + "description": "Ensures elements with an ARIA role that require parent roles are contained by them", + "help": "Certain ARIA roles must be contained by particular parents", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-required-parent?application=webdriverjs", + "id": "aria-required-parent", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag131" + ] + }, + { + "description": "Ensures all elements with a role attribute use a valid value", + "help": "ARIA roles used must conform to valid values", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-roles?application=webdriverjs", + "id": "aria-roles", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag412" + ] + }, + { + "description": "Ensures all ARIA attributes have valid values", + "help": "ARIA attributes must conform to valid values", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-valid-attr-value?application=webdriverjs", + "id": "aria-valid-attr-value", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag412" + ] + }, + { + "description": "Ensures attributes that begin with aria- are valid ARIA attributes", + "help": "ARIA attributes must conform to valid names", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/aria-valid-attr?application=webdriverjs", + "id": "aria-valid-attr", + "impact": null, + "nodes": [], + "tags": [ + "cat.aria", + "wcag2a", + "wcag412" + ] + }, + { + "description": "Ensure the autocomplete attribute is correct and suitable for the form field", + "help": "autocomplete attribute must be used correctly", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/autocomplete-valid?application=webdriverjs", + "id": "autocomplete-valid", + "impact": null, + "nodes": [], + "tags": [ + "cat.forms", + "wcag21aa", + "wcag135" + ] + }, + { + "description": "Ensures elements are not used", + "help": " elements are deprecated and must not be used", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/blink?application=webdriverjs", + "id": "blink", + "impact": null, + "nodes": [], + "tags": [ + "cat.time-and-media", + "wcag2a", + "wcag222", + "section508", + "section508.22.j" + ] + }, + { + "description": "Ensures buttons have discernible text", + "help": "Buttons must have discernible text", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/button-name?application=webdriverjs", + "id": "button-name", + "impact": null, + "nodes": [], + "tags": [ + "cat.name-role-value", + "wcag2a", + "wcag412", + "section508", + "section508.22.a" + ] + }, + { + "description": "Ensures related elements have a group and that the group designation is consistent", + "help": "Checkbox inputs with the same name attribute value must be part of a group", + "helpUrl": "https://dequeuniversity.com/rules/axe/3.2/checkboxgroup?application=webdriverjs", + "id": "checkboxgroup", + "impact": null, + "nodes": [], + "tags": [ + "cat.forms", + "best-practice" + ] + }, + { + "description": "Ensures
elements are structured correctly", + "help": "
elements must only directly contain properly-ordered
and
groups,