feat: axe-core 3.3.2 compatibility (#104)

The library was already compatible with axe-core 3.3 results (the output format didn't change since 3.2), however there were no tests of it and no note of it in the README. This PR: * Restructures test files into a common format and uses `it.each` in the integration tests to reference them * Adds test files to pin axe 3.3.2 results' output * Updates yarn.lock (but not package.json) to reference axe-core 3.3.2 (relevant for the raw converter tests that use its global state) * Updates README documentation to better describe version compatibility and what we test against Technically, the only actual user-visible change is in the README file, but I felt this warranted a feat: update anyway since I wanted to make sure we forced an `npm publish` to get the NPM readme content updated.
microsoft · Sep 26, 2019 · d9d2610 · d9d2610
1 parent d14e66e
commit d9d2610
Show file tree

Hide file tree

Showing 22 changed files with 62,247 additions and 31,330 deletions.
diff --git a/README.md b/README.md
@@ -66,8 +66,8 @@ The [microsoft/axe-pipelines-samples](https://github.com/microsoft/axe-pipelines
 
 The version number of this library is **independent** from the version numbers of the axe-core inputs and SARIF outputs it supports.
 
--   axe-sarif-converter version 2.x supports input from version ^3.0.0 of axe-core and outputs SARIF v2.1
--   axe-sarif-converter version 1.x supports input from version ^3.0.0 of axe-core and outputs SARIF v2.0
+-   axe-sarif-converter version 2.x supports input from version ^3.2.0 of axe-core (tested with 3.2.2 and 3.3.2) and outputs SARIF v2.1
+-   axe-sarif-converter version 1.x supports input from version >= 3.2.0 < 3.3.0 of axe-core (tested with 3.2.2) and outputs SARIF v2.0
 
 Note that the SARIF format _does not use semantic versioning_, and there are breaking changes between the v2.0 and v2.1 SARIF formats. If you need compatibility with a SARIF viewer that only supports v2.0, you should use version 1.x of this library.
 

diff --git a/src/__snapshots__/index.test.ts.snap b/src/__snapshots__/index.test.ts.snap
diff --git a/src/axe-raw-sarif-converter.test.ts b/src/axe-raw-sarif-converter.test.ts
@@ -28,7 +28,7 @@ describe('AxeRawSarifConverter', () => {
 
         it('produces the same output as the v2 converter for equivalent raw input', () => {
             const axeJSON: string = fs.readFileSync(
-                './src/test-resources/axe-v3.2.2.reporter-v2.json',
+                './src/test-resources/basic-axe-v3.3.2.reporter-v2.json',
                 'utf8',
             );
             const axeResult: AxeResults = JSON.parse(axeJSON) as AxeResults;
@@ -38,7 +38,7 @@ describe('AxeRawSarifConverter', () => {
             );
 
             const axeRawJSON: string = fs.readFileSync(
-                './src/test-resources/axe-v3.2.2.reporter-raw.json',
+                './src/test-resources/basic-axe-v3.3.2.reporter-raw.json',
                 'utf8',
             );
             const axeRawResult: AxeRawResult[] = JSON.parse(

diff --git a/src/cli.test.ts b/src/cli.test.ts
@@ -152,13 +152,16 @@ describe('axe-sarif-converter CLI', () => {
     const testResultsDir = path.join(__dirname, '..', 'test-results');
     const basicAxeV2File = path.join(
         testResourcesDir,
-        'basic-axe-v3.2.2-reporter-v2.json',
+        'basic-axe-v3.3.2.reporter-v2.json',
     );
     const basicSarifFile = path.join(
         testResourcesDir,
-        'basic-axe-v3.2.2-sarif-v2.1.2.sarif',
+        'basic-axe-v3.3.2.sarif',
+    );
+    const axeCliFile = path.join(
+        testResourcesDir,
+        'w3citylights-axe-v3.3.2.axe-cli-v3.1.1.json',
     );
-    const axeCliFile = path.join(testResourcesDir, 'axe-cli-v3.1.1.json');
 
     const mkdir = promisify(fs.mkdir);
     const writeFile = promisify(fs.writeFile);

diff --git a/src/index.test.ts b/src/index.test.ts
@@ -44,26 +44,25 @@ describe('public convertAxeToSarif API', () => {
         expect(convertAxeToSarif(minimalAxeV2Input)).toMatchSnapshot();
     });
 
-    it('converts minimal axe v2 input with a basic result to the pinned SARIF output', () => {
-        const basicAxeV2Input: AxeResults = readTestResourceJSON(
-            'basic-axe-v3.2.2-reporter-v2.json',
-        );
-        const expectedBasicSarifOutput: SarifLog = readTestResourceJSON(
-            'basic-axe-v3.2.2-sarif-v2.1.2.sarif',
-        );
-
-        expect(convertAxeToSarif(basicAxeV2Input)).toEqual(
-            expectedBasicSarifOutput,
-        );
-    });
-
-    it('converts axe v2 input from a real scan to the pinned SARIF output snapshot', () => {
-        const realisticAxeV2Input: AxeResults = readTestResourceJSON(
-            'axe-v3.2.2.reporter-v2.json',
-        );
-
-        expect(convertAxeToSarif(realisticAxeV2Input)).toMatchSnapshot();
-    });
+    it.each`
+        inputFile                                     | outputFile
+        ${'basic-axe-v3.2.2.reporter-v2.json'}        | ${'basic-axe-v3.2.2.sarif'}
+        ${'w3citylights-axe-v3.2.2.reporter-v2.json'} | ${'w3citylights-axe-v3.2.2.sarif'}
+        ${'basic-axe-v3.3.2.reporter-v1.json'}        | ${'basic-axe-v3.3.2.sarif'}
+        ${'basic-axe-v3.3.2.reporter-v2.json'}        | ${'basic-axe-v3.3.2.sarif'}
+        ${'w3citylights-axe-v3.3.2.reporter-v1.json'} | ${'w3citylights-axe-v3.3.2.sarif'}
+        ${'w3citylights-axe-v3.3.2.reporter-v2.json'} | ${'w3citylights-axe-v3.3.2.sarif'}
+    `(
+        'converts pinned v1/v2 input $inputFile to pinned output $outputFile',
+        ({ inputFile, outputFile }) => {
+            const input: AxeResults = readTestResourceJSON(inputFile);
+            const expectedOutput: SarifLog = readTestResourceJSON(outputFile);
+
+            const actualOutput: SarifLog = convertAxeToSarif(input);
+
+            expect(actualOutput).toEqual(expectedOutput);
+        },
+    );
 });
 
 // This initializes global state that the reporter API assumes is available
@@ -93,36 +92,31 @@ describe('public sarifReporter API', () => {
         sarifReporter(minimalAxeRawInput, emptyAxeRunOptions, callback);
     });
 
-    it('converts minimal axe rawObject input with a basic result to the pinned SARIF output', done => {
-        const basicAxeRawInput: AxeRawResult[] = readTestResourceJSON(
-            'basic-axe-v3.2.2-reporter-raw.json',
-        );
-        const expectedBasicSarifOutput: SarifLog = readTestResourceJSON(
-            'basic-axe-v3.2.2-sarif-v2.1.2.sarif',
-        );
-
-        function callback(convertedSarifResults: SarifLog) {
-            normalizeEnvironmentDerivedSarifProperties(convertedSarifResults);
-
-            expect(convertedSarifResults).toEqual(expectedBasicSarifOutput);
-            done();
-        }
-
-        sarifReporter(basicAxeRawInput, emptyAxeRunOptions, callback);
-    });
-
-    it('converts axe rawObject input from a real scan to the pinned SARIF output snapshot', done => {
-        const realisticAxeRawInput: AxeRawResult[] = readTestResourceJSON(
-            'axe-v3.2.2.reporter-raw.json',
-        );
-
-        function callback(convertedSarifResults: SarifLog) {
-            normalizeEnvironmentDerivedSarifProperties(convertedSarifResults);
-
-            expect(convertedSarifResults).toMatchSnapshot();
-            done();
-        }
-
-        sarifReporter(realisticAxeRawInput, emptyAxeRunOptions, callback);
-    });
+    // Since the integration tests of the raw converter involve global page/axe state,
+    // it isn't very meaningful to test cases that involve old axe versions here.
+    it.each`
+        inputFile                               | outputFile
+        ${'basic-axe-v3.3.2.reporter-raw.json'} | ${'basic-axe-v3.3.2.sarif'}
+    `(
+        'converts pinned raw input $inputFile to pinned output $outputFile',
+        async ({ inputFile, outputFile }) => {
+            return new Promise(resolve => {
+                const input: AxeRawResult[] = readTestResourceJSON(inputFile);
+                const expectedOutput: SarifLog = readTestResourceJSON(
+                    outputFile,
+                );
+
+                function callback(convertedSarifResults: SarifLog) {
+                    normalizeEnvironmentDerivedSarifProperties(
+                        convertedSarifResults,
+                    );
+
+                    expect(convertedSarifResults).toEqual(expectedOutput);
+                    resolve();
+                }
+
+                sarifReporter(input, emptyAxeRunOptions, callback);
+            });
+        },
+    );
 });
diff --git a/src/sarif-viewer.test.ts b/src/sarif-viewer.test.ts
@@ -7,7 +7,7 @@ import { convertAxeToSarif } from './index';
 describe('Azure DevOps sarif viewer extension test', () => {
     it('save sarif file to artifact to be shown by sarif viewer tab', () => {
         const axeJSON: string = fs.readFileSync(
-            './src/test-resources/axe-v3.2.2.reporter-v2.json',
+            './src/test-resources/w3citylights-axe-v3.3.2.reporter-v2.json',
             'utf8',
         );
         const axeResult: AxeResults = JSON.parse(axeJSON) as AxeResults;

diff --git a/...ources/basic-axe-v3.2.2-reporter-raw.json → ...ources/basic-axe-v3.2.2.reporter-raw.json b/...ources/basic-axe-v3.2.2-reporter-raw.json → ...ources/basic-axe-v3.2.2.reporter-raw.json
diff --git a/...sources/basic-axe-v3.2.2-reporter-v2.json → ...sources/basic-axe-v3.2.2.reporter-v2.json b/...sources/basic-axe-v3.2.2-reporter-v2.json → ...sources/basic-axe-v3.2.2.reporter-v2.json
diff --git a/...urces/basic-axe-v3.2.2-sarif-v2.1.2.sarif → src/test-resources/basic-axe-v3.2.2.sarif b/...urces/basic-axe-v3.2.2-sarif-v2.1.2.sarif → src/test-resources/basic-axe-v3.2.2.sarif
diff --git a/src/test-resources/basic-axe-v3.3.2.reporter-raw.json b/src/test-resources/basic-axe-v3.3.2.reporter-raw.json
@@ -0,0 +1,45 @@
+[
+  {
+    "id": "document-title",
+    "result": "failed",
+    "pageLevel": false,
+    "impact": "serious",
+    "tags": [
+      "cat.text-alternatives",
+      "wcag2a",
+      "wcag242"
+    ],
+    "description": "Ensures each HTML document contains a non-empty <title> element",
+    "help": "Documents must have <title> element to aid in navigation",
+    "helpUrl": "https://dequeuniversity.com/rules/axe/3.3/document-title?application=axe-puppeteer",
+    "inapplicable": [],
+    "passes": [],
+    "incomplete": [],
+    "violations": [
+      {
+        "any": [
+          {
+            "id": "doc-has-title",
+            "data": null,
+            "relatedNodes": [],
+            "impact": "serious",
+            "message": "Document does not have a non-empty <title> element"
+          }
+        ],
+        "all": [],
+        "none": [],
+        "node": {
+          "selector": [
+            "html"
+          ],
+          "source": "<html><head></head><body>\n</body></html>",
+          "xpath": [
+            "/html"
+          ]
+        },
+        "impact": "serious",
+        "result": "failed"
+      }
+    ]
+  }
+]
diff --git a/src/test-resources/basic-axe-v3.3.2.reporter-v1.json b/src/test-resources/basic-axe-v3.3.2.reporter-v1.json
@@ -0,0 +1,62 @@
+{
+  "testEngine": {
+    "name": "axe-core",
+    "version": "3.3.2"
+  },
+  "testRunner": {
+    "name": "axe"
+  },
+  "testEnvironment": {
+    "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/78.0.3882.0 Safari/537.36",
+    "windowWidth": 800,
+    "windowHeight": 600,
+    "orientationAngle": 0,
+    "orientationType": "portrait-primary"
+  },
+  "timestamp": "2019-03-22T19:12:06.129Z",
+  "url": "http://localhost/",
+  "toolOptions": {
+    "reporter": "v1"
+  },
+  "violations": [
+    {
+      "id": "document-title",
+      "impact": "serious",
+      "tags": [
+        "cat.text-alternatives",
+        "wcag2a",
+        "wcag242"
+      ],
+      "description": "Ensures each HTML document contains a non-empty <title> element",
+      "help": "Documents must have <title> element to aid in navigation",
+      "helpUrl": "https://dequeuniversity.com/rules/axe/3.3/document-title?application=axe-puppeteer",
+      "nodes": [
+        {
+          "any": [
+            {
+              "id": "doc-has-title",
+              "data": null,
+              "relatedNodes": [],
+              "impact": "serious",
+              "message": "Document does not have a non-empty <title> element"
+            }
+          ],
+          "all": [],
+          "none": [],
+          "impact": "serious",
+          "html": "<html><head></head><body>\n</body></html>",
+          "target": [
+            "html"
+          ],
+          "xpath": [
+            "/html"
+          ],
+          "failureSummary": "Fix any of the following:\n  Document does not have a non-empty <title> element"
+        }
+      ]
+    }
+  ],
+  "passes": [],
+  "incomplete": [],
+  "inapplicable": []
+}
diff --git a/src/test-resources/basic-axe-v3.3.2.reporter-v2.json b/src/test-resources/basic-axe-v3.3.2.reporter-v2.json
@@ -0,0 +1,61 @@
+{
+  "testEngine": {
+    "name": "axe-core",
+    "version": "3.3.2"
+  },
+  "testRunner": {
+    "name": "axe"
+  },
+  "testEnvironment": {
+    "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/78.0.3882.0 Safari/537.36",
+    "windowWidth": 800,
+    "windowHeight": 600,
+    "orientationAngle": 0,
+    "orientationType": "portrait-primary"
+  },
+  "timestamp": "2019-03-22T19:12:06.129Z",
+  "url": "http://localhost/",
+  "toolOptions": {
+    "reporter": "v2"
+  },
+  "violations": [
+    {
+      "id": "document-title",
+      "impact": "serious",
+      "tags": [
+        "cat.text-alternatives",
+        "wcag2a",
+        "wcag242"
+      ],
+      "description": "Ensures each HTML document contains a non-empty <title> element",
+      "help": "Documents must have <title> element to aid in navigation",
+      "helpUrl": "https://dequeuniversity.com/rules/axe/3.3/document-title?application=axe-puppeteer",
+      "nodes": [
+        {
+          "any": [
+            {
+              "id": "doc-has-title",
+              "data": null,
+              "relatedNodes": [],
+              "impact": "serious",
+              "message": "Document does not have a non-empty <title> element"
+            }
+          ],
+          "all": [],
+          "none": [],
+          "impact": "serious",
+          "html": "<html><head></head><body>\n</body></html>",
+          "target": [
+            "html"
+          ],
+          "xpath": [
+            "/html"
+          ]
+        }
+      ]
+    }
+  ],
+  "passes": [],
+  "incomplete": [],
+  "inapplicable": []
+}