From b7763e065f7e166f363f3e69beab0489403b23f7 Mon Sep 17 00:00:00 2001 From: Eric Cornelissen Date: Sat, 14 Oct 2023 11:42:02 +0200 Subject: [PATCH] Introduce "breakage testing" Add a new type of testing including a new test suite that's aimed solely at preventing the introduction of breaking changes. To this end the suite runs both the current version of the library and the latest released version and compares their behaviour. --- .c8/breakage.json | 8 +++ .github/workflows/checks.yml | 31 +++++++++ .licensee.json | 3 +- package-lock.json | 14 ++++ package.json | 7 +- test/breakage/_.js | 8 +++ test/breakage/index.test.js | 122 +++++++++++++++++++++++++++++++++++ 7 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 .c8/breakage.json create mode 100644 test/breakage/_.js create mode 100644 test/breakage/index.test.js diff --git a/.c8/breakage.json b/.c8/breakage.json new file mode 100644 index 000000000..1ee706561 --- /dev/null +++ b/.c8/breakage.json @@ -0,0 +1,8 @@ +{ + "all": true, + "include": ["src/", "index.js"], + "exclude": [], + "check-coverage": false, + "reports-dir": "_reports/coverage/breakage", + "reporter": ["lcov", "text"] +} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a1672dec5..6d298df5c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -218,6 +218,37 @@ jobs: if: ${{ failure() || success() }} with: sarif_file: njsscan-results.sarif + test-breakage: + name: Breakage + runs-on: ubuntu-22.04 + needs: + - test-integration + steps: + - name: Harden runner + uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + artifactcache.actions.githubusercontent.com:443 + github.com:443 + gitlab.com:443 + nodejs.org:443 + objects.githubusercontent.com:443 + registry.npmjs.org:443 + - name: Checkout repository + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - name: Install Node.js + uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 + with: + cache: npm + node-version-file: .nvmrc + - name: Install dependencies + run: npm clean-install + - name: Run breakage tests + run: npm run coverage:breakage test-compatibility: name: Compatibility runs-on: ubuntu-22.04 diff --git a/.licensee.json b/.licensee.json index 54f124e0a..e2709888b 100644 --- a/.licensee.json +++ b/.licensee.json @@ -19,7 +19,8 @@ "@andrewbranch/untar.js": "1.0.2", "deep-freeze": "0.0.1", "filter-iterator": "0.0.1", - "identity-function": "1.0.0" + "identity-function": "1.0.0", + "shescape": "*" }, "corrections": true } diff --git a/package-lock.json b/package-lock.json index b8c28b45d..0b62bd68e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "prettier": "3.0.3", "publint": "0.2.3", "rollup": "4.0.2", + "shescape-previous": "npm:shescape@2.0.0", "sinon": "16.1.0" }, "engines": { @@ -11183,6 +11184,19 @@ "integrity": "sha512-lT297f1WLAdq0A4O+AknIFRP6kkiI3s8C913eJ0XqBxJbZPGWUNkRQk2u8zk4bEAjUJ5i+fSLwB6z1HzeT+DEg==", "dev": true }, + "node_modules/shescape-previous": { + "name": "shescape", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shescape/-/shescape-2.0.0.tgz", + "integrity": "sha512-5NO165joyGNZHTEm5BLiZ+lbq9UEZnhRD3Z4FiQ3nYOUN/bXQfgXtyNNEDjWtI7vLDyrfpvfBGmT5mPLSURa5A==", + "dev": true, + "dependencies": { + "which": "^3.0.0" + }, + "engines": { + "node": "^14.18.0 || ^16.13.0 || ^18 || ^19 || ^20" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", diff --git a/package.json b/package.json index 3b345886e..5eec7102d 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "prettier": "3.0.3", "publint": "0.2.3", "rollup": "4.0.2", + "shescape-previous": "npm:shescape@2.0.0", "sinon": "16.1.0" }, "scripts": { @@ -98,7 +99,8 @@ "audit:runtime": "better-npm-audit audit --production", "benchmark": "node bench/bench.js", "clean": "node script/clean.js", - "coverage": "npm run coverage:unit && npm run coverage:integration && npm run coverage:e2e && npm run coverage:compat", + "coverage": "npm run coverage:unit && npm run coverage:integration && npm run coverage:e2e && npm run coverage:compat && npm run coverage:breakage", + "coverage:breakage": "c8 --config .c8/breakage.json npm run test:breakage", "coverage:compat": "c8 --config .c8/compat.json npm run test:compat", "coverage:e2e": "node script/run-platform-coverage.js e2e", "coverage:e2e:unix": "c8 --config .c8/e2e-unix.json npm run test:e2e", @@ -121,7 +123,8 @@ "mutation": "npm run mutation:unit && npm run mutation:integration", "mutation:integration": "stryker run stryker.integration.config.js", "mutation:unit": "stryker run stryker.unit.config.js", - "test": "npm run test:unit && npm run test:integration && npm run test:e2e && npm run test:compat", + "test": "npm run test:unit && npm run test:integration && npm run test:e2e && npm run test:compat && npm run test:breakage", + "test:breakage": "ava test/breakage/**/*.test.js", "test:compat": "ava test/compat/**/*.test.js", "test:compat-all": "nve 14.18.0,16.13.0,18.0.0,19.0.0,20.0.0 npm run test:compat --ignore-scripts", "test:e2e": "ava test/e2e/**/*.test.js --timeout 1m", diff --git a/test/breakage/_.js b/test/breakage/_.js new file mode 100644 index 000000000..9c14ed685 --- /dev/null +++ b/test/breakage/_.js @@ -0,0 +1,8 @@ +/** + * @overview Provides testing utilities. + * @license MIT + */ + +import * as arbitrary from "../_arbitraries.js"; + +export { arbitrary }; diff --git a/test/breakage/index.test.js b/test/breakage/index.test.js new file mode 100644 index 000000000..a9e4a1325 --- /dev/null +++ b/test/breakage/index.test.js @@ -0,0 +1,122 @@ +/** + * @overview Contains breakage tests for the `Shescape` class. + * @license MIT + */ + +import { testProp } from "@fast-check/ava"; +import * as fc from "fast-check"; + +import { arbitrary } from "./_.js"; + +import { Shescape } from "shescape"; +import { Shescape as Previouscape } from "shescape-previous"; + +testProp( + "Shescape#escape", + [arbitrary.shescapeOptions(), fc.anything()], + (t, options, arg) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.escape(arg); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.escape(arg); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Shescape#escapeAll", + [ + arbitrary.shescapeOptions(), + fc.oneof(fc.anything(), fc.array(fc.anything())), + ], + (t, options, args) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.escapeAll(args); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.escapeAll(args); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Shescape#quote", + [arbitrary.shescapeOptions(), fc.anything()], + (t, options, arg) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.quote(arg); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.quote(arg); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Shescape#quoteAll", + [ + arbitrary.shescapeOptions(), + fc.oneof(fc.anything(), fc.array(fc.anything())), + ], + (t, options, args) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.quoteAll(args); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.quoteAll(args); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +);