From b7763e065f7e166f363f3e69beab0489403b23f7 Mon Sep 17 00:00:00 2001 From: Eric Cornelissen Date: Sat, 14 Oct 2023 11:42:02 +0200 Subject: [PATCH 1/3] Introduce "breakage testing" Add a new type of testing including a new test suite that's aimed solely at preventing the introduction of breaking changes. To this end the suite runs both the current version of the library and the latest released version and compares their behaviour. --- .c8/breakage.json | 8 +++ .github/workflows/checks.yml | 31 +++++++++ .licensee.json | 3 +- package-lock.json | 14 ++++ package.json | 7 +- test/breakage/_.js | 8 +++ test/breakage/index.test.js | 122 +++++++++++++++++++++++++++++++++++ 7 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 .c8/breakage.json create mode 100644 test/breakage/_.js create mode 100644 test/breakage/index.test.js diff --git a/.c8/breakage.json b/.c8/breakage.json new file mode 100644 index 000000000..1ee706561 --- /dev/null +++ b/.c8/breakage.json @@ -0,0 +1,8 @@ +{ + "all": true, + "include": ["src/", "index.js"], + "exclude": [], + "check-coverage": false, + "reports-dir": "_reports/coverage/breakage", + "reporter": ["lcov", "text"] +} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a1672dec5..6d298df5c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -218,6 +218,37 @@ jobs: if: ${{ failure() || success() }} with: sarif_file: njsscan-results.sarif + test-breakage: + name: Breakage + runs-on: ubuntu-22.04 + needs: + - test-integration + steps: + - name: Harden runner + uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + artifactcache.actions.githubusercontent.com:443 + github.com:443 + gitlab.com:443 + nodejs.org:443 + objects.githubusercontent.com:443 + registry.npmjs.org:443 + - name: Checkout repository + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - name: Install Node.js + uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 + with: + cache: npm + node-version-file: .nvmrc + - name: Install dependencies + run: npm clean-install + - name: Run breakage tests + run: npm run coverage:breakage test-compatibility: name: Compatibility runs-on: ubuntu-22.04 diff --git a/.licensee.json b/.licensee.json index 54f124e0a..e2709888b 100644 --- a/.licensee.json +++ b/.licensee.json @@ -19,7 +19,8 @@ "@andrewbranch/untar.js": "1.0.2", "deep-freeze": "0.0.1", "filter-iterator": "0.0.1", - "identity-function": "1.0.0" + "identity-function": "1.0.0", + "shescape": "*" }, "corrections": true } diff --git a/package-lock.json b/package-lock.json index b8c28b45d..0b62bd68e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "prettier": "3.0.3", "publint": "0.2.3", "rollup": "4.0.2", + "shescape-previous": "npm:shescape@2.0.0", "sinon": "16.1.0" }, "engines": { @@ -11183,6 +11184,19 @@ "integrity": "sha512-lT297f1WLAdq0A4O+AknIFRP6kkiI3s8C913eJ0XqBxJbZPGWUNkRQk2u8zk4bEAjUJ5i+fSLwB6z1HzeT+DEg==", "dev": true }, + "node_modules/shescape-previous": { + "name": "shescape", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shescape/-/shescape-2.0.0.tgz", + "integrity": "sha512-5NO165joyGNZHTEm5BLiZ+lbq9UEZnhRD3Z4FiQ3nYOUN/bXQfgXtyNNEDjWtI7vLDyrfpvfBGmT5mPLSURa5A==", + "dev": true, + "dependencies": { + "which": "^3.0.0" + }, + "engines": { + "node": "^14.18.0 || ^16.13.0 || ^18 || ^19 || ^20" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", diff --git a/package.json b/package.json index 3b345886e..5eec7102d 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "prettier": "3.0.3", "publint": "0.2.3", "rollup": "4.0.2", + "shescape-previous": "npm:shescape@2.0.0", "sinon": "16.1.0" }, "scripts": { @@ -98,7 +99,8 @@ "audit:runtime": "better-npm-audit audit --production", "benchmark": "node bench/bench.js", "clean": "node script/clean.js", - "coverage": "npm run coverage:unit && npm run coverage:integration && npm run coverage:e2e && npm run coverage:compat", + "coverage": "npm run coverage:unit && npm run coverage:integration && npm run coverage:e2e && npm run coverage:compat && npm run coverage:breakage", + "coverage:breakage": "c8 --config .c8/breakage.json npm run test:breakage", "coverage:compat": "c8 --config .c8/compat.json npm run test:compat", "coverage:e2e": "node script/run-platform-coverage.js e2e", "coverage:e2e:unix": "c8 --config .c8/e2e-unix.json npm run test:e2e", @@ -121,7 +123,8 @@ "mutation": "npm run mutation:unit && npm run mutation:integration", "mutation:integration": "stryker run stryker.integration.config.js", "mutation:unit": "stryker run stryker.unit.config.js", - "test": "npm run test:unit && npm run test:integration && npm run test:e2e && npm run test:compat", + "test": "npm run test:unit && npm run test:integration && npm run test:e2e && npm run test:compat && npm run test:breakage", + "test:breakage": "ava test/breakage/**/*.test.js", "test:compat": "ava test/compat/**/*.test.js", "test:compat-all": "nve 14.18.0,16.13.0,18.0.0,19.0.0,20.0.0 npm run test:compat --ignore-scripts", "test:e2e": "ava test/e2e/**/*.test.js --timeout 1m", diff --git a/test/breakage/_.js b/test/breakage/_.js new file mode 100644 index 000000000..9c14ed685 --- /dev/null +++ b/test/breakage/_.js @@ -0,0 +1,8 @@ +/** + * @overview Provides testing utilities. + * @license MIT + */ + +import * as arbitrary from "../_arbitraries.js"; + +export { arbitrary }; diff --git a/test/breakage/index.test.js b/test/breakage/index.test.js new file mode 100644 index 000000000..a9e4a1325 --- /dev/null +++ b/test/breakage/index.test.js @@ -0,0 +1,122 @@ +/** + * @overview Contains breakage tests for the `Shescape` class. + * @license MIT + */ + +import { testProp } from "@fast-check/ava"; +import * as fc from "fast-check"; + +import { arbitrary } from "./_.js"; + +import { Shescape } from "shescape"; +import { Shescape as Previouscape } from "shescape-previous"; + +testProp( + "Shescape#escape", + [arbitrary.shescapeOptions(), fc.anything()], + (t, options, arg) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.escape(arg); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.escape(arg); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Shescape#escapeAll", + [ + arbitrary.shescapeOptions(), + fc.oneof(fc.anything(), fc.array(fc.anything())), + ], + (t, options, args) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.escapeAll(args); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.escapeAll(args); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Shescape#quote", + [arbitrary.shescapeOptions(), fc.anything()], + (t, options, arg) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.quote(arg); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.quote(arg); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Shescape#quoteAll", + [ + arbitrary.shescapeOptions(), + fc.oneof(fc.anything(), fc.array(fc.anything())), + ], + (t, options, args) => { + let result, previousResult, errored, previousErrored; + let shescape, previouscape; + + try { + shescape = new Shescape(options); + result = shescape.quoteAll(args); + } catch (_) { + errored = true; + } + + try { + previouscape = new Previouscape(options); + previousResult = previouscape.quoteAll(args); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); From 73961bffc16b2941d5c2674faa12b08d4a9d94ff Mon Sep 17 00:00:00 2001 From: Eric Cornelissen Date: Sat, 21 Oct 2023 17:42:18 +0200 Subject: [PATCH 2/3] Document breakage testing in the contributing guidelines --- CONTRIBUTING.md | 13 +++++++++++++ package.json | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4fd64032f..0ad7924c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -353,6 +353,19 @@ compatibility tests on all applicable Node.js versions. In the project's continuous integration the compatibility tests are run for all supported Node.js versions as well. +#### Breakage Testing + +The breakage tests aim to ensure that the API of the library isn't broken from +release to release. All breakage tests go into the `test/breakage/` folder. You +can run the breakage test using the command `npm run test:breakage`. + +Breakage test compare both the API itself as well as the behavior of every +function of the API. This is achieved by depending on the latest version of the +library and using it in a differential test for each function in the API. + +Unless the API is extended or a breaking API change is necessary this suite does +not need to be updated. + ### Writing Tests Tests can be written in different ways and using different strategies. This diff --git a/package.json b/package.json index 51cc1ebf3..bb6f7c8a3 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,8 @@ "audit:runtime": "better-npm-audit audit --production", "benchmark": "node bench/bench.js", "clean": "node script/clean.js", - "coverage": "npm run coverage:unit && npm run coverage:integration && npm run coverage:e2e && npm run coverage:compat", + "coverage": "npm run coverage:unit && npm run coverage:integration && npm run coverage:e2e && npm run coverage:compat && npm run coverage:breakage", + "coverage:breakage": "c8 --config .c8/breakage.json npm run test:breakage", "coverage:compat": "c8 --config .c8/compat.json npm run test:compat", "coverage:e2e": "node script/run-platform-coverage.js e2e", "coverage:e2e:unix": "c8 --config .c8/e2e-unix.json npm run test:e2e", @@ -122,7 +123,8 @@ "mutation": "npm run mutation:unit && npm run mutation:integration", "mutation:integration": "stryker run stryker.integration.config.js", "mutation:unit": "stryker run stryker.unit.config.js", - "test": "npm run test:unit && npm run test:integration && npm run test:e2e && npm run test:compat", + "test": "npm run test:unit && npm run test:integration && npm run test:e2e && npm run test:compat && npm run test:breakage", + "test:breakage": "ava test/breakage/**/*.test.js", "test:compat": "ava test/compat/**/*.test.js", "test:compat-all": "nve 14.18.0,16.13.0,18.0.0,19.0.0,20.0.0 npm run test:compat --ignore-scripts", "test:e2e": "ava test/e2e/**/*.test.js --timeout 1m", From 58c45f97ff53cf4fd3b1e71f7aa72e95c15a1736 Mon Sep 17 00:00:00 2001 From: Eric Cornelissen Date: Sat, 21 Oct 2023 18:01:51 +0200 Subject: [PATCH 3/3] Create breakage tests for the testing modules Expand the breakage test suite with a test file for the shescape/testing module. This suite is a bit interesting because it contains some skipped tests. These tests are skipped because of bugfixes and an API extension which both occurred in [1]. The breakage coverage configuration is also updated to include the testing module in the coverage report. -- 1. 4f03fd897eef0bcf39ffda6d5f4d0840f5f15c4d --- .c8/breakage.json | 2 +- test/breakage/testing.test.js | 151 ++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 test/breakage/testing.test.js diff --git a/.c8/breakage.json b/.c8/breakage.json index 1ee706561..80bfe7237 100644 --- a/.c8/breakage.json +++ b/.c8/breakage.json @@ -1,6 +1,6 @@ { "all": true, - "include": ["src/", "index.js"], + "include": ["src/", "index.js", "testing.js"], "exclude": [], "check-coverage": false, "reports-dir": "_reports/coverage/breakage", diff --git a/test/breakage/testing.test.js b/test/breakage/testing.test.js new file mode 100644 index 000000000..2e76c9eb3 --- /dev/null +++ b/test/breakage/testing.test.js @@ -0,0 +1,151 @@ +/** + * @overview Contains breakage tests for the shescape testing module. + * @license MIT + */ + +import { testProp } from "@fast-check/ava"; +import * as fc from "fast-check"; + +import { arbitrary } from "./_.js"; + +import { Shescape as Stubscape, Throwscape } from "shescape/testing"; +import { Shescape as Previoustub } from "shescape-previous/testing"; + +testProp( + "Stubscape#escape", + [arbitrary.shescapeOptions(), fc.anything()], + (t, options, arg) => { + let result, previousResult, errored, previousErrored; + let stubscape, previoustub; + + try { + stubscape = new Stubscape(options); + result = stubscape.escape(arg); + } catch (_) { + errored = true; + } + + try { + previoustub = new Previoustub(options); + previousResult = previoustub.escape(arg); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +testProp( + "Stubscape#escapeAll", + [ + arbitrary.shescapeOptions(), + fc.oneof(fc.anything(), fc.array(fc.anything())), + ], + (t, options, args) => { + let result, previousResult, errored, previousErrored; + let stubscape, previoustub; + + try { + stubscape = new Stubscape(options); + result = stubscape.escapeAll(args); + } catch (_) { + errored = true; + } + + try { + previoustub = new Previoustub(options); + previousResult = previoustub.escapeAll(args); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +// TODO: unskip upon release 2.0.1/2.1.0. It's currently skipped because the +// implementation was incorrect in 2.0.0 and has been fixed since (in 4f03fd8). +testProp.skip( + "Stubscape#quote", + [arbitrary.shescapeOptions(), fc.anything()], + (t, options, arg) => { + let result, previousResult, errored, previousErrored; + let stubscape, previoustub; + + try { + stubscape = new Stubscape(options); + result = stubscape.quote(arg); + } catch (_) { + errored = true; + } + + try { + previoustub = new Previoustub(options); + previousResult = previoustub.quote(arg); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +// TODO: unskip upon release 2.0.1/2.1.0. It's currently skipped because the +// implementation was incorrect in 2.0.0 and has been fixed since (in 4f03fd8). +testProp.skip( + "Stubscape#quoteAll", + [ + arbitrary.shescapeOptions(), + fc.oneof(fc.anything(), fc.array(fc.anything())), + ], + (t, options, args) => { + let result, previousResult, errored, previousErrored; + let stubscape, previoustub; + + try { + stubscape = new Stubscape(options); + result = stubscape.quoteAll(args); + } catch (_) { + errored = true; + } + + try { + previoustub = new Previoustub(options); + previousResult = previoustub.quoteAll(args); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + t.is(typeof result, typeof previousResult); + }, +); + +// TODO: unskip upon release 2.0.1/2.1.0. It's currently skipped because the +// `Throwscape` class was not yet release in 2.0.0 (added in 4f03fd8). +testProp.skip( + "Throwscape#constructor", + [arbitrary.shescapeOptions()], + (t, options) => { + let errored, previousErrored; + let throwscape, previousthrow; + + try { + throwscape = new Throwscape(options); + } catch (_) { + errored = true; + } + + try { + previousthrow = new Previousthrow(options); + } catch (_) { + previousErrored = true; + } + + t.is(errored, previousErrored); + }, +);