diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index f258aee9..6c8e4bdf 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -111,6 +111,7 @@ blocks:
     - name: "@appsignal/nodejs - nodejs"
       commands:
       - mono test --package=@appsignal/nodejs
+      - mono run --package @appsignal/nodejs -- npm run test:failure
     - name: "@appsignal/nodejs - nodejs - diagnose"
       commands:
       - git submodule init
@@ -206,6 +207,7 @@ blocks:
     - name: "@appsignal/nodejs - nodejs"
       commands:
       - mono test --package=@appsignal/nodejs
+      - mono run --package @appsignal/nodejs -- npm run test:failure
     - name: "@appsignal/nodejs - nodejs - diagnose"
       commands:
       - git submodule init
@@ -299,6 +301,7 @@ blocks:
     - name: "@appsignal/nodejs - nodejs"
       commands:
       - mono test --package=@appsignal/nodejs
+      - mono run --package @appsignal/nodejs -- npm run test:failure
     - name: "@appsignal/nodejs - nodejs - diagnose"
       commands:
       - git submodule init
@@ -392,6 +395,7 @@ blocks:
     - name: "@appsignal/nodejs - nodejs"
       commands:
       - mono test --package=@appsignal/nodejs
+      - mono run --package @appsignal/nodejs -- npm run test:failure
     - name: "@appsignal/nodejs - nodejs - diagnose"
       commands:
       - git submodule init
diff --git a/build_matrix.yml b/build_matrix.yml
index ec066de7..af8e5a9f 100644
--- a/build_matrix.yml
+++ b/build_matrix.yml
@@ -98,12 +98,8 @@ matrix:
           - git submodule init
           - git submodule update
           - LANGUAGE=nodejs test/integration/diagnose/bin/test
-    - package: "@appsignal/nodejs-ext"
-      path: "packages/nodejs-ext"
-      variations:
-        - name: "nodejs-ext"
       extra_commands:
-        - mono run --package @appsignal/nodejs-ext -- npm run test:failure
+        - mono run --package @appsignal/nodejs -- npm run test:failure
     # Library integrations
     - package: "@appsignal/apollo-server"
       path: "packages/apollo-server"
diff --git a/packages/nodejs/jest.config.js b/packages/nodejs/jest.config.js
index 826ae11d..e132ed18 100644
--- a/packages/nodejs/jest.config.js
+++ b/packages/nodejs/jest.config.js
@@ -1,7 +1,7 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
-  roots: ["<rootDir>/src"],
+  roots: ["<rootDir>/src", "<rootDir>/scripts"],
   transform: {
     "^.+\\.tsx?$": "ts-jest"
   }
diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json
index 031e244c..aedc06ce 100644
--- a/packages/nodejs/package.json
+++ b/packages/nodejs/package.json
@@ -27,13 +27,14 @@
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "build:watch": "tsc -p tsconfig.json -w --preserveWatchOutput",
-    "clean": "rimraf dist coverage && rimraf ext/appsignal-agent ext/libappsignal.a ext/appsignal.* ext/*.tar.gz",
+    "clean": "rimraf dist coverage build && rimraf ext/appsignal-agent ext/libappsignal.a ext/appsignal.* ext/*.tar.gz ext/*.report",
     "install": "node scripts/extension/extension.js",
     "link:npm": "npm link",
     "link:yarn": "yarn link",
     "test": "jest --filter=./test/filter.js",
-    "test:ext:failure": "_TEST_APPSIGNAL_EXTENSION_FAILURE=true _APPSIGNAL_EXTENSION_INSTALL=true npm run install; _TEST_APPSIGNAL_EXTENSION_FAILURE=true jest --filter=./test/filter.js",
-    "test:watch": "jest --filter=./test/filter.js --watch"
+    "test:watch": "jest --filter=./test/filter.js --watch",
+    "pretest:failure": "npm run clean",
+    "test:failure": "_TEST_APPSIGNAL_EXTENSION_FAILURE=true _APPSIGNAL_EXTENSION_INSTALL=true npm run install; _TEST_APPSIGNAL_EXTENSION_FAILURE=true jest --filter=./test/filter.js"
   },
   "os": [
     "linux",