diff --git a/Dockerfile b/Dockerfile index b30fddde..2ee4c0e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,4 +33,4 @@ COPY --from=production-deps /app/node_modules node_modules COPY --from=build /app/dist dist EXPOSE 5000 -CMD ["node", "dist/server.js"] \ No newline at end of file +CMD ["node", "dist/start/server.js"] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f1bcefdf..392f8ea9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ "@tus/file-store": "1.3.1", "@tus/s3-store": "1.4.1", "@tus/server": "1.4.1", - "agentkeepalive": "^4.2.1", + "agentkeepalive": "^4.5.0", "ajv": "^8.12.0", "async-retry": "^1.3.3", "axios": "^1.6.8", @@ -53,7 +53,7 @@ "md5-file": "^5.0.0", "multistream": "^4.1.0", "object-sizeof": "^2.6.4", - "pg": "^8.11.3", + "pg": "^8.12.0", "pg-boss": "^9.0.3", "pg-listen": "^1.7.0", "pino": "^8.15.4", @@ -97,7 +97,7 @@ "stream-buffers": "^3.0.2", "ts-jest": "^29.0.3", "ts-node-dev": "^1.1.8", - "tsx": "^3.13.0", + "tsx": "^4.16.0", "tus-js-client": "^3.1.0", "typescript": "^4.5.5" }, @@ -2611,9 +2611,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.10.8", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.8.tgz", - "integrity": "sha512-vYVqYzHicDqyKB+NQhAc54I1QWCBLCrYG6unqOIcBTHx+7x8C9lcoLj3KVJXs2VB4lUbpWY+Kk9NipcbXYWmvg==", + "version": "1.10.10", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.10.tgz", + "integrity": "sha512-HPa/K5NX6ahMoeBv15njAc/sfF4/jmiXLar9UlC2UfHFKZzsCVLc3wbe7+7qua7w9VPh2/L6EBxyAV7/E8Wftg==", "dependencies": { "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" @@ -5754,12 +5754,10 @@ } }, "node_modules/agentkeepalive": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", - "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", "dependencies": { - "debug": "^4.1.0", - "depd": "^1.1.2", "humanize-ms": "^1.2.1" }, "engines": { @@ -6123,12 +6121,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -6203,14 +6201,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "node_modules/buffer-writer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", - "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", - "engines": { - "node": ">=4" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -6612,14 +6602,6 @@ "node": ">=0.10" } }, - "node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -7392,9 +7374,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -7637,9 +7619,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.7.2", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.2.tgz", - "integrity": "sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==", + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz", + "integrity": "sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==", "dev": true, "dependencies": { "resolve-pkg-maps": "^1.0.0" @@ -9554,11 +9536,6 @@ "node": ">=6" } }, - "node_modules/packet-reader": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", - "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -9631,15 +9608,13 @@ } }, "node_modules/pg": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", - "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", - "dependencies": { - "buffer-writer": "2.0.0", - "packet-reader": "1.0.0", - "pg-connection-string": "^2.6.2", - "pg-pool": "^3.6.1", - "pg-protocol": "^1.6.0", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", + "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", + "dependencies": { + "pg-connection-string": "^2.6.4", + "pg-pool": "^3.6.2", + "pg-protocol": "^1.6.1", "pg-types": "^2.1.0", "pgpass": "1.x" }, @@ -9724,17 +9699,17 @@ } }, "node_modules/pg-pool": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", - "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", + "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", + "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, "node_modules/pg-types": { "version": "2.2.0", @@ -9751,6 +9726,11 @@ "node": ">=4" } }, + "node_modules/pg/node_modules/pg-connection-string": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", + "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" + }, "node_modules/pgpass": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", @@ -11270,409 +11250,22 @@ "dev": true }, "node_modules/tsx": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-3.13.0.tgz", - "integrity": "sha512-rjmRpTu3as/5fjNq/kOkOtihgLxuIz6pbKdj9xwP4J5jOLkBxw/rjN5ANw+KyrrOXV5uB7HC8+SrrSJxT65y+A==", + "version": "4.16.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.16.0.tgz", + "integrity": "sha512-MPgN+CuY+4iKxGoJNPv+1pyo5YWZAQ5XfsyobUG+zoKG7IkvCPLZDEyoIb8yLS2FcWci1nlxAqmvPlFWD5AFiQ==", "dev": true, "dependencies": { - "esbuild": "~0.18.20", - "get-tsconfig": "^4.7.2", - "source-map-support": "^0.5.21" + "esbuild": "~0.21.5", + "get-tsconfig": "^4.7.5" }, "bin": { "tsx": "dist/cli.mjs" }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/tsx/node_modules/@esbuild/android-arm": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", - "integrity": "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/android-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz", - "integrity": "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/android-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.20.tgz", - "integrity": "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz", - "integrity": "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/darwin-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz", - "integrity": "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz", - "integrity": "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz", - "integrity": "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-arm": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz", - "integrity": "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz", - "integrity": "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-ia32": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz", - "integrity": "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-loong64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz", - "integrity": "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==", - "cpu": [ - "loong64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz", - "integrity": "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==", - "cpu": [ - "mips64el" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz", - "integrity": "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz", - "integrity": "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==", - "cpu": [ - "riscv64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-s390x": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz", - "integrity": "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz", - "integrity": "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz", - "integrity": "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz", - "integrity": "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/sunos-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz", - "integrity": "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/win32-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz", - "integrity": "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/win32-ia32": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz", - "integrity": "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/@esbuild/win32-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz", - "integrity": "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/tsx/node_modules/esbuild": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz", - "integrity": "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==", - "dev": true, - "hasInstallScript": true, - "bin": { - "esbuild": "bin/esbuild" - }, "engines": { - "node": ">=12" + "node": ">=18.0.0" }, "optionalDependencies": { - "@esbuild/android-arm": "0.18.20", - "@esbuild/android-arm64": "0.18.20", - "@esbuild/android-x64": "0.18.20", - "@esbuild/darwin-arm64": "0.18.20", - "@esbuild/darwin-x64": "0.18.20", - "@esbuild/freebsd-arm64": "0.18.20", - "@esbuild/freebsd-x64": "0.18.20", - "@esbuild/linux-arm": "0.18.20", - "@esbuild/linux-arm64": "0.18.20", - "@esbuild/linux-ia32": "0.18.20", - "@esbuild/linux-loong64": "0.18.20", - "@esbuild/linux-mips64el": "0.18.20", - "@esbuild/linux-ppc64": "0.18.20", - "@esbuild/linux-riscv64": "0.18.20", - "@esbuild/linux-s390x": "0.18.20", - "@esbuild/linux-x64": "0.18.20", - "@esbuild/netbsd-x64": "0.18.20", - "@esbuild/openbsd-x64": "0.18.20", - "@esbuild/sunos-x64": "0.18.20", - "@esbuild/win32-arm64": "0.18.20", - "@esbuild/win32-ia32": "0.18.20", - "@esbuild/win32-x64": "0.18.20" + "fsevents": "~2.3.3" } }, "node_modules/tus-js-client": { @@ -14008,9 +13601,9 @@ } }, "@grpc/grpc-js": { - "version": "1.10.8", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.8.tgz", - "integrity": "sha512-vYVqYzHicDqyKB+NQhAc54I1QWCBLCrYG6unqOIcBTHx+7x8C9lcoLj3KVJXs2VB4lUbpWY+Kk9NipcbXYWmvg==", + "version": "1.10.10", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.10.tgz", + "integrity": "sha512-HPa/K5NX6ahMoeBv15njAc/sfF4/jmiXLar9UlC2UfHFKZzsCVLc3wbe7+7qua7w9VPh2/L6EBxyAV7/E8Wftg==", "requires": { "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" @@ -16382,12 +15975,10 @@ } }, "agentkeepalive": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", - "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", "requires": { - "debug": "^4.1.0", - "depd": "^1.1.2", "humanize-ms": "^1.2.1" } }, @@ -16658,12 +16249,12 @@ } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "browserslist": { @@ -16716,11 +16307,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "buffer-writer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", - "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" - }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -17015,11 +16601,6 @@ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" - }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -17627,9 +17208,9 @@ } }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "requires": { "to-regex-range": "^5.0.1" @@ -17791,9 +17372,9 @@ "dev": true }, "get-tsconfig": { - "version": "4.7.2", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.2.tgz", - "integrity": "sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==", + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz", + "integrity": "sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==", "dev": true, "requires": { "resolve-pkg-maps": "^1.0.0" @@ -19233,11 +18814,6 @@ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, - "packet-reader": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", - "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" - }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -19289,18 +18865,23 @@ "dev": true }, "pg": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", - "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", + "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", "requires": { - "buffer-writer": "2.0.0", - "packet-reader": "1.0.0", "pg-cloudflare": "^1.1.1", - "pg-connection-string": "^2.6.2", - "pg-pool": "^3.6.1", - "pg-protocol": "^1.6.0", + "pg-connection-string": "^2.6.4", + "pg-pool": "^3.6.2", + "pg-protocol": "^1.6.1", "pg-types": "^2.1.0", "pgpass": "1.x" + }, + "dependencies": { + "pg-connection-string": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", + "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" + } } }, "pg-boss": { @@ -19356,15 +18937,15 @@ } }, "pg-pool": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", - "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", + "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", "requires": {} }, "pg-protocol": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", + "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, "pg-types": { "version": "2.2.0", @@ -20493,201 +20074,14 @@ } }, "tsx": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-3.13.0.tgz", - "integrity": "sha512-rjmRpTu3as/5fjNq/kOkOtihgLxuIz6pbKdj9xwP4J5jOLkBxw/rjN5ANw+KyrrOXV5uB7HC8+SrrSJxT65y+A==", + "version": "4.16.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.16.0.tgz", + "integrity": "sha512-MPgN+CuY+4iKxGoJNPv+1pyo5YWZAQ5XfsyobUG+zoKG7IkvCPLZDEyoIb8yLS2FcWci1nlxAqmvPlFWD5AFiQ==", "dev": true, "requires": { - "esbuild": "~0.18.20", + "esbuild": "~0.21.5", "fsevents": "~2.3.3", - "get-tsconfig": "^4.7.2", - "source-map-support": "^0.5.21" - }, - "dependencies": { - "@esbuild/android-arm": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", - "integrity": "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==", - "dev": true, - "optional": true - }, - "@esbuild/android-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz", - "integrity": "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==", - "dev": true, - "optional": true - }, - "@esbuild/android-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.20.tgz", - "integrity": "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==", - "dev": true, - "optional": true - }, - "@esbuild/darwin-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz", - "integrity": "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==", - "dev": true, - "optional": true - }, - "@esbuild/darwin-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz", - "integrity": "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==", - "dev": true, - "optional": true - }, - "@esbuild/freebsd-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz", - "integrity": "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==", - "dev": true, - "optional": true - }, - "@esbuild/freebsd-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz", - "integrity": "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==", - "dev": true, - "optional": true - }, - "@esbuild/linux-arm": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz", - "integrity": "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==", - "dev": true, - "optional": true - }, - "@esbuild/linux-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz", - "integrity": "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==", - "dev": true, - "optional": true - }, - "@esbuild/linux-ia32": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz", - "integrity": "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==", - "dev": true, - "optional": true - }, - "@esbuild/linux-loong64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz", - "integrity": "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==", - "dev": true, - "optional": true - }, - "@esbuild/linux-mips64el": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz", - "integrity": "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==", - "dev": true, - "optional": true - }, - "@esbuild/linux-ppc64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz", - "integrity": "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==", - "dev": true, - "optional": true - }, - "@esbuild/linux-riscv64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz", - "integrity": "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==", - "dev": true, - "optional": true - }, - "@esbuild/linux-s390x": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz", - "integrity": "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==", - "dev": true, - "optional": true - }, - "@esbuild/linux-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz", - "integrity": "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==", - "dev": true, - "optional": true - }, - "@esbuild/netbsd-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz", - "integrity": "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==", - "dev": true, - "optional": true - }, - "@esbuild/openbsd-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz", - "integrity": "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==", - "dev": true, - "optional": true - }, - "@esbuild/sunos-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz", - "integrity": "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==", - "dev": true, - "optional": true - }, - "@esbuild/win32-arm64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz", - "integrity": "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==", - "dev": true, - "optional": true - }, - "@esbuild/win32-ia32": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz", - "integrity": "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==", - "dev": true, - "optional": true - }, - "@esbuild/win32-x64": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz", - "integrity": "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==", - "dev": true, - "optional": true - }, - "esbuild": { - "version": "0.18.20", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz", - "integrity": "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==", - "dev": true, - "requires": { - "@esbuild/android-arm": "0.18.20", - "@esbuild/android-arm64": "0.18.20", - "@esbuild/android-x64": "0.18.20", - "@esbuild/darwin-arm64": "0.18.20", - "@esbuild/darwin-x64": "0.18.20", - "@esbuild/freebsd-arm64": "0.18.20", - "@esbuild/freebsd-x64": "0.18.20", - "@esbuild/linux-arm": "0.18.20", - "@esbuild/linux-arm64": "0.18.20", - "@esbuild/linux-ia32": "0.18.20", - "@esbuild/linux-loong64": "0.18.20", - "@esbuild/linux-mips64el": "0.18.20", - "@esbuild/linux-ppc64": "0.18.20", - "@esbuild/linux-riscv64": "0.18.20", - "@esbuild/linux-s390x": "0.18.20", - "@esbuild/linux-x64": "0.18.20", - "@esbuild/netbsd-x64": "0.18.20", - "@esbuild/openbsd-x64": "0.18.20", - "@esbuild/sunos-x64": "0.18.20", - "@esbuild/win32-arm64": "0.18.20", - "@esbuild/win32-ia32": "0.18.20", - "@esbuild/win32-x64": "0.18.20" - } - } + "get-tsconfig": "^4.7.5" } }, "tus-js-client": { diff --git a/package.json b/package.json index 56af9e57..b5d9158a 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,9 @@ "description": "Supabase storage middleend", "main": "index.js", "scripts": { - "dev": "tsx watch ./src/server.ts | pino-pretty", + "dev": "tsx watch src/start/server.ts | pino-pretty", "build": "node ./build.js && resolve-tspaths", - "start": "NODE_ENV=production node dist/server.js", + "start": "NODE_ENV=production node dist/start/server.js", "migration:run": "tsx ./src/scripts/migrate-call.ts", "docs:export": "tsx ./src/scripts/export-docs.ts", "test:dummy-data": "tsx -r dotenv/config ./src/test/db/import-dummy-data.ts", @@ -47,7 +47,7 @@ "@tus/file-store": "1.3.1", "@tus/s3-store": "1.4.1", "@tus/server": "1.4.1", - "agentkeepalive": "^4.2.1", + "agentkeepalive": "^4.5.0", "ajv": "^8.12.0", "async-retry": "^1.3.3", "axios": "^1.6.8", @@ -69,7 +69,7 @@ "md5-file": "^5.0.0", "multistream": "^4.1.0", "object-sizeof": "^2.6.4", - "pg": "^8.11.3", + "pg": "^8.12.0", "pg-boss": "^9.0.3", "pg-listen": "^1.7.0", "pino": "^8.15.4", @@ -110,7 +110,7 @@ "stream-buffers": "^3.0.2", "ts-jest": "^29.0.3", "ts-node-dev": "^1.1.8", - "tsx": "^3.13.0", + "tsx": "^4.16.0", "tus-js-client": "^3.1.0", "typescript": "^4.5.5" }, diff --git a/src/config.ts b/src/config.ts index 3df35300..ca9622a3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,6 +9,7 @@ export enum MultitenantMigrationStrategy { } type StorageConfigType = { + isProduction: boolean version: string exposeDocs: boolean keepAliveTimeout: number @@ -19,7 +20,7 @@ type StorageConfigType = { uploadFileSizeLimit: number uploadFileSizeLimitStandard?: number storageFilePath?: string - storageS3MaxSockets?: number + storageS3MaxSockets: number storageS3Bucket: string storageS3Endpoint?: string storageS3ForcePathStyle?: boolean @@ -60,7 +61,10 @@ type StorageConfigType = { logflareSourceToken?: string pgQueueEnable: boolean pgQueueEnableWorkers?: boolean + pgQueueReadWriteTimeout: number + pgQueueMaxConnections: number pgQueueConnectionURL?: string + pgQueueDeleteAfterHours?: number pgQueueDeleteAfterDays?: number pgQueueArchiveCompletedAfterSeconds?: number pgQueueRetentionDays?: number @@ -69,6 +73,8 @@ type StorageConfigType = { webhookQueuePullInterval?: number webhookQueueTeamSize?: number webhookQueueConcurrency?: number + webhookMaxConnections: number + webhookQueueMaxFreeSockets: number adminDeleteQueueTeamSize?: number adminDeleteConcurrency?: number imageTransformationEnabled: boolean @@ -155,6 +161,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { envPaths.map((envPath) => dotenv.config({ path: envPath, override: false })) config = { + isProduction: process.env.NODE_ENV === 'production', exposeDocs: getOptionalConfigFromEnv('EXPOSE_DOCS') !== 'false', // Tenant tenantId: @@ -305,11 +312,15 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { // Queue pgQueueEnable: getOptionalConfigFromEnv('PG_QUEUE_ENABLE', 'ENABLE_QUEUE_EVENTS') === 'true', pgQueueEnableWorkers: getOptionalConfigFromEnv('PG_QUEUE_WORKERS_ENABLE') !== 'false', + pgQueueReadWriteTimeout: Number(getOptionalConfigFromEnv('PG_QUEUE_READ_WRITE_TIMEOUT')) || 0, + pgQueueMaxConnections: Number(getOptionalConfigFromEnv('PG_QUEUE_MAX_CONNECTIONS')) || 4, pgQueueConnectionURL: getOptionalConfigFromEnv('PG_QUEUE_CONNECTION_URL'), pgQueueDeleteAfterDays: parseInt( getOptionalConfigFromEnv('PG_QUEUE_DELETE_AFTER_DAYS') || '2', 10 ), + pgQueueDeleteAfterHours: + Number(getOptionalConfigFromEnv('PG_QUEUE_DELETE_AFTER_HOURS')) || undefined, pgQueueArchiveCompletedAfterSeconds: parseInt( getOptionalConfigFromEnv('PG_QUEUE_ARCHIVE_COMPLETED_AFTER_SECONDS') || '7200', 10 @@ -324,6 +335,12 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { ), webhookQueueTeamSize: parseInt(getOptionalConfigFromEnv('QUEUE_WEBHOOKS_TEAM_SIZE') || '50'), webhookQueueConcurrency: parseInt(getOptionalConfigFromEnv('QUEUE_WEBHOOK_CONCURRENCY') || '5'), + webhookMaxConnections: parseInt( + getOptionalConfigFromEnv('QUEUE_WEBHOOK_MAX_CONNECTIONS') || '500' + ), + webhookQueueMaxFreeSockets: parseInt( + getOptionalConfigFromEnv('QUEUE_WEBHOOK_MAX_FREE_SOCKETS') || '20' + ), adminDeleteQueueTeamSize: parseInt( getOptionalConfigFromEnv('QUEUE_ADMIN_DELETE_TEAM_SIZE') || '50' ), diff --git a/src/http/plugins/log-request.ts b/src/http/plugins/log-request.ts index 0de1866b..26bd191e 100644 --- a/src/http/plugins/log-request.ts +++ b/src/http/plugins/log-request.ts @@ -11,6 +11,7 @@ declare module 'fastify' { executionError?: Error operation?: { type: string } resources?: string[] + startTime: number } interface FastifyContextConfig { @@ -21,6 +22,10 @@ declare module 'fastify' { export const logRequest = (options: RequestLoggerOptions) => fastifyPlugin(async (fastify) => { + fastify.addHook('onRequest', async (req) => { + req.startTime = Date.now() + }) + fastify.addHook('preHandler', async (req) => { const resourceFromParams = Object.values(req.params || {}).join('/') const resources = getFirstDefined( @@ -69,7 +74,7 @@ export const logRequest = (options: RequestLoggerOptions) => logSchema.request(req.log, buildLogMessage, { type: 'request', req, - responseTime: 0, + responseTime: (Date.now() - req.startTime) / 1000, error: error, owner: req.owner, operation: req.operation?.type ?? req.routeConfig.operation?.type, diff --git a/src/http/plugins/storage.ts b/src/http/plugins/storage.ts index 5dd95d2d..8c6127c3 100644 --- a/src/http/plugins/storage.ts +++ b/src/http/plugins/storage.ts @@ -15,7 +15,7 @@ const { storageBackendType } = getConfig() const storageBackend = createStorageBackend(storageBackendType) -export const storage = fastifyPlugin(async (fastify) => { +export const storage = fastifyPlugin(async function storagePlugin(fastify) { fastify.decorateRequest('storage', undefined) fastify.addHook('preHandler', async (request) => { const database = new StorageKnexDB(request.db, { diff --git a/src/http/routes/tus/index.ts b/src/http/routes/tus/index.ts index bbdb32ea..ed6f64a9 100644 --- a/src/http/routes/tus/index.ts +++ b/src/http/routes/tus/index.ts @@ -24,7 +24,7 @@ import { getFileIdFromRequest, SIGNED_URL_SUFFIX, } from './lifecycle' -import { TenantConnection, PubSub } from '../../../internal/database' +import { TenantConnection, PubSub } from '@internal/database' import { S3Store } from '@tus/s3-store' import { NodeHttpHandler } from '@smithy/node-http-handler' import { createAgent } from '@storage/backend' diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 00000000..0eaee6e8 --- /dev/null +++ b/src/index.ts @@ -0,0 +1 @@ +import './start/server' diff --git a/src/internal/database/multitenant-db.ts b/src/internal/database/multitenant-db.ts index 51913af2..cd947a42 100644 --- a/src/internal/database/multitenant-db.ts +++ b/src/internal/database/multitenant-db.ts @@ -5,9 +5,18 @@ const { multitenantDatabaseUrl } = getConfig() export const multitenantKnex = Knex({ client: 'pg', - connection: multitenantDatabaseUrl, + connection: { + connectionString: multitenantDatabaseUrl, + connectionTimeoutMillis: 5000, + }, + version: '12', pool: { min: 0, max: 10, + createTimeoutMillis: 5000, + acquireTimeoutMillis: 5000, + idleTimeoutMillis: 5000, + reapIntervalMillis: 1000, + createRetryIntervalMillis: 100, }, }) diff --git a/src/internal/monitoring/logger.ts b/src/internal/monitoring/logger.ts index 8f70e672..78a4058a 100644 --- a/src/internal/monitoring/logger.ts +++ b/src/internal/monitoring/logger.ts @@ -6,7 +6,7 @@ import { normalizeRawError } from '@internal/errors' const { logLevel, logflareApiKey, logflareSourceToken, logflareEnabled, region } = getConfig() -export const logger = pino({ +export const baseLogger = pino({ transport: buildTransport(), serializers: { error(error) { @@ -40,6 +40,8 @@ export const logger = pino({ timestamp: pino.stdTimeFunctions.isoTime, }) +export const logger = baseLogger.child({ region }) + export interface RequestLog { type: 'request' req: FastifyRequest @@ -77,6 +79,8 @@ interface InfoLog { export const logSchema = { info: (logger: BaseLogger, message: string, log: InfoLog) => logger.info(log, message), + warning: (logger: BaseLogger, message: string, log: InfoLog | ErrorLog) => + logger.warn(log, message), request: (logger: BaseLogger, message: string, log: RequestLog) => { if (!log.res) { logger.warn(log, message) diff --git a/src/internal/queue/database.ts b/src/internal/queue/database.ts new file mode 100644 index 00000000..e1172556 --- /dev/null +++ b/src/internal/queue/database.ts @@ -0,0 +1,40 @@ +import { Db } from 'pg-boss' +import EventEmitter from 'events' +import pg from 'pg' +import { ERRORS } from '@internal/errors' + +export class QueueDB extends EventEmitter implements Db { + opened = false + isOurs = true + + protected config: pg.PoolConfig + protected pool?: pg.Pool + + constructor(config: pg.PoolConfig) { + super() + + config.application_name = config.application_name || 'pgboss' + + this.config = config + } + + async open() { + this.pool = new pg.Pool({ ...this.config, min: 0 }) + this.pool.on('error', (error) => this.emit('error', error)) + + this.opened = true + } + + async close() { + this.opened = false + await this.pool?.end() + } + + async executeSql(text: string, values: any[]) { + if (this.opened && this.pool) { + return await this.pool.query(text, values) + } + + throw ERRORS.InternalError(undefined, `QueueDB not opened ${this.opened} ${text}`) + } +} diff --git a/src/internal/queue/event.ts b/src/internal/queue/event.ts new file mode 100644 index 00000000..4d6d3128 --- /dev/null +++ b/src/internal/queue/event.ts @@ -0,0 +1,216 @@ +import { Queue } from './queue' +import PgBoss, { BatchWorkOptions, Job, SendOptions, WorkOptions } from 'pg-boss' +import { getConfig } from '../../config' +import { QueueJobScheduled, QueueJobSchedulingTime } from '@internal/monitoring/metrics' +import { logger, logSchema } from '@internal/monitoring' + +export interface BasePayload { + $version?: string + singletonKey?: string + reqId?: string + tenant: { + ref: string + host: string + } +} + +export interface SlowRetryQueueOptions { + retryLimit: number + retryDelay: number +} + +const { pgQueueEnable, region } = getConfig() + +export type StaticThis> = BaseEventConstructor + +interface BaseEventConstructor> { + version: string + + new (...args: any): Base + + send( + this: StaticThis, + payload: Omit + ): Promise + + eventName(): string + getWorkerOptions(): WorkOptions | BatchWorkOptions +} + +export abstract class Event> { + public static readonly version: string = 'v1' + protected static queueName = '' + + constructor(public readonly payload: T & BasePayload) {} + + static eventName() { + return this.name + } + + static getQueueName() { + if (!this.queueName) { + throw new Error(`Queue name not set on ${this.constructor.name}`) + } + + return this.queueName + } + + static getQueueOptions>(payload: T['payload']): SendOptions | undefined { + return undefined + } + + static getWorkerOptions(): WorkOptions | BatchWorkOptions { + return {} + } + + static withSlowRetryQueue(): undefined | SlowRetryQueueOptions { + return undefined + } + + static getSlowRetryQueueName() { + if (!this.queueName) { + throw new Error(`Queue name not set on ${this.constructor.name}`) + } + + return this.queueName + '-slow' + } + + static batchSend[]>(messages: T) { + return Queue.getInstance().insert( + messages.map((message) => { + const sendOptions = (this.getQueueOptions(message.payload) as PgBoss.JobInsert) || {} + if (!message.payload.$version) { + ;(message.payload as (typeof message)['payload']).$version = this.version + } + return { + ...sendOptions, + name: this.getQueueName(), + data: message.payload, + } + }) + ) + } + + static send>(this: StaticThis, payload: Omit) { + if (!payload.$version) { + ;(payload as T['payload']).$version = this.version + } + const that = new this(payload) + return that.send() + } + + static sendSlowRetryQueue>( + this: StaticThis, + payload: Omit + ) { + if (!payload.$version) { + ;(payload as T['payload']).$version = this.version + } + const that = new this(payload) + return that.sendSlowRetryQueue() + } + + static handle(job: Job['payload']> | Job['payload']>[]) { + throw new Error('not implemented') + } + + async send(): Promise { + const constructor = this.constructor as typeof Event + + if (!pgQueueEnable) { + return constructor.handle({ + id: '__sync', + name: constructor.getQueueName(), + data: { + region, + ...this.payload, + $version: constructor.version, + }, + }) + } + + const timer = QueueJobSchedulingTime.startTimer() + const sendOptions = constructor.getQueueOptions(this.payload) + + try { + const res = await Queue.getInstance().send({ + name: constructor.getQueueName(), + data: { + region, + ...this.payload, + $version: constructor.version, + }, + options: sendOptions, + }) + + QueueJobScheduled.inc({ + name: constructor.getQueueName(), + }) + + return res + } catch (e) { + // If we can't queue the message for some reason, + // we run its handler right away. + // This might create some latency with the benefit of being more fault-tolerant + logSchema.warning( + logger, + `[Queue Sender] Error while sending job to queue, sending synchronously`, + { + type: 'queue', + error: e, + metadata: JSON.stringify(this.payload), + } + ) + return constructor.handle({ + id: '__sync', + name: constructor.getQueueName(), + data: { + region, + ...this.payload, + $version: constructor.version, + }, + }) + } finally { + timer({ + name: constructor.getQueueName(), + }) + } + } + + async sendSlowRetryQueue() { + const constructor = this.constructor as typeof Event + const slowRetryQueue = constructor.withSlowRetryQueue() + + if (!pgQueueEnable || !slowRetryQueue) { + return + } + + const timer = QueueJobSchedulingTime.startTimer() + const sendOptions = constructor.getQueueOptions(this.payload) || {} + + const res = await Queue.getInstance().send({ + name: constructor.getSlowRetryQueueName(), + data: { + region, + ...this.payload, + $version: constructor.version, + }, + options: { + retryBackoff: true, + startAfter: 60 * 60 * 30, // 30 mins + ...sendOptions, + ...slowRetryQueue, + }, + }) + + timer({ + name: constructor.getSlowRetryQueueName(), + }) + + QueueJobScheduled.inc({ + name: constructor.getSlowRetryQueueName(), + }) + + return res + } +} diff --git a/src/internal/queue/index.ts b/src/internal/queue/index.ts index 2de5032b..0210f46b 100644 --- a/src/internal/queue/index.ts +++ b/src/internal/queue/index.ts @@ -1 +1,2 @@ export * from './queue' +export * from './event' diff --git a/src/internal/queue/queue.ts b/src/internal/queue/queue.ts index 81b451d8..2320ab9a 100644 --- a/src/internal/queue/queue.ts +++ b/src/internal/queue/queue.ts @@ -1,13 +1,14 @@ import PgBoss, { Job, JobWithMetadata } from 'pg-boss' +import { ERRORS } from '@internal/errors' +import { QueueDB } from '@internal/queue/database' import { getConfig } from '../../config' -import { BaseEvent, BasePayload } from '../../storage/events' -import { QueueJobRetryFailed, QueueJobCompleted, QueueJobError } from '../monitoring/metrics' import { logger, logSchema } from '../monitoring' -import { ERRORS } from '@internal/errors' +import { QueueJobRetryFailed, QueueJobCompleted, QueueJobError } from '../monitoring/metrics' +import { BasePayload, Event } from './event' //eslint-disable-next-line @typescript-eslint/no-explicit-any -type SubclassOfBaseClass = (new (payload: any) => BaseEvent) & { - [K in keyof typeof BaseEvent]: (typeof BaseEvent)[K] +type SubclassOfBaseClass = (new (payload: any) => Event) & { + [K in keyof typeof Event]: (typeof Event)[K] } export abstract class Queue { @@ -33,9 +34,12 @@ export abstract class Queue { multitenantDatabaseUrl, pgQueueConnectionURL, pgQueueDeleteAfterDays, + pgQueueDeleteAfterHours, pgQueueArchiveCompletedAfterSeconds, pgQueueRetentionDays, pgQueueEnableWorkers, + pgQueueReadWriteTimeout, + pgQueueMaxConnections, } = getConfig() let url = pgQueueConnectionURL ?? databaseURL @@ -51,14 +55,22 @@ export abstract class Queue { Queue.pgBoss = new PgBoss({ connectionString: url, - max: 4, + db: new QueueDB({ + min: 0, + max: pgQueueMaxConnections, + connectionString: url, + statement_timeout: pgQueueReadWriteTimeout > 0 ? pgQueueReadWriteTimeout : undefined, + }), application_name: 'storage-pgboss', - deleteAfterDays: pgQueueDeleteAfterDays, + deleteAfterDays: pgQueueDeleteAfterHours ? undefined : pgQueueDeleteAfterDays, + deleteAfterHours: pgQueueDeleteAfterHours, archiveCompletedAfterSeconds: pgQueueArchiveCompletedAfterSeconds, retentionDays: pgQueueRetentionDays, retryBackoff: true, retryLimit: 20, expireInHours: 48, + noSupervisor: pgQueueEnableWorkers === false, + noScheduling: pgQueueEnableWorkers === false, }) Queue.pgBoss.on('error', (error) => { @@ -121,10 +133,11 @@ export abstract class Queue { } const boss = this.pgBoss + const { isProduction } = getConfig() await boss.stop({ timeout: 20 * 1000, - graceful: true, + graceful: isProduction, destroy: true, }) diff --git a/src/server.ts b/src/start/server.ts similarity index 90% rename from src/server.ts rename to src/start/server.ts index e60b3e8d..c90d8dcb 100644 --- a/src/server.ts +++ b/src/start/server.ts @@ -1,10 +1,10 @@ -import './internal/monitoring/otel' +import '@internal/monitoring/otel' import { FastifyInstance } from 'fastify' import { IncomingMessage, Server, ServerResponse } from 'http' -import build from './app' -import buildAdmin from './admin-app' -import { getConfig } from './config' +import build from '../app' +import buildAdmin from '../admin-app' +import { getConfig } from '../config' import { runMultitenantMigrations, runMigrationsOnTenant, @@ -16,6 +16,7 @@ import { logger, logSchema } from '@internal/monitoring' import { Queue } from '@internal/queue' import { registerWorkers } from '@storage/events' import { AsyncAbortController } from '@internal/concurrency' + import { bindShutdownSignals, createServerClosedPromise, shutdown } from './shutdown' const shutdownSignal = new AsyncAbortController() @@ -46,13 +47,12 @@ main() * Start Storage API server */ async function main() { - const { databaseURL, isMultitenant, pgQueueEnable, pgQueueEnableWorkers } = getConfig() + const { databaseURL, isMultitenant, pgQueueEnable } = getConfig() // Migrations if (isMultitenant) { await runMultitenantMigrations() await listenForTenantUpdate(PubSub) - startAsyncMigrations(shutdownSignal.nextGroup.signal) } else { await runMigrationsOnTenant(databaseURL) } @@ -70,12 +70,17 @@ async function main() { signal: shutdownSignal.nextGroup.signal, }) + // Start async migrations background process + if (isMultitenant) { + startAsyncMigrations(shutdownSignal.nextGroup.signal) + } + // HTTP Server const app = await httpServer(shutdownSignal.signal) - // HTTP Server Admin + // HTTP Admin Server if (isMultitenant) { - await httpAdminApp(app, shutdownSignal.signal) + await httpAdminServer(app, shutdownSignal.signal) } } @@ -128,7 +133,7 @@ async function httpServer(signal: AbortSignal) { * @param app * @param signal */ -async function httpAdminApp( +async function httpAdminServer( app: FastifyInstance, signal: AbortSignal ) { diff --git a/src/shutdown.ts b/src/start/shutdown.ts similarity index 100% rename from src/shutdown.ts rename to src/start/shutdown.ts diff --git a/src/worker.ts b/src/start/worker.ts similarity index 96% rename from src/worker.ts rename to src/start/worker.ts index 2eea0c86..b14fc966 100644 --- a/src/worker.ts +++ b/src/start/worker.ts @@ -4,8 +4,8 @@ import { listenForTenantUpdate, PubSub } from '@internal/database' import { AsyncAbortController } from '@internal/concurrency' import { registerWorkers } from '@storage/events' -import { getConfig } from './config' -import adminApp from './admin-app' +import { getConfig } from '../config' +import adminApp from '../admin-app' import { bindShutdownSignals, createServerClosedPromise, shutdown } from './shutdown' const shutdownSignal = new AsyncAbortController() diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index 6c97e278..9e53ddbd 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -101,7 +101,12 @@ export interface Database { listBuckets(columns: string): Promise mustLockObject(bucketId: string, objectName: string, version?: string): Promise - waitObjectLock(bucketId: string, objectName: string, version?: string): Promise + waitObjectLock( + bucketId: string, + objectName: string, + version?: string, + opts?: { timeout?: number } + ): Promise updateBucket( bucketId: string, diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 6ab3261c..c771576b 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -530,10 +530,38 @@ export class StorageKnexDB implements Database { }) } - async waitObjectLock(bucketId: string, objectName: string, version?: string) { + async waitObjectLock( + bucketId: string, + objectName: string, + version?: string, + opts?: { timeout: number } + ) { return this.runQuery('WaitObjectLock', async (knex) => { const hash = hashStringToInt(`${bucketId}/${objectName}${version ? `/${version}` : ''}`) - await knex.raw(`SELECT pg_advisory_xact_lock(?)`, [hash]) + const query = knex.raw(`SELECT pg_advisory_xact_lock(?)`, [hash]) + + if (opts?.timeout) { + let timeoutInterval: undefined | NodeJS.Timeout + + try { + await Promise.race([ + query, + new Promise( + (_, reject) => + (timeoutInterval = setTimeout(() => reject(ERRORS.LockTimeout()), opts.timeout)) + ), + ]) + } catch (e) { + throw e + } finally { + if (timeoutInterval) { + clearTimeout(timeoutInterval) + } + } + } else { + await query + } + return true }) } diff --git a/src/storage/events/base-event.ts b/src/storage/events/base-event.ts index 846dc20d..c8624779 100644 --- a/src/storage/events/base-event.ts +++ b/src/storage/events/base-event.ts @@ -1,125 +1,21 @@ -import { Queue } from '@internal/queue' -import PgBoss, { BatchWorkOptions, Job, SendOptions, WorkOptions } from 'pg-boss' +import { Event as QueueBaseEvent, BasePayload, StaticThis, Event } from '@internal/queue' import { getPostgresConnection, getServiceKeyUser } from '@internal/database' -import { Storage } from '../index' import { StorageKnexDB } from '../database' import { createAgent, createStorageBackend } from '../backend' +import { Storage } from '../index' import { getConfig } from '../../config' -import { QueueJobScheduled, QueueJobSchedulingTime } from '@internal/monitoring/metrics' import { logger } from '@internal/monitoring' -export interface BasePayload { - $version?: string - singletonKey?: string - reqId?: string - tenant: { - ref: string - host: string - } -} - -export interface SlowRetryQueueOptions { - retryLimit: number - retryDelay: number -} - -const { pgQueueEnable, storageBackendType, storageS3Endpoint, region } = getConfig() +const { storageBackendType, storageS3Endpoint, region } = getConfig() const storageS3Protocol = storageS3Endpoint?.includes('http://') ? 'http' : 'https' const httpAgent = createAgent(storageS3Protocol) -type StaticThis> = BaseEventConstructor - -interface BaseEventConstructor> { - version: string - - new (...args: any): Base - - send( - this: StaticThis, - payload: Omit - ): Promise - - eventName(): string - getWorkerOptions(): WorkOptions | BatchWorkOptions -} - -export abstract class BaseEvent> { - public static readonly version: string = 'v1' - protected static queueName = '' - - constructor(public readonly payload: T & BasePayload) {} - - static eventName() { - return this.name - } - - static getQueueName() { - if (!this.queueName) { - throw new Error(`Queue name not set on ${this.constructor.name}`) - } - - return this.queueName - } - - static getQueueOptions>(payload: T['payload']): SendOptions | undefined { - return undefined - } - - static getWorkerOptions(): WorkOptions | BatchWorkOptions { - return {} - } - - static withSlowRetryQueue(): undefined | SlowRetryQueueOptions { - return undefined - } - - static getSlowRetryQueueName() { - if (!this.queueName) { - throw new Error(`Queue name not set on ${this.constructor.name}`) - } - - return this.queueName + '-slow' - } - - static batchSend[]>(messages: T) { - return Queue.getInstance().insert( - messages.map((message) => { - const sendOptions = (this.getQueueOptions(message.payload) as PgBoss.JobInsert) || {} - if (!message.payload.$version) { - ;(message.payload as (typeof message)['payload']).$version = this.version - } - return { - ...sendOptions, - name: this.getQueueName(), - data: message.payload, - } - }) - ) - } - - static send>( - this: StaticThis, - payload: Omit - ) { - if (!payload.$version) { - ;(payload as T['payload']).$version = this.version - } - const that = new this(payload) - return that.send() - } - - static sendSlowRetryQueue>( - this: StaticThis, - payload: Omit - ) { - if (!payload.$version) { - ;(payload as T['payload']).$version = this.version - } - const that = new this(payload) - return that.sendSlowRetryQueue() - } - - static async sendWebhook>( +export abstract class BaseEvent> extends QueueBaseEvent { + /** + * Sends a message as a webhook + * @param payload + */ + static async sendWebhook>( this: StaticThis, payload: Omit ) { @@ -155,10 +51,6 @@ export abstract class BaseEvent> { } } - static handle(job: Job['payload']> | Job['payload']>[]) { - throw new Error('not implemented') - } - protected static async createStorage(payload: BasePayload) { const adminUser = await getServiceKeyUser(payload.tenant.ref) @@ -181,80 +73,4 @@ export abstract class BaseEvent> { return new Storage(storageBackend, db) } - - async send(): Promise { - const constructor = this.constructor as typeof BaseEvent - - if (!pgQueueEnable) { - return constructor.handle({ - id: '', - name: constructor.getQueueName(), - data: { - region, - ...this.payload, - $version: constructor.version, - }, - }) - } - - const timer = QueueJobSchedulingTime.startTimer() - const sendOptions = constructor.getQueueOptions(this.payload) - - const res = await Queue.getInstance().send({ - name: constructor.getQueueName(), - data: { - region, - ...this.payload, - $version: constructor.version, - }, - options: sendOptions, - }) - - timer({ - name: constructor.getQueueName(), - }) - - QueueJobScheduled.inc({ - name: constructor.getQueueName(), - }) - - return res - } - - async sendSlowRetryQueue() { - const constructor = this.constructor as typeof BaseEvent - const slowRetryQueue = constructor.withSlowRetryQueue() - - if (!pgQueueEnable || !slowRetryQueue) { - return - } - - const timer = QueueJobSchedulingTime.startTimer() - const sendOptions = constructor.getQueueOptions(this.payload) || {} - - const res = await Queue.getInstance().send({ - name: constructor.getSlowRetryQueueName(), - data: { - region, - ...this.payload, - $version: constructor.version, - }, - options: { - retryBackoff: true, - startAfter: 60 * 60 * 30, // 30 mins - ...sendOptions, - ...slowRetryQueue, - }, - }) - - timer({ - name: constructor.getSlowRetryQueueName(), - }) - - QueueJobScheduled.inc({ - name: constructor.getSlowRetryQueueName(), - }) - - return res - } } diff --git a/src/storage/events/object-admin-delete.ts b/src/storage/events/object-admin-delete.ts index d22f89c4..5f7152b2 100644 --- a/src/storage/events/object-admin-delete.ts +++ b/src/storage/events/object-admin-delete.ts @@ -1,9 +1,10 @@ -import { BaseEvent, BasePayload } from './base-event' +import { BaseEvent } from './base-event' import { getConfig } from '../../config' import { Job, SendOptions, WorkOptions } from 'pg-boss' import { withOptionalVersion } from '../backend' import { logger, logSchema } from '@internal/monitoring' import { Storage } from '../index' +import { BasePayload } from '@internal/queue' export interface ObjectDeleteEvent extends BasePayload { name: string @@ -65,6 +66,7 @@ export class ObjectAdminDelete extends BaseEvent { event: 'ObjectAdminDelete', payload: JSON.stringify(job.data), objectPath: s3Key, + objectVersion: job.data.version, tenantId: job.data.tenant.ref, project: job.data.tenant.ref, reqId: job.data.reqId, diff --git a/src/storage/events/object-created.ts b/src/storage/events/object-created.ts index ede20611..bcbe61fe 100644 --- a/src/storage/events/object-created.ts +++ b/src/storage/events/object-created.ts @@ -1,9 +1,11 @@ -import { BaseEvent, BasePayload } from './base-event' +import { BasePayload } from '@internal/queue' +import { BaseEvent } from './base-event' import { ObjectMetadata } from '../backend' import { ObjectRemovedEvent } from './object-removed' interface ObjectCreatedEvent extends BasePayload { name: string + version: string bucketId: string metadata: ObjectMetadata uploadType: 'standard' | 'resumable' | 's3' diff --git a/src/storage/events/object-removed.ts b/src/storage/events/object-removed.ts index b333615d..bb239eba 100644 --- a/src/storage/events/object-removed.ts +++ b/src/storage/events/object-removed.ts @@ -1,8 +1,12 @@ -import { BaseEvent, BasePayload } from './base-event' +import { BasePayload } from '@internal/queue' +import { BaseEvent } from './base-event' +import { ObjectMetadata } from '@storage/backend' export interface ObjectRemovedEvent extends BasePayload { name: string bucketId: string + version: string + metadata?: ObjectMetadata } export class ObjectRemoved extends BaseEvent { diff --git a/src/storage/events/object-updated.ts b/src/storage/events/object-updated.ts index 98b550d9..6cc6673c 100644 --- a/src/storage/events/object-updated.ts +++ b/src/storage/events/object-updated.ts @@ -1,9 +1,11 @@ -import { BaseEvent, BasePayload } from './base-event' +import { BasePayload } from '@internal/queue' +import { BaseEvent } from './base-event' import { ObjectMetadata } from '../backend' interface ObjectUpdatedMetadataEvent extends BasePayload { name: string bucketId: string + version: string metadata: ObjectMetadata } diff --git a/src/storage/events/run-migrations.ts b/src/storage/events/run-migrations.ts index 4cb2e70f..e2cba669 100644 --- a/src/storage/events/run-migrations.ts +++ b/src/storage/events/run-migrations.ts @@ -1,4 +1,4 @@ -import { BaseEvent, BasePayload } from './base-event' +import { BaseEvent } from './base-event' import { areMigrationsUpToDate, getTenantConfig, @@ -8,6 +8,7 @@ import { } from '@internal/database' import { JobWithMetadata, SendOptions, WorkOptions } from 'pg-boss' import { logger, logSchema } from '@internal/monitoring' +import { BasePayload } from '@internal/queue' interface RunMigrationsPayload extends BasePayload { tenantId: string diff --git a/src/storage/events/webhook.ts b/src/storage/events/webhook.ts index 5e20b776..38e3e17c 100644 --- a/src/storage/events/webhook.ts +++ b/src/storage/events/webhook.ts @@ -12,6 +12,8 @@ const { webhookQueuePullInterval, webhookQueueTeamSize, webhookQueueConcurrency, + webhookMaxConnections, + webhookQueueMaxFreeSockets, } = getConfig() interface WebhookEvent { @@ -31,12 +33,14 @@ interface WebhookEvent { const httpAgent = webhookURL?.startsWith('https://') ? { httpsAgent: new HttpsAgent({ - maxSockets: 100, + maxSockets: webhookMaxConnections, + maxFreeSockets: webhookQueueMaxFreeSockets, }), } : { httpAgent: new HttpAgent({ - maxSockets: 100, + maxSockets: webhookMaxConnections, + maxFreeSockets: webhookQueueMaxFreeSockets, }), } @@ -73,7 +77,7 @@ export class Webhook extends BaseEvent { event: job.data.event.type, payload: JSON.stringify(job.data.event.payload), objectPath: path, - resources: [path], + resources: ['/' + path], tenantId: job.data.tenant.ref, project: job.data.tenant.ref, reqId: job.data.event.payload.reqId, diff --git a/src/storage/object.ts b/src/storage/object.ts index 6ccd0aa8..48ce9f3e 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -110,7 +110,7 @@ export class ObjectStorage { * @param objectName */ async deleteObject(objectName: string) { - await this.db.withTransaction(async (db) => { + const obj = await this.db.withTransaction(async (db) => { const obj = await db.asSuperUser().findObject(this.bucketId, objectName, 'id,version', { forUpdate: true, }) @@ -126,13 +126,17 @@ export class ObjectStorage { `${this.db.tenantId}/${this.bucketId}/${objectName}`, obj.version ) + + return obj }) await ObjectRemoved.sendWebhook({ tenant: this.db.tenant(), name: objectName, + version: obj.version, bucketId: this.bucketId, reqId: this.db.reqId, + metadata: obj.metadata, }) } @@ -182,6 +186,8 @@ export class ObjectStorage { name: object.name, bucketId: this.bucketId, reqId: this.db.reqId, + version: object.version, + metadata: object.metadata, }) ) ) @@ -205,6 +211,7 @@ export class ObjectStorage { await ObjectUpdatedMetadata.sendWebhook({ tenant: this.db.tenant(), name: objectName, + version: result.version, bucketId: this.bucketId, metadata, reqId: this.db.reqId, @@ -319,6 +326,7 @@ export class ObjectStorage { await ObjectCreatedCopyEvent.sendWebhook({ tenant: this.db.tenant(), name: destinationKey, + version: newVersion, bucketId: this.bucketId, metadata, reqId: this.db.reqId, @@ -395,13 +403,20 @@ export class ObjectStorage { const metadata = await this.backend.headObject(storageS3Bucket, s3DestinationKey, newVersion) return this.db.asSuperUser().withTransaction(async (db) => { - await db.waitObjectLock(this.bucketId, destinationObjectName) - - const sourceObject = await db.findObject(this.bucketId, sourceObjectName, 'id', { - forUpdate: true, - dontErrorOnEmpty: false, + await db.waitObjectLock(this.bucketId, destinationObjectName, undefined, { + timeout: 5000, }) + const sourceObject = await db.findObject( + this.bucketId, + sourceObjectName, + 'id,version,metadata', + { + forUpdate: true, + dontErrorOnEmpty: false, + } + ) + await db.updateObject(this.bucketId, sourceObjectName, { name: destinationObjectName, bucket_id: destinationBucket, @@ -424,16 +439,20 @@ export class ObjectStorage { name: sourceObjectName, bucketId: this.bucketId, reqId: this.db.reqId, + version: sourceObject.version, + metadata: sourceObject.metadata, }), ObjectCreatedMove.sendWebhook({ tenant: this.db.tenant(), name: destinationObjectName, + version: newVersion, bucketId: this.bucketId, metadata: metadata, oldObject: { name: sourceObjectName, bucketId: this.bucketId, reqId: this.db.reqId, + version: sourceObject.version, }, reqId: this.db.reqId, }), diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index 30502bc5..86bd1fb5 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -1,5 +1,6 @@ import crypto from 'crypto' import { ERRORS } from '@internal/errors' +import { signatureV4 } from '../../../http/plugins' interface SignatureV4Options { enforceRegion: boolean @@ -105,7 +106,7 @@ export class SignatureV4 { return { credentials: { accessKey, shortDate, region, service }, signedHeaders, - signature, + signature: signature as string, longDate, contentSha, sessionToken, @@ -114,11 +115,11 @@ export class SignatureV4 { static parseQuerySignature(query: Record) { const credentialPart = query['X-Amz-Credential'] - const signedHeaders = query['X-Amz-SignedHeaders'] - const signature = query['X-Amz-Signature'] - const longDate = query['X-Amz-Date'] - const contentSha = query['X-Amz-Content-Sha256'] - const sessionToken = query['X-Amz-Security-Token'] + const signedHeaders: string = query['X-Amz-SignedHeaders'] + const signature: string = query['X-Amz-Signature'] + const longDate: string = query['X-Amz-Date'] + const contentSha: string = query['X-Amz-Content-Sha256'] + const sessionToken: string | undefined = query['X-Amz-Security-Token'] const expires = query['X-Amz-Expires'] if (!validateTypeOfStrings(credentialPart, signedHeaders, signature, longDate)) { @@ -129,7 +130,7 @@ export class SignatureV4 { this.checkExpiration(longDate, expires) } - const credentialsPart = credentialPart.split('/') + const credentialsPart = credentialPart.split('/') as string[] if (credentialsPart.length !== 5) { throw ERRORS.InvalidSignature('Invalid credentials') } diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index 663272c2..86ae602f 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -9,6 +9,7 @@ import { getFileSizeLimit, isEmptyFolder } from './limits' import { Database } from './database' import { ObjectAdminDelete, ObjectCreatedPostEvent, ObjectCreatedPutEvent } from './events' import { getConfig } from '../config' +import { logger, logSchema } from '@internal/monitoring' interface UploaderOptions extends UploadObjectOptions { fileSizeLimit?: number | null @@ -141,20 +142,20 @@ export class Uploader { uploadType?: 'standard' | 's3' | 'resumable' }) { try { - return await this.db.withTransaction(async (db) => { - await db.waitObjectLock(bucketId, objectName) + return await this.db.asSuperUser().withTransaction(async (db) => { + await db.waitObjectLock(bucketId, objectName, undefined, { + timeout: 5000, + }) - const currentObj = await db - .asSuperUser() - .findObject(bucketId, objectName, 'id, version, metadata', { - forUpdate: true, - dontErrorOnEmpty: true, - }) + const currentObj = await db.findObject(bucketId, objectName, 'id, version, metadata', { + forUpdate: true, + dontErrorOnEmpty: true, + }) const isNew = !Boolean(currentObj) // update object - const newObject = await db.asSuperUser().upsertObject({ + const newObject = await db.upsertObject({ bucket_id: bucketId, name: objectName, metadata: objectMetadata, @@ -180,14 +181,30 @@ export class Uploader { const event = isUpsert && !isNew ? ObjectCreatedPutEvent : ObjectCreatedPostEvent events.push( - event.sendWebhook({ - tenant: this.db.tenant(), - name: objectName, - bucketId: bucketId, - metadata: objectMetadata, - reqId: this.db.reqId, - uploadType, - }) + event + .sendWebhook({ + tenant: this.db.tenant(), + name: objectName, + version: version, + bucketId: bucketId, + metadata: objectMetadata, + reqId: this.db.reqId, + uploadType, + }) + .catch((e) => { + logSchema.error(logger, 'Failed to send webhook', { + type: 'event', + error: e, + project: this.db.tenantId, + metadata: JSON.stringify({ + name: objectName, + bucketId: bucketId, + metadata: objectMetadata, + reqId: this.db.reqId, + uploadType, + }), + }) + }) ) await Promise.all(events) diff --git a/src/test/webhooks.test.ts b/src/test/webhooks.test.ts index 805565e0..ae280fdb 100644 --- a/src/test/webhooks.test.ts +++ b/src/test/webhooks.test.ts @@ -1,4 +1,4 @@ -import { TenantConnection } from '../internal/database/connection' +import { TenantConnection } from '@internal/database' import { getConfig, mergeConfig } from '../config' const { serviceKey, tenantId } = getConfig() @@ -12,10 +12,10 @@ import FormData from 'form-data' import fs from 'fs' import app from '../app' -import { getPostgresConnection } from '../internal/database' -import { Obj } from '../storage/schemas' +import { getPostgresConnection } from '@internal/database' +import { Obj } from '@storage/schemas' import { randomUUID } from 'crypto' -import { getServiceKeyUser } from '../internal/database/tenant' +import { getServiceKeyUser } from '@internal/database' describe('Webhooks', () => { useMockObject() @@ -170,6 +170,7 @@ describe('Webhooks', () => { payload: expect.objectContaining({ bucketId: 'bucket6', name: obj.name, + version: expect.any(String), tenant: { host: undefined, ref: 'bjhaohmqunupljrqypxz', @@ -198,6 +199,7 @@ describe('Webhooks', () => { applyTime: expect.any(Number), payload: expect.objectContaining({ bucketId: 'bucket6', + version: expect.any(String), metadata: expect.objectContaining({ cacheControl: 'no-cache', contentLength: 3746, @@ -212,6 +214,7 @@ describe('Webhooks', () => { bucketId: 'bucket6', name: obj.name, reqId: expect.any(String), + version: expect.any(String), }, tenant: { host: undefined,