Skip to content

Commit

Permalink
fix: custom webhooks-max-sockets env
Browse files Browse the repository at this point in the history
  • Loading branch information
fenos committed Jul 2, 2024
1 parent 74492da commit 57eebd6
Show file tree
Hide file tree
Showing 28 changed files with 457 additions and 322 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ COPY --from=production-deps /app/node_modules node_modules
COPY --from=build /app/dist dist

EXPOSE 5000
CMD ["node", "dist/server.js"]
CMD ["node", "dist/start/server.js"]
105 changes: 45 additions & 60 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
"description": "Supabase storage middleend",
"main": "index.js",
"scripts": {
"dev": "tsx watch ./src/server.ts | pino-pretty",
"dev": "tsx watch src/start/server.ts | pino-pretty",
"build": "node ./build.js && resolve-tspaths",
"start": "NODE_ENV=production node dist/server.js",
"start": "NODE_ENV=production node dist/start/server.js",
"migration:run": "tsx ./src/scripts/migrate-call.ts",
"docs:export": "tsx ./src/scripts/export-docs.ts",
"test:dummy-data": "tsx -r dotenv/config ./src/test/db/import-dummy-data.ts",
Expand Down Expand Up @@ -69,7 +69,7 @@
"md5-file": "^5.0.0",
"multistream": "^4.1.0",
"object-sizeof": "^2.6.4",
"pg": "^8.11.3",
"pg": "^8.12.0",
"pg-boss": "^9.0.3",
"pg-listen": "^1.7.0",
"pino": "^8.15.4",
Expand Down Expand Up @@ -110,7 +110,7 @@
"stream-buffers": "^3.0.2",
"ts-jest": "^29.0.3",
"ts-node-dev": "^1.1.8",
"tsx": "^3.13.0",
"tsx": "^3.14.0",
"tus-js-client": "^3.1.0",
"typescript": "^4.5.5"
},
Expand Down
8 changes: 7 additions & 1 deletion src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export enum MultitenantMigrationStrategy {
}

type StorageConfigType = {
isProduction: boolean
version: string
exposeDocs: boolean
keepAliveTimeout: number
Expand All @@ -19,7 +20,7 @@ type StorageConfigType = {
uploadFileSizeLimit: number
uploadFileSizeLimitStandard?: number
storageFilePath?: string
storageS3MaxSockets?: number
storageS3MaxSockets: number
storageS3Bucket: string
storageS3Endpoint?: string
storageS3ForcePathStyle?: boolean
Expand Down Expand Up @@ -69,6 +70,7 @@ type StorageConfigType = {
webhookQueuePullInterval?: number
webhookQueueTeamSize?: number
webhookQueueConcurrency?: number
webhookMaxConnections: number
adminDeleteQueueTeamSize?: number
adminDeleteConcurrency?: number
imageTransformationEnabled: boolean
Expand Down Expand Up @@ -155,6 +157,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType {
envPaths.map((envPath) => dotenv.config({ path: envPath, override: false }))

config = {
isProduction: process.env.NODE_ENV === 'production',
exposeDocs: getOptionalConfigFromEnv('EXPOSE_DOCS') !== 'false',
// Tenant
tenantId:
Expand Down Expand Up @@ -324,6 +327,9 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType {
),
webhookQueueTeamSize: parseInt(getOptionalConfigFromEnv('QUEUE_WEBHOOKS_TEAM_SIZE') || '50'),
webhookQueueConcurrency: parseInt(getOptionalConfigFromEnv('QUEUE_WEBHOOK_CONCURRENCY') || '5'),
webhookMaxConnections: parseInt(
getOptionalConfigFromEnv('QUEUE_WEBHOOK_MAX_CONNECTIONS') || '500'
),
adminDeleteQueueTeamSize: parseInt(
getOptionalConfigFromEnv('QUEUE_ADMIN_DELETE_TEAM_SIZE') || '50'
),
Expand Down
7 changes: 6 additions & 1 deletion src/http/plugins/log-request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ declare module 'fastify' {
executionError?: Error
operation?: { type: string }
resources?: string[]
startTime: number
}

interface FastifyContextConfig {
Expand All @@ -21,6 +22,10 @@ declare module 'fastify' {

export const logRequest = (options: RequestLoggerOptions) =>
fastifyPlugin(async (fastify) => {
fastify.addHook('onRequest', async (req) => {
req.startTime = Date.now()
})

fastify.addHook('preHandler', async (req) => {
const resourceFromParams = Object.values(req.params || {}).join('/')
const resources = getFirstDefined<string[]>(
Expand Down Expand Up @@ -69,7 +74,7 @@ export const logRequest = (options: RequestLoggerOptions) =>
logSchema.request(req.log, buildLogMessage, {
type: 'request',
req,
responseTime: 0,
responseTime: (Date.now() - req.startTime) / 1000,
error: error,
owner: req.owner,
operation: req.operation?.type ?? req.routeConfig.operation?.type,
Expand Down
2 changes: 1 addition & 1 deletion src/http/plugins/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const { storageBackendType } = getConfig()

const storageBackend = createStorageBackend(storageBackendType)

export const storage = fastifyPlugin(async (fastify) => {
export const storage = fastifyPlugin(async function storagePlugin(fastify) {
fastify.decorateRequest('storage', undefined)
fastify.addHook('preHandler', async (request) => {
const database = new StorageKnexDB(request.db, {
Expand Down
11 changes: 10 additions & 1 deletion src/internal/database/multitenant-db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,18 @@ const { multitenantDatabaseUrl } = getConfig()

export const multitenantKnex = Knex({
client: 'pg',
connection: multitenantDatabaseUrl,
connection: {
connectionString: multitenantDatabaseUrl,
connectionTimeoutMillis: 5000,
},
version: '12',
pool: {
min: 0,
max: 10,
createTimeoutMillis: 5000,
acquireTimeoutMillis: 5000,
idleTimeoutMillis: 5000,
reapIntervalMillis: 1000,
createRetryIntervalMillis: 100,
},
})
6 changes: 5 additions & 1 deletion src/internal/monitoring/logger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { normalizeRawError } from '@internal/errors'

const { logLevel, logflareApiKey, logflareSourceToken, logflareEnabled, region } = getConfig()

export const logger = pino({
export const baseLogger = pino({
transport: buildTransport(),
serializers: {
error(error) {
Expand Down Expand Up @@ -40,6 +40,8 @@ export const logger = pino({
timestamp: pino.stdTimeFunctions.isoTime,
})

export const logger = baseLogger.child({ region })

export interface RequestLog {
type: 'request'
req: FastifyRequest
Expand Down Expand Up @@ -77,6 +79,8 @@ interface InfoLog {

export const logSchema = {
info: (logger: BaseLogger, message: string, log: InfoLog) => logger.info(log, message),
warning: (logger: BaseLogger, message: string, log: InfoLog | ErrorLog) =>
logger.warn(log, message),
request: (logger: BaseLogger, message: string, log: RequestLog) => {
if (!log.res) {
logger.warn(log, message)
Expand Down
Empty file added src/internal/queue/database.ts
Empty file.
Loading

0 comments on commit 57eebd6

Please sign in to comment.