From 6fed87363ff9937f790930c44808dd196ab94c96 Mon Sep 17 00:00:00 2001 From: Fabrizio Date: Thu, 16 May 2024 08:13:28 +0100 Subject: [PATCH] feat: operation based logging and RLS (#467) --- migrations/tenant/0024-operation-function.sql | 6 ++ src/database/client.ts | 1 + src/database/connection.ts | 5 +- src/http/plugins/db.ts | 1 + src/http/plugins/log-request.ts | 27 +++++++- src/http/routes/bucket/createBucket.ts | 9 ++- src/http/routes/bucket/deleteBucket.ts | 4 ++ src/http/routes/bucket/emptyBucket.ts | 4 ++ src/http/routes/bucket/getAllBuckets.ts | 4 ++ src/http/routes/bucket/getBucket.ts | 4 ++ src/http/routes/bucket/updateBucket.ts | 4 ++ src/http/routes/object/copyObject.ts | 16 +++-- src/http/routes/object/createObject.ts | 4 ++ src/http/routes/object/deleteObject.ts | 4 ++ src/http/routes/object/deleteObjects.ts | 10 ++- src/http/routes/object/getObject.ts | 7 ++ src/http/routes/object/getObjectInfo.ts | 19 ++++++ src/http/routes/object/getPublicObject.ts | 4 ++ src/http/routes/object/getSignedObject.ts | 4 ++ src/http/routes/object/getSignedURL.ts | 4 ++ src/http/routes/object/getSignedURLs.ts | 10 ++- src/http/routes/object/getSignedUploadURL.ts | 4 ++ src/http/routes/object/listObjects.ts | 4 ++ src/http/routes/object/moveObject.ts | 10 ++- src/http/routes/object/updateObject.ts | 4 ++ src/http/routes/object/uploadSignedObject.ts | 4 ++ src/http/routes/operations.ts | 66 +++++++++++++++++++ .../routes/render/renderAuthenticatedImage.ts | 4 ++ src/http/routes/render/renderPublicImage.ts | 4 ++ src/http/routes/render/renderSignedImage.ts | 4 ++ .../s3/commands/abort-multipart-upload.ts | 21 +++--- .../s3/commands/complete-multipart-upload.ts | 27 ++++---- src/http/routes/s3/commands/copy-object.ts | 45 +++++++------ src/http/routes/s3/commands/create-bucket.ts | 16 +++-- .../s3/commands/create-multipart-upload.ts | 27 ++++---- src/http/routes/s3/commands/delete-bucket.ts | 13 ++-- src/http/routes/s3/commands/delete-object.ts | 41 +++++++----- src/http/routes/s3/commands/get-bucket.ts | 29 +++++--- src/http/routes/s3/commands/get-object.ts | 45 ++++++++----- src/http/routes/s3/commands/head-bucket.ts | 13 ++-- src/http/routes/s3/commands/head-object.ts | 19 ++++-- src/http/routes/s3/commands/list-buckets.ts | 13 ++-- .../s3/commands/list-multipart-uploads.ts | 29 ++++---- src/http/routes/s3/commands/list-objects.ts | 55 +++++++++------- src/http/routes/s3/commands/list-parts.ts | 25 ++++--- .../routes/s3/commands/upload-part-copy.ts | 3 +- src/http/routes/s3/commands/upload-part.ts | 19 ++++-- src/http/routes/s3/index.ts | 2 + src/http/routes/s3/router.ts | 55 ++++++---------- src/http/routes/tus/index.ts | 32 ++++++++- src/http/routes/tus/lifecycle.ts | 7 +- src/monitoring/logger.ts | 3 + src/monitoring/metrics.ts | 2 +- src/queue/events/base-event.ts | 6 +- src/queue/events/object-admin-delete.ts | 1 + src/queue/events/object-created.ts | 1 + src/queue/events/webhook.ts | 2 + src/storage/object.ts | 2 + src/storage/protocols/s3/s3-handler.ts | 4 +- src/storage/uploader.ts | 14 ++-- src/test/tenant.test.ts | 4 +- 61 files changed, 599 insertions(+), 231 deletions(-) create mode 100644 migrations/tenant/0024-operation-function.sql create mode 100644 src/http/routes/operations.ts diff --git a/migrations/tenant/0024-operation-function.sql b/migrations/tenant/0024-operation-function.sql new file mode 100644 index 00000000..e098225c --- /dev/null +++ b/migrations/tenant/0024-operation-function.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE FUNCTION storage.operation() + RETURNS text AS $$ +BEGIN + RETURN current_setting('storage.operation', true); +END; +$$ LANGUAGE plpgsql STABLE; \ No newline at end of file diff --git a/src/database/client.ts b/src/database/client.ts index 71e2d1eb..2be87990 100644 --- a/src/database/client.ts +++ b/src/database/client.ts @@ -12,6 +12,7 @@ interface ConnectionOptions { user: User superUser: User disableHostCheck?: boolean + operation?: string } /** diff --git a/src/database/connection.ts b/src/database/connection.ts index ee638630..9d54e677 100644 --- a/src/database/connection.ts +++ b/src/database/connection.ts @@ -34,6 +34,7 @@ interface TenantConnectionOptions { headers?: Record method?: string path?: string + operation?: string } export interface User { @@ -238,7 +239,8 @@ export class TenantConnection { set_config('request.jwt.claims', ?, true), set_config('request.headers', ?, true), set_config('request.method', ?, true), - set_config('request.path', ?, true); + set_config('request.path', ?, true), + set_config('storage.operation', ?, true); `, [ this.role, @@ -249,6 +251,7 @@ export class TenantConnection { headers, this.options.method || '', this.options.path || '', + this.options.operation || '', ] ) } diff --git a/src/http/plugins/db.ts b/src/http/plugins/db.ts index bad637eb..e2773405 100644 --- a/src/http/plugins/db.ts +++ b/src/http/plugins/db.ts @@ -43,6 +43,7 @@ export const db = fastifyPlugin(async (fastify) => { headers: request.headers, path: request.url, method: request.method, + operation: request.operation?.type, }) }) diff --git a/src/http/plugins/log-request.ts b/src/http/plugins/log-request.ts index dd3ee4d5..cd43af2a 100644 --- a/src/http/plugins/log-request.ts +++ b/src/http/plugins/log-request.ts @@ -8,12 +8,33 @@ interface RequestLoggerOptions { declare module 'fastify' { interface FastifyRequest { executionError?: Error + operation?: { type: string } + resources?: string[] + } + + interface FastifyContextConfig { + operation?: { type: string } + resources?: (req: FastifyRequest) => string[] } } export const logRequest = (options: RequestLoggerOptions) => fastifyPlugin(async (fastify) => { - fastify.addHook('onRequestAbort', (req) => { + fastify.addHook('preHandler', async (req) => { + const resourceFromParams = Object.values(req.params || {}).join('/') + const resources = + req.resources ?? + req.routeConfig.resources?.(req) ?? + (req.raw as any).resources ?? + resourceFromParams + ? ['/' + resourceFromParams] + : ([] as string[]) + + req.resources = resources + req.operation = req.routeConfig.operation + }) + + fastify.addHook('onRequestAbort', async (req) => { if (options.excludeUrls?.includes(req.url)) { return } @@ -34,6 +55,8 @@ export const logRequest = (options: RequestLoggerOptions) => responseTime: 0, error: error, owner: req.owner, + operation: req.operation?.type ?? req.routeConfig.operation?.type, + resources: req.resources, }) }) @@ -60,6 +83,8 @@ export const logRequest = (options: RequestLoggerOptions) => responseTime: reply.getResponseTime(), error: error, owner: req.owner, + resources: req.resources, + operation: req.operation?.type ?? req.routeConfig.operation?.type, }) }) }) diff --git a/src/http/routes/bucket/createBucket.ts b/src/http/routes/bucket/createBucket.ts index 64d45763..6ca57987 100644 --- a/src/http/routes/bucket/createBucket.ts +++ b/src/http/routes/bucket/createBucket.ts @@ -1,7 +1,8 @@ -import { FastifyInstance } from 'fastify' +import { FastifyInstance, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const createBucketBodySchema = { type: 'object', @@ -47,6 +48,12 @@ export default async function routes(fastify: FastifyInstance) { fastify.post( '/', { + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_BUCKET }, + resources: (req: FastifyRequest) => [ + req.body.id || req.body.name || '', + ], + }, schema, }, async (request, response) => { diff --git a/src/http/routes/bucket/deleteBucket.ts b/src/http/routes/bucket/deleteBucket.ts index ac4c76a8..f16645a5 100644 --- a/src/http/routes/bucket/deleteBucket.ts +++ b/src/http/routes/bucket/deleteBucket.ts @@ -2,6 +2,7 @@ import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema, createResponse } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const deleteBucketParamsSchema = { type: 'object', @@ -31,6 +32,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketId', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_BUCKET }, + }, }, async (request, response) => { const { bucketId } = request.params diff --git a/src/http/routes/bucket/emptyBucket.ts b/src/http/routes/bucket/emptyBucket.ts index 3133c782..6801b7b2 100644 --- a/src/http/routes/bucket/emptyBucket.ts +++ b/src/http/routes/bucket/emptyBucket.ts @@ -2,6 +2,7 @@ import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema, createResponse } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const emptyBucketParamsSchema = { type: 'object', @@ -31,6 +32,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketId/empty', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.EMPTY_BUCKET }, + }, }, async (request, response) => { const { bucketId } = request.params diff --git a/src/http/routes/bucket/getAllBuckets.ts b/src/http/routes/bucket/getAllBuckets.ts index 9cb2de01..4ade6fcc 100644 --- a/src/http/routes/bucket/getAllBuckets.ts +++ b/src/http/routes/bucket/getAllBuckets.ts @@ -2,6 +2,7 @@ import { FastifyInstance } from 'fastify' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' import { bucketSchema } from '../../../storage/schemas' +import { ROUTE_OPERATIONS } from '../operations' const successResponseSchema = { type: 'array', @@ -33,6 +34,9 @@ export default async function routes(fastify: FastifyInstance) { '/', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.LIST_BUCKET }, + }, }, async (request, response) => { const results = await request.storage.listBuckets( diff --git a/src/http/routes/bucket/getBucket.ts b/src/http/routes/bucket/getBucket.ts index 2a58abbe..c87edc91 100644 --- a/src/http/routes/bucket/getBucket.ts +++ b/src/http/routes/bucket/getBucket.ts @@ -3,6 +3,7 @@ import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { bucketSchema } from '../../../storage/schemas' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const getBucketParamsSchema = { type: 'object', @@ -28,6 +29,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketId', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.GET_BUCKET }, + }, }, async (request, response) => { const { bucketId } = request.params diff --git a/src/http/routes/bucket/updateBucket.ts b/src/http/routes/bucket/updateBucket.ts index 9bf6e78d..d7dd0694 100644 --- a/src/http/routes/bucket/updateBucket.ts +++ b/src/http/routes/bucket/updateBucket.ts @@ -2,6 +2,7 @@ import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema, createResponse } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const updateBucketBodySchema = { type: 'object', @@ -51,6 +52,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketId', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.UPDATE_BUCKET }, + }, }, async (request, response) => { const { bucketId } = request.params diff --git a/src/http/routes/object/copyObject.ts b/src/http/routes/object/copyObject.ts index b8c3533a..2512939a 100644 --- a/src/http/routes/object/copyObject.ts +++ b/src/http/routes/object/copyObject.ts @@ -1,7 +1,8 @@ -import { FastifyInstance } from 'fastify' +import { FastifyInstance, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const copyRequestBodySchema = { type: 'object', @@ -37,15 +38,16 @@ export default async function routes(fastify: FastifyInstance) { '/copy', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.COPY_OBJECT }, + resources: (req: FastifyRequest) => { + const { sourceKey, destinationKey, bucketId, destinationBucket } = req.body + return [`${bucketId}/${sourceKey}`, `${destinationBucket || bucketId}/${destinationKey}`] + }, + }, }, async (request, response) => { const { sourceKey, destinationKey, bucketId, destinationBucket } = request.body - request.log.info( - 'sourceKey is %s and bucketName is %s and destinationKey is %s', - sourceKey, - bucketId, - destinationKey - ) const destinationBucketId = destinationBucket || bucketId diff --git a/src/http/routes/object/createObject.ts b/src/http/routes/object/createObject.ts index 19d4a5a8..7f5509d1 100644 --- a/src/http/routes/object/createObject.ts +++ b/src/http/routes/object/createObject.ts @@ -1,6 +1,7 @@ import { FastifyInstance, RequestGenericInterface } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' +import { ROUTE_OPERATIONS } from '../operations' const createObjectParamsSchema = { type: 'object', @@ -53,6 +54,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketName/*', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_OBJECT }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/deleteObject.ts b/src/http/routes/object/deleteObject.ts index b17d1472..bedd7bdb 100644 --- a/src/http/routes/object/deleteObject.ts +++ b/src/http/routes/object/deleteObject.ts @@ -2,6 +2,7 @@ import { FastifyInstance } from 'fastify' import { FromSchema, JSONSchema } from 'json-schema-to-ts' import { createDefaultSchema, createResponse } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const deleteObjectParamsSchema = { type: 'object', @@ -34,6 +35,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketName/*', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_OBJECT }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/deleteObjects.ts b/src/http/routes/object/deleteObjects.ts index 14016ee5..ffbeba94 100644 --- a/src/http/routes/object/deleteObjects.ts +++ b/src/http/routes/object/deleteObjects.ts @@ -1,8 +1,9 @@ -import { FastifyInstance } from 'fastify' +import { FastifyInstance, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' import { objectSchema } from '../../../storage/schemas/object' +import { ROUTE_OPERATIONS } from '../operations' const deleteObjectsParamsSchema = { type: 'object', properties: { @@ -45,6 +46,13 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketName', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_OBJECTS }, + resources: (req: FastifyRequest) => { + const { prefixes } = req.body + return prefixes.map((prefix) => `${req.params.bucketName}/${prefix}`) + }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/getObject.ts b/src/http/routes/object/getObject.ts index 511153a3..04724874 100644 --- a/src/http/routes/object/getObject.ts +++ b/src/http/routes/object/getObject.ts @@ -3,6 +3,7 @@ import { FromSchema } from 'json-schema-to-ts' import { IncomingMessage, Server, ServerResponse } from 'http' import { getConfig } from '../../../config' import { AuthenticatedRangeRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -68,6 +69,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['object'], }, + config: { + operation: { type: ROUTE_OPERATIONS.GET_AUTH_OBJECT }, + }, }, async (request, response) => { return requestHandler(request, response) @@ -87,6 +91,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#' } }, tags: ['deprecated'], }, + config: { + operation: { type: ROUTE_OPERATIONS.GET_AUTH_OBJECT }, + }, }, async (request, response) => { return requestHandler(request, response) diff --git a/src/http/routes/object/getObjectInfo.ts b/src/http/routes/object/getObjectInfo.ts index 31c23389..6bf0cb22 100644 --- a/src/http/routes/object/getObjectInfo.ts +++ b/src/http/routes/object/getObjectInfo.ts @@ -4,6 +4,7 @@ import { IncomingMessage, Server, ServerResponse } from 'http' import { getConfig } from '../../../config' import { AuthenticatedRangeRequest } from '../../request' import { Obj } from '../../../storage/schemas' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -64,6 +65,9 @@ export async function publicRoutes(fastify: FastifyInstance) { tags: ['object'], response: { '4xx': { $ref: 'errorSchema#' } }, }, + config: { + operation: { type: ROUTE_OPERATIONS.INFO_PUBLIC_OBJECT }, + }, }, async (request, response) => { return requestHandler(request, response, true) @@ -81,6 +85,9 @@ export async function publicRoutes(fastify: FastifyInstance) { tags: ['object'], response: { '4xx': { $ref: 'errorSchema#' } }, }, + config: { + operation: { type: ROUTE_OPERATIONS.INFO_PUBLIC_OBJECT }, + }, }, async (request, response) => { return requestHandler(request, response, true) @@ -100,6 +107,9 @@ export async function authenticatedRoutes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['object'], }, + config: { + operation: { type: 'object.head_authenticated_info' }, + }, }, async (request, response) => { return requestHandler(request, response) @@ -116,6 +126,9 @@ export async function authenticatedRoutes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['object'], }, + config: { + operation: { type: 'object.get_authenticated_info' }, + }, }, async (request, response) => { return requestHandler(request, response) @@ -133,6 +146,9 @@ export async function authenticatedRoutes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#' } }, tags: ['deprecated'], }, + config: { + operation: { type: 'object.get_authenticated_info' }, + }, }, async (request, response) => { return requestHandler(request, response) @@ -150,6 +166,9 @@ export async function authenticatedRoutes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#' } }, tags: ['deprecated'], }, + config: { + operation: { type: 'object.head_authenticated_info' }, + }, }, async (request, response) => { return requestHandler(request, response) diff --git a/src/http/routes/object/getPublicObject.ts b/src/http/routes/object/getPublicObject.ts index c73b980e..e5611111 100644 --- a/src/http/routes/object/getPublicObject.ts +++ b/src/http/routes/object/getPublicObject.ts @@ -1,6 +1,7 @@ import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { getConfig } from '../../../config' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -41,6 +42,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['object'], }, + config: { + operation: { type: ROUTE_OPERATIONS.GET_PUBLIC_OBJECT }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/getSignedObject.ts b/src/http/routes/object/getSignedObject.ts index 3e22e292..ec9661e7 100644 --- a/src/http/routes/object/getSignedObject.ts +++ b/src/http/routes/object/getSignedObject.ts @@ -4,6 +4,7 @@ import { getConfig } from '../../../config' import { SignedToken, verifyJWT } from '../../../auth' import { ERRORS } from '../../../storage' import { getJwtSecret } from '../../../database/tenant' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -51,6 +52,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['object'], }, + config: { + operation: { type: ROUTE_OPERATIONS.GET_SIGNED_OBJECT }, + }, }, async (request, response) => { const { token } = request.query diff --git a/src/http/routes/object/getSignedURL.ts b/src/http/routes/object/getSignedURL.ts index 8185679d..0342e29e 100644 --- a/src/http/routes/object/getSignedURL.ts +++ b/src/http/routes/object/getSignedURL.ts @@ -5,6 +5,7 @@ import { AuthenticatedRequest } from '../../request' import { ImageRenderer } from '../../../storage/renderer' import { transformationOptionsSchema } from '../../schemas/transformations' import { isImageTransformationEnabled } from '../../../storage/limits' +import { ROUTE_OPERATIONS } from '../operations' const getSignedURLParamsSchema = { type: 'object', @@ -57,6 +58,9 @@ export default async function routes(fastify: FastifyInstance) { '/sign/:bucketName/*', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.SIGN_OBJECT_URL }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/getSignedURLs.ts b/src/http/routes/object/getSignedURLs.ts index b412de91..38ef1808 100644 --- a/src/http/routes/object/getSignedURLs.ts +++ b/src/http/routes/object/getSignedURLs.ts @@ -1,7 +1,8 @@ -import { FastifyInstance } from 'fastify' +import { FastifyInstance, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const getSignedURLsParamsSchema = { type: 'object', @@ -65,6 +66,13 @@ export default async function routes(fastify: FastifyInstance) { '/sign/:bucketName', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.SIGN_OBJECT_URLS }, + resources: (req: FastifyRequest) => { + const { paths } = req.body + return paths.map((path) => `${req.params.bucketName}/${path}`) + }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/getSignedUploadURL.ts b/src/http/routes/object/getSignedUploadURL.ts index a27c2039..b2943109 100644 --- a/src/http/routes/object/getSignedUploadURL.ts +++ b/src/http/routes/object/getSignedUploadURL.ts @@ -3,6 +3,7 @@ import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' import { getConfig } from '../../../config' +import { ROUTE_OPERATIONS } from '../operations' const { uploadSignedUrlExpirationTime } = getConfig() @@ -57,6 +58,9 @@ export default async function routes(fastify: FastifyInstance) { '/upload/sign/:bucketName/*', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.SIGN_UPLOAD_URL }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/listObjects.ts b/src/http/routes/object/listObjects.ts index a1c0a5f7..0f741a78 100644 --- a/src/http/routes/object/listObjects.ts +++ b/src/http/routes/object/listObjects.ts @@ -3,6 +3,7 @@ import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' import { objectSchema } from '../../../storage/schemas' +import { ROUTE_OPERATIONS } from '../operations' const searchRequestParamsSchema = { type: 'object', @@ -54,6 +55,9 @@ export default async function routes(fastify: FastifyInstance) { '/list/:bucketName', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.LIST_OBJECTS }, + }, }, async (request, response) => { const { bucketName } = request.params diff --git a/src/http/routes/object/moveObject.ts b/src/http/routes/object/moveObject.ts index fc45e28b..e260ab2b 100644 --- a/src/http/routes/object/moveObject.ts +++ b/src/http/routes/object/moveObject.ts @@ -1,7 +1,8 @@ -import { FastifyInstance } from 'fastify' +import { FastifyInstance, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema, createResponse } from '../../generic-routes' import { AuthenticatedRequest } from '../../request' +import { ROUTE_OPERATIONS } from '../operations' const moveObjectsBodySchema = { type: 'object', @@ -37,6 +38,13 @@ export default async function routes(fastify: FastifyInstance) { '/move', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.MOVE_OBJECT }, + resources: (req: FastifyRequest) => { + const { sourceKey, destinationKey, bucketId, destinationBucket } = req.body + return [`${bucketId}/${sourceKey}`, `${destinationBucket || bucketId}/${destinationKey}`] + }, + }, }, async (request, response) => { const { destinationKey, sourceKey, bucketId, destinationBucket } = request.body diff --git a/src/http/routes/object/updateObject.ts b/src/http/routes/object/updateObject.ts index 23d3b96b..4678eec7 100644 --- a/src/http/routes/object/updateObject.ts +++ b/src/http/routes/object/updateObject.ts @@ -1,6 +1,7 @@ import { FastifyInstance, RequestGenericInterface } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../generic-routes' +import { ROUTE_OPERATIONS } from '../operations' const updateObjectParamsSchema = { type: 'object', @@ -50,6 +51,9 @@ export default async function routes(fastify: FastifyInstance) { '/:bucketName/*', { schema, + config: { + operation: { type: ROUTE_OPERATIONS.UPDATE_OBJECT }, + }, }, async (request, response) => { const contentType = request.headers['content-type'] diff --git a/src/http/routes/object/uploadSignedObject.ts b/src/http/routes/object/uploadSignedObject.ts index 76532745..b4609734 100644 --- a/src/http/routes/object/uploadSignedObject.ts +++ b/src/http/routes/object/uploadSignedObject.ts @@ -1,5 +1,6 @@ import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' +import { ROUTE_OPERATIONS } from '../operations' const uploadSignedObjectParamsSchema = { type: 'object', @@ -63,6 +64,9 @@ export default async function routes(fastify: FastifyInstance) { }, tags: ['object'], }, + config: { + operation: { type: ROUTE_OPERATIONS.UPLOAD_SIGN_OBJECT }, + }, }, async (request, response) => { // Validate sender diff --git a/src/http/routes/operations.ts b/src/http/routes/operations.ts new file mode 100644 index 00000000..f75dbb41 --- /dev/null +++ b/src/http/routes/operations.ts @@ -0,0 +1,66 @@ +export const ROUTE_OPERATIONS = { + // Bucket + CREATE_BUCKET: 'storage.bucket.create', + DELETE_BUCKET: 'storage.bucket.delete', + EMPTY_BUCKET: 'storage.bucket.empty', + LIST_BUCKET: 'storage.bucket.list', + GET_BUCKET: 'storage.bucket.get', + UPDATE_BUCKET: 'storage.bucket.update', + + // Object + COPY_OBJECT: 'storage.object.copy', + CREATE_OBJECT: 'storage.object.upload', + DELETE_OBJECT: 'storage.object.delete', + DELETE_OBJECTS: 'storage.object.delete_many', + GET_PUBLIC_OBJECT: 'storage.object.get_public', + GET_AUTH_OBJECT: 'storage.object.get_authenticated', + INFO_AUTH_OBJECT: 'storage.object.info_authenticated', + INFO_PUBLIC_OBJECT: 'storage.object.info_public', + GET_SIGNED_OBJECT: 'storage.object.get_signed', + SIGN_UPLOAD_URL: 'storage.object.sign_upload_url', + SIGN_OBJECT_URL: 'storage.object.sign', + SIGN_OBJECT_URLS: 'storage.object.sign_many', + LIST_OBJECTS: 'storage.object.list', + MOVE_OBJECT: 'storage.object.move', + UPDATE_OBJECT: 'storage.object.upload_update', + UPLOAD_SIGN_OBJECT: 'storage.object.upload_signed', + + // Image Transformation + RENDER_AUTH_IMAGE: 'storage.render.image_authenticated', + RENDER_PUBLIC_IMAGE: 'storage.render.image_public', + RENDER_SIGNED_IMAGE: 'storage.render.image_sign', + + // S3 + S3_ABORT_MULTIPART: 'storage.s3.upload.abort_multipart', + S3_COMPLETE_MULTIPART: 'storage.s3.upload.complete_multipart', + S3_CREATE_MULTIPART: 'storage.s3.upload.create_multipart', + S3_LIST_MULTIPART: 'storage.s3.upload.list_multipart', + S3_LIST_PARTS: 'storage.s3.upload.list_parts', + S3_UPLOAD_PART: 'storage.s3.upload.part', + S3_UPLOAD_PART_COPY: 'storage.s3.upload.part_copy', + S3_UPLOAD: 'storage.s3.upload', + + // S3 Object + S3_COPY_OBJECT: 'storage.s3.object.copy', + S3_GET_OBJECT: 'storage.s3.object.get', + S3_LIST_OBJECT: 'storage.s3.object.list', + S3_HEAD_OBJECT: 'storage.s3.object.info', + S3_DELETE_OBJECTS: 'storage.s3.object.delete_many', + S3_GET_OBJECT_TAGGING: 'storage.s3.object.get_tagging', + + // S3 Bucket + S3_CREATE_BUCKET: 'storage.s3.bucket.create', + S3_DELETE_BUCKET: 'storage.s3.bucket.delete', + S3_GET_BUCKET: 'storage.s3.bucket.get', + S3_HEAD_BUCKET: 'storage.s3.bucket.head', + S3_LIST_BUCKET: 'storage.s3.bucket.list', + S3_GET_BUCKET_LOCATION: 'storage.s3.bucket.get_location', + S3_GET_BUCKET_VERSIONING: 'storage.s3.bucket.get_versioning', + + // Tus + TUS_CREATE_UPLOAD: 'storage.tus.upload.create', + TUS_UPLOAD_PART: 'storage.tus.upload.part', + TUS_GET_UPLOAD: 'storage.tus.upload.get', + TUS_DELETE_UPLOAD: 'storage.tus.upload.delete', + TUS_OPTIONS: 'storage.tus.options', +} diff --git a/src/http/routes/render/renderAuthenticatedImage.ts b/src/http/routes/render/renderAuthenticatedImage.ts index 5ac828bb..2ac52dec 100644 --- a/src/http/routes/render/renderAuthenticatedImage.ts +++ b/src/http/routes/render/renderAuthenticatedImage.ts @@ -3,6 +3,7 @@ import { FromSchema } from 'json-schema-to-ts' import { FastifyInstance } from 'fastify' import { ImageRenderer } from '../../../storage/renderer' import { transformationOptionsSchema } from '../../schemas/transformations' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -40,6 +41,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['transformation'], }, + config: { + operation: { type: ROUTE_OPERATIONS.RENDER_SIGNED_IMAGE }, + }, }, async (request, response) => { const { download } = request.query diff --git a/src/http/routes/render/renderPublicImage.ts b/src/http/routes/render/renderPublicImage.ts index 26706c5c..203aed65 100644 --- a/src/http/routes/render/renderPublicImage.ts +++ b/src/http/routes/render/renderPublicImage.ts @@ -3,6 +3,7 @@ import { FromSchema } from 'json-schema-to-ts' import { FastifyInstance } from 'fastify' import { ImageRenderer } from '../../../storage/renderer' import { transformationOptionsSchema } from '../../schemas/transformations' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -40,6 +41,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['transformation'], }, + config: { + operation: { type: ROUTE_OPERATIONS.RENDER_PUBLIC_IMAGE }, + }, }, async (request, response) => { const { download } = request.query diff --git a/src/http/routes/render/renderSignedImage.ts b/src/http/routes/render/renderSignedImage.ts index b18319ae..9f1f2093 100644 --- a/src/http/routes/render/renderSignedImage.ts +++ b/src/http/routes/render/renderSignedImage.ts @@ -5,6 +5,7 @@ import { ImageRenderer } from '../../../storage/renderer' import { SignedToken, verifyJWT } from '../../../auth' import { ERRORS } from '../../../storage' import { getJwtSecret } from '../../../database/tenant' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -48,6 +49,9 @@ export default async function routes(fastify: FastifyInstance) { response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } }, tags: ['transformation'], }, + config: { + operation: { type: ROUTE_OPERATIONS.RENDER_SIGNED_IMAGE }, + }, }, async (request, response) => { const { token } = request.query diff --git a/src/http/routes/s3/commands/abort-multipart-upload.ts b/src/http/routes/s3/commands/abort-multipart-upload.ts index 4eafbedc..3c2610be 100644 --- a/src/http/routes/s3/commands/abort-multipart-upload.ts +++ b/src/http/routes/s3/commands/abort-multipart-upload.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const AbortMultiPartUploadInput = { summary: 'Abort MultiPart Upload', @@ -21,13 +22,17 @@ const AbortMultiPartUploadInput = { } as const export default function AbortMultiPartUpload(s3Router: S3Router) { - s3Router.delete('/:Bucket/*?uploadId', AbortMultiPartUploadInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.delete( + '/:Bucket/*?uploadId', + { schema: AbortMultiPartUploadInput, operation: ROUTE_OPERATIONS.S3_ABORT_MULTIPART }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.abortMultipartUpload({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - UploadId: req.Querystring.uploadId, - }) - }) + return s3Protocol.abortMultipartUpload({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + UploadId: req.Querystring.uploadId, + }) + } + ) } diff --git a/src/http/routes/s3/commands/complete-multipart-upload.ts b/src/http/routes/s3/commands/complete-multipart-upload.ts index 55c938f3..6029e756 100644 --- a/src/http/routes/s3/commands/complete-multipart-upload.ts +++ b/src/http/routes/s3/commands/complete-multipart-upload.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const CompletedMultipartUpload = { summary: 'Complete multipart upload', @@ -51,15 +52,19 @@ const CompletedMultipartUpload = { } as const export default function CompleteMultipartUpload(s3Router: S3Router) { - s3Router.post('/:Bucket/*?uploadId', CompletedMultipartUpload, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.completeMultiPartUpload({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - UploadId: req.Querystring.uploadId, - MultipartUpload: { - Parts: req.Body?.CompleteMultipartUpload?.Parts || [], - }, - }) - }) + s3Router.post( + '/:Bucket/*?uploadId', + { schema: CompletedMultipartUpload, operation: ROUTE_OPERATIONS.S3_COMPLETE_MULTIPART }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + return s3Protocol.completeMultiPartUpload({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + UploadId: req.Querystring.uploadId, + MultipartUpload: { + Parts: req.Body?.CompleteMultipartUpload?.Parts || [], + }, + }) + } + ) } diff --git a/src/http/routes/s3/commands/copy-object.ts b/src/http/routes/s3/commands/copy-object.ts index c3279e04..c98b266d 100644 --- a/src/http/routes/s3/commands/copy-object.ts +++ b/src/http/routes/s3/commands/copy-object.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const CopyObjectInput = { summary: 'Copy Object', @@ -29,25 +30,29 @@ const CopyObjectInput = { } as const export default function CopyObject(s3Router: S3Router) { - s3Router.put('/:Bucket/*|x-amz-copy-source', CopyObjectInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.put( + '/:Bucket/*|x-amz-copy-source', + { schema: CopyObjectInput, operation: ROUTE_OPERATIONS.S3_COPY_OBJECT }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.copyObject({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - CopySource: req.Headers['x-amz-copy-source'], - ContentType: req.Headers['content-type'], - CacheControl: req.Headers['cache-control'], - Expires: req.Headers.expires ? new Date(req.Headers.expires) : undefined, - ContentEncoding: req.Headers['content-encoding'], - CopySourceIfMatch: req.Headers['x-amz-copy-source-if-match'], - CopySourceIfModifiedSince: req.Headers['x-amz-copy-source-if-modified-since'] - ? new Date(req.Headers['x-amz-copy-source-if-modified-since']) - : undefined, - CopySourceIfNoneMatch: req.Headers['x-amz-copy-source-if-none-match'], - CopySourceIfUnmodifiedSince: req.Headers['x-amz-copy-source-if-unmodified-since'] - ? new Date(req.Headers['x-amz-copy-source-if-unmodified-since']) - : undefined, - }) - }) + return s3Protocol.copyObject({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + CopySource: req.Headers['x-amz-copy-source'], + ContentType: req.Headers['content-type'], + CacheControl: req.Headers['cache-control'], + Expires: req.Headers.expires ? new Date(req.Headers.expires) : undefined, + ContentEncoding: req.Headers['content-encoding'], + CopySourceIfMatch: req.Headers['x-amz-copy-source-if-match'], + CopySourceIfModifiedSince: req.Headers['x-amz-copy-source-if-modified-since'] + ? new Date(req.Headers['x-amz-copy-source-if-modified-since']) + : undefined, + CopySourceIfNoneMatch: req.Headers['x-amz-copy-source-if-none-match'], + CopySourceIfUnmodifiedSince: req.Headers['x-amz-copy-source-if-unmodified-since'] + ? new Date(req.Headers['x-amz-copy-source-if-unmodified-since']) + : undefined, + }) + } + ) } diff --git a/src/http/routes/s3/commands/create-bucket.ts b/src/http/routes/s3/commands/create-bucket.ts index 249b6d8e..091b7a06 100644 --- a/src/http/routes/s3/commands/create-bucket.ts +++ b/src/http/routes/s3/commands/create-bucket.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const CreateBucketInput = { summary: 'Create Bucket', @@ -19,9 +20,16 @@ const CreateBucketInput = { } as const export default function CreateBucket(s3Router: S3Router) { - s3Router.put('/:Bucket', CreateBucketInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.put( + '/:Bucket', + { schema: CreateBucketInput, operation: ROUTE_OPERATIONS.S3_CREATE_BUCKET }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.createBucket(req.Params.Bucket, req.Headers?.['x-amz-acl'] === 'public-read') - }) + return s3Protocol.createBucket( + req.Params.Bucket, + req.Headers?.['x-amz-acl'] === 'public-read' + ) + } + ) } diff --git a/src/http/routes/s3/commands/create-multipart-upload.ts b/src/http/routes/s3/commands/create-multipart-upload.ts index fe1f8fac..2c4c091d 100644 --- a/src/http/routes/s3/commands/create-multipart-upload.ts +++ b/src/http/routes/s3/commands/create-multipart-upload.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const CreateMultiPartUploadInput = { summary: 'Create multipart upload', @@ -32,16 +33,20 @@ const CreateMultiPartUploadInput = { } as const export default function CreateMultipartUpload(s3Router: S3Router) { - s3Router.post('/:Bucket/*?uploads', CreateMultiPartUploadInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.post( + '/:Bucket/*?uploads', + { schema: CreateMultiPartUploadInput, operation: ROUTE_OPERATIONS.S3_CREATE_MULTIPART }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.createMultiPartUpload({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - ContentType: req.Headers?.['content-type'], - CacheControl: req.Headers?.['cache-control'], - ContentDisposition: req.Headers?.['content-disposition'], - ContentEncoding: req.Headers?.['content-encoding'], - }) - }) + return s3Protocol.createMultiPartUpload({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + ContentType: req.Headers?.['content-type'], + CacheControl: req.Headers?.['cache-control'], + ContentDisposition: req.Headers?.['content-disposition'], + ContentEncoding: req.Headers?.['content-encoding'], + }) + } + ) } diff --git a/src/http/routes/s3/commands/delete-bucket.ts b/src/http/routes/s3/commands/delete-bucket.ts index 207f4b1d..6b519318 100644 --- a/src/http/routes/s3/commands/delete-bucket.ts +++ b/src/http/routes/s3/commands/delete-bucket.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const DeleteBucketInput = { summary: 'Delete Bucket', @@ -13,9 +14,13 @@ const DeleteBucketInput = { } as const export default function DeleteBucket(s3Router: S3Router) { - s3Router.delete('/:Bucket', DeleteBucketInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.delete( + '/:Bucket', + { schema: DeleteBucketInput, operation: ROUTE_OPERATIONS.S3_DELETE_BUCKET }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.deleteBucket(req.Params.Bucket) - }) + return s3Protocol.deleteBucket(req.Params.Bucket) + } + ) } diff --git a/src/http/routes/s3/commands/delete-object.ts b/src/http/routes/s3/commands/delete-object.ts index 8843355b..55b2544b 100644 --- a/src/http/routes/s3/commands/delete-object.ts +++ b/src/http/routes/s3/commands/delete-object.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const DeleteObjectInput = { summary: 'Delete Object', @@ -56,24 +57,32 @@ const DeleteObjectsInput = { export default function DeleteObject(s3Router: S3Router) { // Delete multiple objects - s3Router.post('/:Bucket?delete', DeleteObjectsInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.post( + '/:Bucket?delete', + { schema: DeleteObjectsInput, operation: ROUTE_OPERATIONS.S3_DELETE_OBJECTS }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.deleteObjects({ - Bucket: req.Params.Bucket, - Delete: { - Objects: req.Body.Delete.Object, - }, - }) - }) + return s3Protocol.deleteObjects({ + Bucket: req.Params.Bucket, + Delete: { + Objects: req.Body.Delete.Object, + }, + }) + } + ) // Delete single object - s3Router.delete('/:Bucket/*', DeleteObjectInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.delete( + '/:Bucket/*', + { schema: DeleteObjectInput, operation: 's3.object.delete' }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.deleteObject({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - }) - }) + return s3Protocol.deleteObject({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + }) + } + ) } diff --git a/src/http/routes/s3/commands/get-bucket.ts b/src/http/routes/s3/commands/get-bucket.ts index ff39f41c..d96746b0 100644 --- a/src/http/routes/s3/commands/get-bucket.ts +++ b/src/http/routes/s3/commands/get-bucket.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const GetBucketLocationInput = { Params: { @@ -36,17 +37,25 @@ const GetBucketVersioningInput = { } as const export default function GetBucket(s3Router: S3Router) { - s3Router.get('/:Bucket?location', GetBucketLocationInput, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - await ctx.storage.findBucket(req.Params.Bucket) + s3Router.get( + '/:Bucket?location', + { schema: GetBucketLocationInput, operation: ROUTE_OPERATIONS.S3_GET_BUCKET_LOCATION }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + await ctx.storage.findBucket(req.Params.Bucket) - return s3Protocol.getBucketLocation() - }) + return s3Protocol.getBucketLocation() + } + ) - s3Router.get('/:Bucket?versioning', GetBucketVersioningInput, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - await ctx.storage.findBucket(req.Params.Bucket) + s3Router.get( + '/:Bucket?versioning', + { schema: GetBucketVersioningInput, operation: ROUTE_OPERATIONS.S3_GET_BUCKET_VERSIONING }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + await ctx.storage.findBucket(req.Params.Bucket) - return s3Protocol.getBucketVersioning() - }) + return s3Protocol.getBucketVersioning() + } + ) } diff --git a/src/http/routes/s3/commands/get-object.ts b/src/http/routes/s3/commands/get-object.ts index b41d6699..243bb1a2 100644 --- a/src/http/routes/s3/commands/get-object.ts +++ b/src/http/routes/s3/commands/get-object.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const GetObjectInput = { summary: 'Get Object', @@ -42,25 +43,33 @@ const GetObjectTagging = { } as const export default function GetObject(s3Router: S3Router) { - s3Router.get('/:Bucket/*?tagging', GetObjectTagging, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.get( + '/:Bucket/*?tagging', + { schema: GetObjectTagging, operation: ROUTE_OPERATIONS.S3_GET_OBJECT_TAGGING }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.getObjectTagging({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - }) - }) + return s3Protocol.getObjectTagging({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + }) + } + ) - s3Router.get('/:Bucket/*', GetObjectInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - const ifModifiedSince = req.Headers?.['if-modified-since'] + s3Router.get( + '/:Bucket/*', + { schema: GetObjectInput, operation: ROUTE_OPERATIONS.S3_GET_OBJECT }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + const ifModifiedSince = req.Headers?.['if-modified-since'] - return s3Protocol.getObject({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - Range: req.Headers?.['range'], - IfNoneMatch: req.Headers?.['if-none-match'], - IfModifiedSince: ifModifiedSince ? new Date(ifModifiedSince) : undefined, - }) - }) + return s3Protocol.getObject({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + Range: req.Headers?.['range'], + IfNoneMatch: req.Headers?.['if-none-match'], + IfModifiedSince: ifModifiedSince ? new Date(ifModifiedSince) : undefined, + }) + } + ) } diff --git a/src/http/routes/s3/commands/head-bucket.ts b/src/http/routes/s3/commands/head-bucket.ts index dab350cb..b008faa5 100644 --- a/src/http/routes/s3/commands/head-bucket.ts +++ b/src/http/routes/s3/commands/head-bucket.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const HeadBucketInput = { Params: { @@ -12,9 +13,13 @@ const HeadBucketInput = { } as const export default function HeadBucket(s3Router: S3Router) { - s3Router.head('/:Bucket', HeadBucketInput, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.head( + '/:Bucket', + { schema: HeadBucketInput, operation: ROUTE_OPERATIONS.S3_HEAD_BUCKET }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.headBucket(req.Params.Bucket) - }) + return s3Protocol.headBucket(req.Params.Bucket) + } + ) } diff --git a/src/http/routes/s3/commands/head-object.ts b/src/http/routes/s3/commands/head-object.ts index e10b7051..e2c912db 100644 --- a/src/http/routes/s3/commands/head-object.ts +++ b/src/http/routes/s3/commands/head-object.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const HeadObjectInput = { summary: 'Head Object', @@ -14,12 +15,16 @@ const HeadObjectInput = { } as const export default function HeadObject(s3Router: S3Router) { - s3Router.head('/:Bucket/*', HeadObjectInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.head( + '/:Bucket/*', + { schema: HeadObjectInput, operation: ROUTE_OPERATIONS.S3_HEAD_OBJECT }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.headObject({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - }) - }) + return s3Protocol.headObject({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + }) + } + ) } diff --git a/src/http/routes/s3/commands/list-buckets.ts b/src/http/routes/s3/commands/list-buckets.ts index 5e4c7852..f86567b3 100644 --- a/src/http/routes/s3/commands/list-buckets.ts +++ b/src/http/routes/s3/commands/list-buckets.ts @@ -1,13 +1,18 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const ListObjectsInput = { summary: 'List buckets', } as const export default function ListBuckets(s3Router: S3Router) { - s3Router.get('/', ListObjectsInput, (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.listBuckets() - }) + s3Router.get( + '/', + { schema: ListObjectsInput, operation: ROUTE_OPERATIONS.S3_LIST_BUCKET }, + (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + return s3Protocol.listBuckets() + } + ) } diff --git a/src/http/routes/s3/commands/list-multipart-uploads.ts b/src/http/routes/s3/commands/list-multipart-uploads.ts index 4b6a8a15..dda47be6 100644 --- a/src/http/routes/s3/commands/list-multipart-uploads.ts +++ b/src/http/routes/s3/commands/list-multipart-uploads.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const ListObjectsInput = { summary: 'List Objects', @@ -26,17 +27,21 @@ const ListObjectsInput = { } as const export default function ListMultipartUploads(s3Router: S3Router) { - s3Router.get('/:Bucket?uploads', ListObjectsInput, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.get( + '/:Bucket?uploads', + { schema: ListObjectsInput, operation: ROUTE_OPERATIONS.S3_LIST_MULTIPART }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.listMultipartUploads({ - Bucket: req.Params.Bucket, - Prefix: req.Querystring?.prefix || '', - KeyMarker: req.Querystring?.['key-marker'], - UploadIdMarker: req.Querystring?.['upload-id-marker'], - EncodingType: req.Querystring?.['encoding-type'], - MaxUploads: req.Querystring?.['max-uploads'], - Delimiter: req.Querystring?.delimiter, - }) - }) + return s3Protocol.listMultipartUploads({ + Bucket: req.Params.Bucket, + Prefix: req.Querystring?.prefix || '', + KeyMarker: req.Querystring?.['key-marker'], + UploadIdMarker: req.Querystring?.['upload-id-marker'], + EncodingType: req.Querystring?.['encoding-type'], + MaxUploads: req.Querystring?.['max-uploads'], + Delimiter: req.Querystring?.delimiter, + }) + } + ) } diff --git a/src/http/routes/s3/commands/list-objects.ts b/src/http/routes/s3/commands/list-objects.ts index 6d76fec6..677631ad 100644 --- a/src/http/routes/s3/commands/list-objects.ts +++ b/src/http/routes/s3/commands/list-objects.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const ListObjectsV2Input = { summary: 'List Objects V2', @@ -47,30 +48,38 @@ const ListObjectsInput = { } as const export default function ListObjects(s3Router: S3Router) { - s3Router.get('/:Bucket?list-type=2', ListObjectsV2Input, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.get( + '/:Bucket?list-type=2', + { schema: ListObjectsV2Input, operation: ROUTE_OPERATIONS.S3_LIST_OBJECT }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.listObjectsV2({ - Bucket: req.Params.Bucket, - Prefix: req.Querystring?.prefix || '', - ContinuationToken: req.Querystring?.['continuation-token'], - StartAfter: req.Querystring?.['start-after'], - EncodingType: req.Querystring?.['encoding-type'], - MaxKeys: req.Querystring?.['max-keys'], - Delimiter: req.Querystring?.delimiter, - }) - }) + return s3Protocol.listObjectsV2({ + Bucket: req.Params.Bucket, + Prefix: req.Querystring?.prefix || '', + ContinuationToken: req.Querystring?.['continuation-token'], + StartAfter: req.Querystring?.['start-after'], + EncodingType: req.Querystring?.['encoding-type'], + MaxKeys: req.Querystring?.['max-keys'], + Delimiter: req.Querystring?.delimiter, + }) + } + ) - s3Router.get('/:Bucket', ListObjectsInput, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.get( + '/:Bucket', + { schema: ListObjectsInput, operation: ROUTE_OPERATIONS.S3_LIST_OBJECT }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.listObjects({ - Bucket: req.Params.Bucket, - Prefix: req.Querystring?.prefix || '', - Marker: req.Querystring?.['marker'], - EncodingType: req.Querystring?.['encoding-type'], - MaxKeys: req.Querystring?.['max-keys'], - Delimiter: req.Querystring?.delimiter, - }) - }) + return s3Protocol.listObjects({ + Bucket: req.Params.Bucket, + Prefix: req.Querystring?.prefix || '', + Marker: req.Querystring?.['marker'], + EncodingType: req.Querystring?.['encoding-type'], + MaxKeys: req.Querystring?.['max-keys'], + Delimiter: req.Querystring?.delimiter, + }) + } + ) } diff --git a/src/http/routes/s3/commands/list-parts.ts b/src/http/routes/s3/commands/list-parts.ts index eb6835b3..d7117955 100644 --- a/src/http/routes/s3/commands/list-parts.ts +++ b/src/http/routes/s3/commands/list-parts.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const ListPartsInput = { summary: 'List Parts', @@ -23,15 +24,19 @@ const ListPartsInput = { } as const export default function ListParts(s3Router: S3Router) { - s3Router.get('/:Bucket/*?uploadId', ListPartsInput, async (req, ctx) => { - const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + s3Router.get( + '/:Bucket/*?uploadId', + { schema: ListPartsInput, operation: ROUTE_OPERATIONS.S3_LIST_PARTS }, + async (req, ctx) => { + const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) - return s3Protocol.listParts({ - Bucket: req.Params.Bucket, - Key: req.Params['*'], - UploadId: req.Querystring.uploadId, - MaxParts: req.Querystring['max-parts'], - PartNumberMarker: req.Querystring['part-number-marker'], - }) - }) + return s3Protocol.listParts({ + Bucket: req.Params.Bucket, + Key: req.Params['*'], + UploadId: req.Querystring.uploadId, + MaxParts: req.Querystring['max-parts'], + PartNumberMarker: req.Querystring['part-number-marker'], + }) + } + ) } diff --git a/src/http/routes/s3/commands/upload-part-copy.ts b/src/http/routes/s3/commands/upload-part-copy.ts index 266d89c5..c2b2ef68 100644 --- a/src/http/routes/s3/commands/upload-part-copy.ts +++ b/src/http/routes/s3/commands/upload-part-copy.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const UploadPartCopyInput = { summary: 'Upload Part Copy', @@ -37,7 +38,7 @@ const UploadPartCopyInput = { export default function UploadPartCopy(s3Router: S3Router) { s3Router.put( '/:Bucket/*?partNumber&uploadId|x-amz-copy-source', - UploadPartCopyInput, + { schema: UploadPartCopyInput, operation: ROUTE_OPERATIONS.S3_UPLOAD_PART_COPY }, (req, ctx) => { const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) diff --git a/src/http/routes/s3/commands/upload-part.ts b/src/http/routes/s3/commands/upload-part.ts index c06e7dde..41c58115 100644 --- a/src/http/routes/s3/commands/upload-part.ts +++ b/src/http/routes/s3/commands/upload-part.ts @@ -1,5 +1,6 @@ import { S3ProtocolHandler } from '../../../../storage/protocols/s3/s3-handler' import { S3Router } from '../router' +import { ROUTE_OPERATIONS } from '../../operations' const PutObjectInput = { summary: 'Put Object', @@ -65,7 +66,11 @@ const UploadPartInput = { export default function UploadPart(s3Router: S3Router) { s3Router.put( '/:Bucket/*?uploadId&partNumber', - UploadPartInput, + { + schema: UploadPartInput, + operation: ROUTE_OPERATIONS.S3_UPLOAD_PART, + disableContentTypeParser: true, + }, (req, ctx) => { const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) @@ -77,13 +82,16 @@ export default function UploadPart(s3Router: S3Router) { PartNumber: req.Querystring?.partNumber, ContentLength: req.Headers?.['content-length'], }) - }, - { disableContentTypeParser: true } + } ) s3Router.put( '/:Bucket/*', - PutObjectInput, + { + schema: PutObjectInput, + operation: ROUTE_OPERATIONS.S3_UPLOAD, + disableContentTypeParser: true, + }, (req, ctx) => { const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) return s3Protocol.putObject({ @@ -95,7 +103,6 @@ export default function UploadPart(s3Router: S3Router) { Expires: req.Headers?.['expires'] ? new Date(req.Headers?.['expires']) : undefined, ContentEncoding: req.Headers?.['content-encoding'], }) - }, - { disableContentTypeParser: true } + } ) } diff --git a/src/http/routes/s3/index.ts b/src/http/routes/s3/index.ts index a5a0077e..265d5461 100644 --- a/src/http/routes/s3/index.ts +++ b/src/http/routes/s3/index.ts @@ -33,6 +33,8 @@ export default async function routes(fastify: FastifyInstance) { throw new Error('no handler found') } + req.operation = { type: route.operation } + const data: RequestInput = { Params: req.params, Body: req.body, diff --git a/src/http/routes/s3/router.ts b/src/http/routes/s3/router.ts index 0a518f64..f38e488e 100644 --- a/src/http/routes/s3/router.ts +++ b/src/http/routes/s3/router.ts @@ -99,11 +99,14 @@ type Route = { handler?: Handler schema: S disableContentTypeParser?: boolean + operation: string compiledSchema: () => ValidateFunction> } -interface RouteOptions { +interface RouteOptions { disableContentTypeParser?: boolean + operation: string + schema: S } export class Router { @@ -121,9 +124,8 @@ export class Router { registerRoute( method: HTTPMethod, url: string, - schema: R, - handler: Handler, - options?: { disableContentTypeParser?: boolean } + options: RouteOptions, + handler: Handler ) { const { query, headers } = this.parseQueryString(url) const normalizedUrl = url.split('?')[0].split('|')[0] @@ -136,6 +138,8 @@ export class Router { Body?: JSONSchema } = {} + const { schema, disableContentTypeParser, operation } = options + if (schema.Params) { schemaToCompile.Params = schema.Params } @@ -166,7 +170,8 @@ export class Router { schema: schema, compiledSchema: () => this.ajv.getSchema(method + url) as ValidateFunction>, handler: handler as Handler, - disableContentTypeParser: options?.disableContentTypeParser, + disableContentTypeParser: disableContentTypeParser, + operation, } as const if (!existingPath) { @@ -178,44 +183,24 @@ export class Router { this._routes.set(normalizedUrl, existingPath) } - get(url: string, schema: R, handler: Handler, options?: RouteOptions) { - this.registerRoute('get', url, schema, handler as any, options) + get(url: string, options: RouteOptions, handler: Handler) { + this.registerRoute('get', url, options, handler as any) } - post( - url: string, - schema: R, - handler: Handler, - options?: RouteOptions - ) { - this.registerRoute('post', url, schema, handler as any, options) + post(url: string, options: RouteOptions, handler: Handler) { + this.registerRoute('post', url, options, handler as any) } - put( - url: string, - schema: R, - handler: Handler, - options?: RouteOptions - ) { - this.registerRoute('put', url, schema, handler as any, options) + put(url: string, options: RouteOptions, handler: Handler) { + this.registerRoute('put', url, options, handler as any) } - delete( - url: string, - schema: R, - handler: Handler, - options?: RouteOptions - ) { - this.registerRoute('delete', url, schema, handler as any, options) + delete(url: string, options: RouteOptions, handler: Handler) { + this.registerRoute('delete', url, options, handler as any) } - head( - url: string, - schema: R, - handler: Handler, - options?: RouteOptions - ) { - this.registerRoute('head', url, schema, handler as any, options) + head(url: string, options: RouteOptions, handler: Handler) { + this.registerRoute('head', url, options, handler as any) } parseQueryMatch(query: string) { diff --git a/src/http/routes/tus/index.ts b/src/http/routes/tus/index.ts index 88511fd3..2ec6470f 100644 --- a/src/http/routes/tus/index.ts +++ b/src/http/routes/tus/index.ts @@ -28,6 +28,7 @@ import { TenantConnection, PubSub } from '../../../database' import { S3Store } from '@tus/s3-store' import { NodeHttpHandler } from '@smithy/node-http-handler' import { createAgent } from '../../../storage/backend' +import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket, @@ -49,6 +50,7 @@ type MultiPartRequest = http.IncomingMessage & { tenantId: string db: TenantConnection isUpsert: boolean + resources?: string[] } } @@ -149,6 +151,7 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(authenticatedRoutes, { tusServer, + operation: '_signed', }) }, { prefix: SIGNED_URL_SUFFIX } @@ -172,6 +175,7 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(publicRoutes, { tusServer, + operation: '_signed', }) }, { prefix: SIGNED_URL_SUFFIX } @@ -179,7 +183,7 @@ export default async function routes(fastify: FastifyInstance) { } const authenticatedRoutes = fastifyPlugin( - async (fastify: FastifyInstance, options: { tusServer: TusServer }) => { + async (fastify: FastifyInstance, options: { tusServer: TusServer; operation?: string }) => { fastify.register(async function authorizationContext(fastify) { fastify.addContentTypeParser('application/offset+octet-stream', (request, payload, done) => done(null) @@ -206,6 +210,9 @@ const authenticatedRoutes = fastifyPlugin( '/', { schema: { summary: 'Handle POST request for TUS Resumable uploads', tags: ['resumable'] }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_CREATE_UPLOAD}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -216,6 +223,9 @@ const authenticatedRoutes = fastifyPlugin( '/*', { schema: { summary: 'Handle POST request for TUS Resumable uploads', tags: ['resumable'] }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_CREATE_UPLOAD}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -226,6 +236,9 @@ const authenticatedRoutes = fastifyPlugin( '/*', { schema: { summary: 'Handle PUT request for TUS Resumable uploads', tags: ['resumable'] }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_UPLOAD_PART}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -238,6 +251,9 @@ const authenticatedRoutes = fastifyPlugin( summary: 'Handle PATCH request for TUS Resumable uploads', tags: ['resumable'], }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_UPLOAD_PART}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -247,6 +263,9 @@ const authenticatedRoutes = fastifyPlugin( '/*', { schema: { summary: 'Handle HEAD request for TUS Resumable uploads', tags: ['resumable'] }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_GET_UPLOAD}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -259,6 +278,9 @@ const authenticatedRoutes = fastifyPlugin( summary: 'Handle DELETE request for TUS Resumable uploads', tags: ['resumable'], }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_DELETE_UPLOAD}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -269,7 +291,7 @@ const authenticatedRoutes = fastifyPlugin( ) const publicRoutes = fastifyPlugin( - async (fastify: FastifyInstance, options: { tusServer: TusServer }) => { + async (fastify: FastifyInstance, options: { tusServer: TusServer; operation?: string }) => { fastify.register(async (fastify) => { fastify.addContentTypeParser('application/offset+octet-stream', (request, payload, done) => done(null) @@ -294,6 +316,9 @@ const publicRoutes = fastifyPlugin( summary: 'Handle OPTIONS request for TUS Resumable uploads', description: 'Handle OPTIONS request for TUS Resumable uploads', }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_OPTIONS}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) @@ -308,6 +333,9 @@ const publicRoutes = fastifyPlugin( summary: 'Handle OPTIONS request for TUS Resumable uploads', description: 'Handle OPTIONS request for TUS Resumable uploads', }, + config: { + operation: { type: `${ROUTE_OPERATIONS.TUS_OPTIONS}${options.operation || ''}` }, + }, }, (req, res) => { options.tusServer.handle(req.raw, res.raw) diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index ad116a7c..2b1d3674 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -21,6 +21,7 @@ export type MultiPartRequest = http.IncomingMessage & { owner?: string tenantId: string isUpsert: boolean + resources?: string[] } } @@ -42,6 +43,8 @@ export async function onIncomingRequest( const uploadID = UploadId.fromString(id) + req.upload.resources = [`${uploadID.bucket}/${uploadID.objectName}`] + // Handle signed url requests if (req.url?.startsWith(`/upload/resumable/sign`)) { const signature = req.headers['x-signature'] @@ -201,10 +204,12 @@ export async function onUploadFinish( objectName: resourceId.objectName, objectMetadata: metadata, isUpsert: req.upload.isUpsert, - isMultipart: true, + uploadType: 'resumable', owner: req.upload.owner, }) + res.setHeader('Tus-Complete', '1') + return res } catch (e) { if (isRenderableError(e)) { diff --git a/src/monitoring/logger.ts b/src/monitoring/logger.ts index 29a054e8..63ed422c 100644 --- a/src/monitoring/logger.ts +++ b/src/monitoring/logger.ts @@ -42,6 +42,8 @@ export interface RequestLog { responseTime: number error?: Error | unknown owner?: string + operation?: string + resources?: string[] } export interface EventLog { @@ -52,6 +54,7 @@ export interface EventLog { objectPath: string tenantId: string project: string + resources?: string[] reqId?: string } diff --git a/src/monitoring/metrics.ts b/src/monitoring/metrics.ts index 52f87fa8..4aecf22a 100644 --- a/src/monitoring/metrics.ts +++ b/src/monitoring/metrics.ts @@ -12,7 +12,7 @@ export const FileUploadStarted = new client.Gauge({ export const FileUploadedSuccess = new client.Gauge({ name: 'storage_api_upload_success', help: 'Successful uploads', - labelNames: ['region', 'is_multipart'], + labelNames: ['region', 'is_multipart', 'is_resumable', 'is_standard', 'is_s3'], }) export const DbQueryPerformance = new client.Histogram({ diff --git a/src/queue/events/base-event.ts b/src/queue/events/base-event.ts index 5811c85b..aaa26358 100644 --- a/src/queue/events/base-event.ts +++ b/src/queue/events/base-event.ts @@ -23,7 +23,7 @@ export interface SlowRetryQueueOptions { retryDelay: number } -const { pgQueueEnable, storageBackendType, storageS3Endpoint } = getConfig() +const { pgQueueEnable, storageBackendType, storageS3Endpoint, region } = getConfig() const storageS3Protocol = storageS3Endpoint?.includes('http://') ? 'http' : 'https' const httpAgent = createAgent(storageS3Protocol) @@ -131,6 +131,7 @@ export abstract class BaseEvent> { await Webhook.send({ event: { type: eventType, + region, $version: this.version, applyTime: Date.now(), payload, @@ -189,6 +190,7 @@ export abstract class BaseEvent> { id: '', name: constructor.getQueueName(), data: { + region, ...this.payload, $version: constructor.version, }, @@ -201,6 +203,7 @@ export abstract class BaseEvent> { const res = await Queue.getInstance().send({ name: constructor.getQueueName(), data: { + region, ...this.payload, $version: constructor.version, }, @@ -232,6 +235,7 @@ export abstract class BaseEvent> { const res = await Queue.getInstance().send({ name: constructor.getSlowRetryQueueName(), data: { + region, ...this.payload, $version: constructor.version, }, diff --git a/src/queue/events/object-admin-delete.ts b/src/queue/events/object-admin-delete.ts index c885cbc7..57089212 100644 --- a/src/queue/events/object-admin-delete.ts +++ b/src/queue/events/object-admin-delete.ts @@ -38,6 +38,7 @@ export class ObjectAdminDelete extends BaseEvent { event: 'ObjectAdminDelete', payload: JSON.stringify(job.data), objectPath: s3Key, + resources: [`${job.data.bucketId}/${job.data.name}`], tenantId: job.data.tenant.ref, project: job.data.tenant.ref, reqId: job.data.reqId, diff --git a/src/queue/events/object-created.ts b/src/queue/events/object-created.ts index a344a6b5..224501e6 100644 --- a/src/queue/events/object-created.ts +++ b/src/queue/events/object-created.ts @@ -6,6 +6,7 @@ interface ObjectCreatedEvent extends BasePayload { name: string bucketId: string metadata: ObjectMetadata + uploadType: 'standard' | 'resumable' | 's3' } abstract class ObjectCreated extends BaseEvent { diff --git a/src/queue/events/webhook.ts b/src/queue/events/webhook.ts index ab53f898..6d990048 100644 --- a/src/queue/events/webhook.ts +++ b/src/queue/events/webhook.ts @@ -52,6 +52,7 @@ export class Webhook extends BaseEvent { event: job.data.event.type, payload: JSON.stringify(job.data.event.payload), objectPath: path, + resources: [path], tenantId: job.data.tenant.ref, project: job.data.tenant.ref, reqId: job.data.event.payload.reqId, @@ -81,6 +82,7 @@ export class Webhook extends BaseEvent { event: job.data.event.type, payload: JSON.stringify(job.data.event.payload), objectPath: path, + resources: [path], tenantId: job.data.tenant.ref, project: job.data.tenant.ref, reqId: job.data.event.payload.reqId, diff --git a/src/storage/object.ts b/src/storage/object.ts index 00c4da90..0548d4dc 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -68,6 +68,7 @@ export class ObjectStorage { bucketId: this.bucketId, fileSizeLimit: bucket.file_size_limit, allowedMimeTypes: bucket.allowed_mime_types, + uploadType: 'standard', }) return { objectMetadata: metadata, path, id: obj.id } @@ -96,6 +97,7 @@ export class ObjectStorage { fileSizeLimit: bucket.file_size_limit, allowedMimeTypes: bucket.allowed_mime_types, isUpsert: true, + uploadType: 'standard', }) return { objectMetadata: metadata, path, id: obj.id } diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index 2872e0d6..4a5d6b08 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -539,7 +539,7 @@ export class S3ProtocolHandler { objectName: Key as string, version: resp.version, isUpsert: true, - isMultipart: false, + uploadType: 's3', objectMetadata: metadata, owner: this.owner, }) @@ -689,7 +689,7 @@ export class S3ProtocolHandler { objectName: command.Key as string, owner: this.owner, isUpsert: true, - isMultipart: false, + uploadType: 's3', fileSizeLimit: bucket.file_size_limit, allowedMimeTypes: bucket.allowed_mime_types, }) diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index 9979598e..2f9c7d86 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -20,7 +20,7 @@ export interface UploadObjectOptions { objectName: string owner?: string isUpsert?: boolean - isMultipart?: boolean + uploadType?: 'standard' | 's3' | 'resumable' } /** @@ -64,7 +64,7 @@ export class Uploader { async prepareUpload(options: UploadObjectOptions) { await this.canUpload(options) FileUploadStarted.inc({ - is_multipart: Boolean(options.isMultipart).toString(), + is_multipart: Boolean(options.uploadType).toString(), }) return randomUUID() @@ -125,13 +125,13 @@ export class Uploader { objectName, owner, objectMetadata, - isMultipart, + uploadType, isUpsert, }: UploadObjectOptions & { objectMetadata: ObjectMetadata version: string emitEvent?: boolean - isMultipart?: boolean + uploadType?: 'standard' | 's3' | 'resumable' }) { try { return await this.db.withTransaction(async (db) => { @@ -179,13 +179,17 @@ export class Uploader { bucketId: bucketId, metadata: objectMetadata, reqId: this.db.reqId, + uploadType, }) ) await Promise.all(events) FileUploadedSuccess.inc({ - is_multipart: Boolean(isMultipart).toString(), + is_multipart: uploadType === 'resumable' ? 1 : 0, + is_resumable: uploadType === 'resumable' ? 1 : 0, + is_standard: uploadType === 'standard' ? 1 : 0, + is_s3: uploadType === 's3' ? 1 : 0, }) return { obj: newObject, isNew, metadata: objectMetadata } diff --git a/src/test/tenant.test.ts b/src/test/tenant.test.ts index 5a6db7d6..98dd08f4 100644 --- a/src/test/tenant.test.ts +++ b/src/test/tenant.test.ts @@ -16,7 +16,7 @@ const payload = { serviceKey: 'd', jwks: { keys: [] }, migrationStatus: 'COMPLETED', - migrationVersion: 'optimize-search-function', + migrationVersion: 'operation-function', features: { imageTransformation: { enabled: true, @@ -34,7 +34,7 @@ const payload2 = { serviceKey: 'h', jwks: null, migrationStatus: 'COMPLETED', - migrationVersion: 'optimize-search-function', + migrationVersion: 'operation-function', features: { imageTransformation: { enabled: false,