diff --git a/migrations/tenant/0025-custom-metadata.sql b/migrations/tenant/0025-custom-metadata.sql new file mode 100644 index 00000000..b18d92f6 --- /dev/null +++ b/migrations/tenant/0025-custom-metadata.sql @@ -0,0 +1,2 @@ +ALTER TABLE storage.objects ADD COLUMN user_metadata jsonb NULL; +ALTER TABLE storage.s3_multipart_uploads ADD COLUMN user_metadata jsonb NULL; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 392f8ea9..d61ff409 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,9 +28,9 @@ "@opentelemetry/instrumentation-pino": "^0.39.0", "@shopify/semaphore": "^3.0.2", "@smithy/node-http-handler": "^2.3.1", - "@tus/file-store": "1.3.1", - "@tus/s3-store": "1.4.1", - "@tus/server": "1.4.1", + "@tus/file-store": "1.4.0", + "@tus/s3-store": "1.5.0", + "@tus/server": "1.7.0", "agentkeepalive": "^4.5.0", "ajv": "^8.12.0", "async-retry": "^1.3.3", @@ -5012,11 +5012,11 @@ "peer": true }, "node_modules/@tus/file-store": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-1.3.1.tgz", - "integrity": "sha512-OuqyD4gRSBew7iLEaf9Slo25AU8NH13NmEkCbEhV4r/mVN8Yyz3seuPmYOT21gkCHDcRmdR+yzaX+5JLzSurtA==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-1.4.0.tgz", + "integrity": "sha512-r+K4vGEvjlF9EEKZSgh1q9pLwbt87tcWUJDgjYyzYVMSPP98tifSOzFuPqQ2GwtQgIlVNLnyYm/PNqPd3RUNFw==", "dependencies": { - "@tus/utils": "^0.1.0", + "@tus/utils": "^0.3.0", "debug": "^4.3.4" }, "engines": { @@ -5027,13 +5027,13 @@ } }, "node_modules/@tus/s3-store": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@tus/s3-store/-/s3-store-1.4.1.tgz", - "integrity": "sha512-ob3GGajN71S2rRy9SE4Rb3G+s2ZswizPE744xEBjzsa3gx7i9utVAww/OhVHDfhxvKqRH4P0SdOZWdifKbbSnQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@tus/s3-store/-/s3-store-1.5.0.tgz", + "integrity": "sha512-YpSL+lZ908mIFdVYME9V0oG46kJFB70CRMkBu0vGckL/7C+YXxfSdBxxfd8nc9TPgGp2hABz8LpTH6fcwoy+QQ==", "dependencies": { "@aws-sdk/client-s3": "^3.490.0", "@shopify/semaphore": "^3.0.2", - "@tus/utils": "^0.1.0", + "@tus/utils": "^0.3.0", "debug": "^4.3.4", "multistream": "^4.1.0" }, @@ -5042,12 +5042,13 @@ } }, "node_modules/@tus/server": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@tus/server/-/server-1.4.1.tgz", - "integrity": "sha512-+cYnRKlZsLdrVFAXKWN3bQZp4F154MqZHmVamUTiNZMECaNO9j151pplNIyB8xCtwxVWHAR2Y1Fy5C2PZ6UX2Q==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@tus/server/-/server-1.7.0.tgz", + "integrity": "sha512-2RlQXkTw3+fMFUbIO4AIrgF8C/OI6WtSQ2R0iWEzpSUD9i8TqJNxa7gNAGFe4SQJUtGLTq7OxUWb6+Dndblr+Q==", "dependencies": { - "@tus/utils": "^0.1.0", - "debug": "^4.3.4" + "@tus/utils": "^0.3.0", + "debug": "^4.3.4", + "lodash.throttle": "^4.1.1" }, "engines": { "node": ">=16" @@ -5057,9 +5058,9 @@ } }, "node_modules/@tus/utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.1.0.tgz", - "integrity": "sha512-RXSeAKPfBJk3G0yyyDAqKPJUb1JsHNvwxNWSjZmvxRlSwtPmOlSkSrXRRReAqHzSlxAlNOGzDWqYiCBkLjOu0g==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-Wt/phitrYEP6T9Tq0ikhJdGKpeEEkOT+vEVUKQKBXBUZdtWubLQKvR2V9jtzekFhLIoqm0KS5uOb7abZgebT3A==", "engines": { "node": ">=16" } @@ -9097,8 +9098,7 @@ "node_modules/lodash.throttle": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", - "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", - "dev": true + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==" }, "node_modules/lodash.uniqby": { "version": "4.5.0", @@ -15360,41 +15360,42 @@ "peer": true }, "@tus/file-store": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-1.3.1.tgz", - "integrity": "sha512-OuqyD4gRSBew7iLEaf9Slo25AU8NH13NmEkCbEhV4r/mVN8Yyz3seuPmYOT21gkCHDcRmdR+yzaX+5JLzSurtA==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-1.4.0.tgz", + "integrity": "sha512-r+K4vGEvjlF9EEKZSgh1q9pLwbt87tcWUJDgjYyzYVMSPP98tifSOzFuPqQ2GwtQgIlVNLnyYm/PNqPd3RUNFw==", "requires": { "@redis/client": "^1.5.13", - "@tus/utils": "^0.1.0", + "@tus/utils": "^0.3.0", "debug": "^4.3.4" } }, "@tus/s3-store": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@tus/s3-store/-/s3-store-1.4.1.tgz", - "integrity": "sha512-ob3GGajN71S2rRy9SE4Rb3G+s2ZswizPE744xEBjzsa3gx7i9utVAww/OhVHDfhxvKqRH4P0SdOZWdifKbbSnQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@tus/s3-store/-/s3-store-1.5.0.tgz", + "integrity": "sha512-YpSL+lZ908mIFdVYME9V0oG46kJFB70CRMkBu0vGckL/7C+YXxfSdBxxfd8nc9TPgGp2hABz8LpTH6fcwoy+QQ==", "requires": { "@aws-sdk/client-s3": "^3.490.0", "@shopify/semaphore": "^3.0.2", - "@tus/utils": "^0.1.0", + "@tus/utils": "^0.3.0", "debug": "^4.3.4", "multistream": "^4.1.0" } }, "@tus/server": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@tus/server/-/server-1.4.1.tgz", - "integrity": "sha512-+cYnRKlZsLdrVFAXKWN3bQZp4F154MqZHmVamUTiNZMECaNO9j151pplNIyB8xCtwxVWHAR2Y1Fy5C2PZ6UX2Q==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@tus/server/-/server-1.7.0.tgz", + "integrity": "sha512-2RlQXkTw3+fMFUbIO4AIrgF8C/OI6WtSQ2R0iWEzpSUD9i8TqJNxa7gNAGFe4SQJUtGLTq7OxUWb6+Dndblr+Q==", "requires": { "@redis/client": "^1.5.13", - "@tus/utils": "^0.1.0", - "debug": "^4.3.4" + "@tus/utils": "^0.3.0", + "debug": "^4.3.4", + "lodash.throttle": "^4.1.1" } }, "@tus/utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.1.0.tgz", - "integrity": "sha512-RXSeAKPfBJk3G0yyyDAqKPJUb1JsHNvwxNWSjZmvxRlSwtPmOlSkSrXRRReAqHzSlxAlNOGzDWqYiCBkLjOu0g==" + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-Wt/phitrYEP6T9Tq0ikhJdGKpeEEkOT+vEVUKQKBXBUZdtWubLQKvR2V9jtzekFhLIoqm0KS5uOb7abZgebT3A==" }, "@types/accepts": { "version": "1.3.7", @@ -18504,8 +18505,7 @@ "lodash.throttle": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", - "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", - "dev": true + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==" }, "lodash.uniqby": { "version": "4.5.0", diff --git a/package.json b/package.json index 39b80461..76c32fb6 100644 --- a/package.json +++ b/package.json @@ -44,9 +44,9 @@ "@opentelemetry/instrumentation-pino": "^0.39.0", "@shopify/semaphore": "^3.0.2", "@smithy/node-http-handler": "^2.3.1", - "@tus/file-store": "1.3.1", - "@tus/s3-store": "1.4.1", - "@tus/server": "1.4.1", + "@tus/file-store": "1.4.0", + "@tus/s3-store": "1.5.0", + "@tus/server": "1.7.0", "agentkeepalive": "^4.5.0", "ajv": "^8.12.0", "async-retry": "^1.3.3", diff --git a/src/http/routes/object/copyObject.ts b/src/http/routes/object/copyObject.ts index a3ba0174..2761bc95 100644 --- a/src/http/routes/object/copyObject.ts +++ b/src/http/routes/object/copyObject.ts @@ -11,6 +11,7 @@ const copyRequestBodySchema = { sourceKey: { type: 'string', examples: ['folder/source.png'] }, destinationBucket: { type: 'string', examples: ['users'] }, destinationKey: { type: 'string', examples: ['folder/destination.png'] }, + copyMetadata: { type: 'boolean', examples: [true] }, }, required: ['sourceKey', 'bucketId', 'destinationKey'], } as const @@ -51,9 +52,13 @@ export default async function routes(fastify: FastifyInstance) { const destinationBucketId = destinationBucket || bucketId - const result = await request.storage - .from(bucketId) - .copyObject(sourceKey, destinationBucketId, destinationKey, request.owner) + const result = await request.storage.from(bucketId).copyObject({ + sourceKey, + destinationBucket: destinationBucketId, + destinationKey, + owner: request.owner, + copyMetadata: request.body.copyMetadata ?? true, + }) return response.status(result.httpStatusCode ?? 200).send({ Id: result.destObject.id, diff --git a/src/http/routes/object/getObjectInfo.ts b/src/http/routes/object/getObjectInfo.ts index 03568827..5f46d864 100644 --- a/src/http/routes/object/getObjectInfo.ts +++ b/src/http/routes/object/getObjectInfo.ts @@ -30,7 +30,8 @@ async function requestHandler( getObjectRequestInterface, unknown >, - publicRoute = false + publicRoute = false, + method: 'head' | 'info' = 'head' ) { const { bucketName } = request.params const objectName = request.params['*'] @@ -42,15 +43,21 @@ async function requestHandler( await request.storage.asSuperUser().findBucket(bucketName, 'id', { isPublic: true, }) - obj = await request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version') + obj = await request.storage + .asSuperUser() + .from(bucketName) + .findObject(objectName, 'id,version,metadata,user_metadata,created_at') } else { - obj = await request.storage.from(bucketName).findObject(objectName, 'id,version') + obj = await request.storage + .from(bucketName) + .findObject(objectName, 'id,version,metadata,user_metadata,created_at') } - return request.storage.renderer('head').render(request, response, { + return request.storage.renderer(method).render(request, response, { bucket: storageS3Bucket, key: s3Key, version: obj.version, + object: obj, }) } @@ -90,7 +97,7 @@ export async function publicRoutes(fastify: FastifyInstance) { }, }, async (request, response) => { - return requestHandler(request, response, true) + return requestHandler(request, response, true, 'info') } ) } @@ -131,7 +138,7 @@ export async function authenticatedRoutes(fastify: FastifyInstance) { }, }, async (request, response) => { - return requestHandler(request, response) + return requestHandler(request, response, false, 'info') } ) @@ -151,7 +158,7 @@ export async function authenticatedRoutes(fastify: FastifyInstance) { }, }, async (request, response) => { - return requestHandler(request, response) + return requestHandler(request, response, false, 'info') } ) diff --git a/src/http/routes/s3/commands/create-multipart-upload.ts b/src/http/routes/s3/commands/create-multipart-upload.ts index 391a85cc..9c0449b4 100644 --- a/src/http/routes/s3/commands/create-multipart-upload.ts +++ b/src/http/routes/s3/commands/create-multipart-upload.ts @@ -21,6 +21,7 @@ const CreateMultiPartUploadInput = { }, Headers: { type: 'object', + additionalProperties: true, properties: { authorization: { type: 'string' }, 'content-type': { type: 'string' }, @@ -39,6 +40,15 @@ export default function CreateMultipartUpload(s3Router: S3Router) { (req, ctx) => { const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + let metadata: undefined | Record = undefined + + Object.keys(req.Headers) + .filter((key) => key.startsWith('x-amz-meta-')) + .forEach((key) => { + if (!metadata) metadata = {} + metadata[key.replace('x-amz-meta-', '')] = req.Headers[key] + }) + return s3Protocol.createMultiPartUpload({ Bucket: req.Params.Bucket, Key: req.Params['*'], @@ -46,6 +56,7 @@ export default function CreateMultipartUpload(s3Router: S3Router) { CacheControl: req.Headers?.['cache-control'], ContentDisposition: req.Headers?.['content-disposition'], ContentEncoding: req.Headers?.['content-encoding'], + Metadata: metadata, }) } ) diff --git a/src/http/routes/s3/commands/upload-part.ts b/src/http/routes/s3/commands/upload-part.ts index 3f3894ea..ca53cfbc 100644 --- a/src/http/routes/s3/commands/upload-part.ts +++ b/src/http/routes/s3/commands/upload-part.ts @@ -94,6 +94,16 @@ export default function UploadPart(s3Router: S3Router) { }, (req, ctx) => { const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner) + + let metadata: undefined | Record = undefined + + Object.keys(req.Headers) + .filter((key) => key.startsWith('x-amz-meta-')) + .forEach((key) => { + if (!metadata) metadata = {} + metadata[key.replace('x-amz-meta-', '')] = req.Headers[key] + }) + return s3Protocol.putObject({ Body: ctx.req as any, Bucket: req.Params.Bucket, @@ -102,6 +112,7 @@ export default function UploadPart(s3Router: S3Router) { ContentType: req.Headers?.['content-type'], Expires: req.Headers?.['expires'] ? new Date(req.Headers?.['expires']) : undefined, ContentEncoding: req.Headers?.['content-encoding'], + Metadata: metadata, }) } ) diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index 736956a6..2a05a8d4 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -154,7 +154,7 @@ export async function onCreate( rawReq: http.IncomingMessage, res: http.ServerResponse, upload: Upload -): Promise { +): Promise<{ res: http.ServerResponse; metadata?: Upload['metadata'] }> { const uploadID = UploadId.fromString(upload.id) const req = rawReq as MultiPartRequest @@ -166,17 +166,21 @@ export async function onCreate( const uploader = new Uploader(storage.backend, storage.db) - if (upload.metadata && /^-?\d+$/.test(upload.metadata.cacheControl || '')) { - upload.metadata.cacheControl = `max-age=${upload.metadata.cacheControl}` - } else if (upload.metadata) { - upload.metadata.cacheControl = 'no-cache' + const metadata = { + ...(upload.metadata ? upload.metadata : {}), } - if (upload.metadata?.contentType && bucket.allowed_mime_types) { - uploader.validateMimeType(upload.metadata.contentType, bucket.allowed_mime_types) + if (/^-?\d+$/.test(metadata.cacheControl || '')) { + metadata.cacheControl = `max-age=${metadata.cacheControl}` + } else if (metadata) { + metadata.cacheControl = 'no-cache' } - return res + if (metadata?.contentType && bucket.allowed_mime_types) { + uploader.validateMimeType(metadata.contentType, bucket.allowed_mime_types) + } + + return { res, metadata } } /** @@ -199,6 +203,14 @@ export async function onUploadFinish( ) const uploader = new Uploader(req.upload.storage.backend, req.upload.storage.db) + let customMd: undefined | Record = undefined + if (upload.metadata?.userMetadata) { + try { + customMd = JSON.parse(upload.metadata.userMetadata) + } catch (e) { + // no-op + } + } await uploader.completeUpload({ version: resourceId.version, @@ -208,6 +220,7 @@ export async function onUploadFinish( isUpsert: req.upload.isUpsert, uploadType: 'resumable', owner: req.upload.owner, + userMetadata: customMd, }) res.setHeader('Tus-Complete', '1') diff --git a/src/internal/database/connection.ts b/src/internal/database/connection.ts index 0f19b63c..b0614540 100644 --- a/src/internal/database/connection.ts +++ b/src/internal/database/connection.ts @@ -53,7 +53,6 @@ export const connections = new TTLCache({ if (!pool) return try { await pool.destroy() - pool.client.removeAllListeners() } catch (e) { logSchema.error(logger, 'pool was not able to be destroyed', { type: 'db', @@ -155,7 +154,6 @@ export class TenantConnection { async dispose() { if (this.options.isExternalPool) { await this.pool.destroy() - this.pool.client.removeAllListeners() } } @@ -201,7 +199,13 @@ export class TenantConnection { // This should never be reached, since the above promise is always rejected in this edge case. throw ERRORS.DatabaseError('Transaction already completed') } - await tnx.raw(`SELECT set_config('search_path', ?, true)`, [searchPath.join(', ')]) + + try { + await tnx.raw(`SELECT set_config('search_path', ?, true)`, [searchPath.join(', ')]) + } catch (e) { + await tnx.rollback() + throw e + } } return tnx diff --git a/src/internal/errors/codes.ts b/src/internal/errors/codes.ts index ac6ca84c..6f1b9383 100644 --- a/src/internal/errors/codes.ts +++ b/src/internal/errors/codes.ts @@ -208,12 +208,12 @@ export const ERRORS = { message: `invalid range provided`, }), - EntityTooLarge: (e?: Error) => + EntityTooLarge: (e?: Error, entity = 'object') => new StorageBackendError({ error: 'Payload too large', code: ErrorCode.EntityTooLarge, httpStatusCode: 413, - message: 'The object exceeded the maximum allowed size', + message: `The ${entity} exceeded the maximum allowed size`, originalError: e, }), diff --git a/src/internal/queue/queue.ts b/src/internal/queue/queue.ts index 4cd5e264..baa5e057 100644 --- a/src/internal/queue/queue.ts +++ b/src/internal/queue/queue.ts @@ -53,10 +53,6 @@ export abstract class Queue { url = multitenantDatabaseUrl } - console.log({ - deleteAfterDays: pgQueueDeleteAfterDays, - deleteAfterHours: pgQueueDeleteAfterHours, - }) Queue.pgBoss = new PgBoss({ connectionString: url, db: new QueueDB({ diff --git a/src/start/server.ts b/src/start/server.ts index c90d8dcb..03293b33 100644 --- a/src/start/server.ts +++ b/src/start/server.ts @@ -71,7 +71,7 @@ async function main() { }) // Start async migrations background process - if (isMultitenant) { + if (isMultitenant && pgQueueEnable) { startAsyncMigrations(shutdownSignal.nextGroup.signal) } diff --git a/src/start/worker.ts b/src/start/worker.ts index b14fc966..2ea35ec7 100644 --- a/src/start/worker.ts +++ b/src/start/worker.ts @@ -16,10 +16,14 @@ bindShutdownSignals(shutdownSignal) main() .then(async () => { logSchema.info(logger, '[Server] Started successfully', { - type: 'server', + type: 'worker', }) }) - .catch(async () => { + .catch(async (e) => { + logSchema.error(logger, '[Queue Server] Error starting server', { + type: 'worker', + error: e, + }) await shutdown(shutdownSignal) process.exit(1) }) diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index 9e53ddbd..e9187c9c 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -114,16 +114,16 @@ export interface Database { ): Promise upsertObject( - data: Pick + data: Pick ): Promise updateObject( bucketId: string, name: string, - data: Pick + data: Pick ): Promise createObject( - data: Pick + data: Pick ): Promise deleteObject(bucketId: string, objectName: string, version?: string): Promise @@ -153,7 +153,8 @@ export interface Database { objectName: string, version: string, signature: string, - owner?: string + owner?: string, + metadata?: Record ): Promise findMultipartUpload( diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index c771576b..785ae36e 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -61,8 +61,6 @@ export class StorageKnexDB implements Database { } catch (e) { await tnx.rollback() throw e - } finally { - tnx.removeAllListeners() } } @@ -320,13 +318,16 @@ export class StorageKnexDB implements Database { return } - async upsertObject(data: Pick) { + async upsertObject( + data: Pick + ) { const objectData = { name: data.name, owner: isUuid(data.owner || '') ? data.owner : undefined, owner_id: data.owner, bucket_id: data.bucket_id, metadata: data.metadata, + user_metadata: data.user_metadata, version: data.version, } const [object] = await this.runQuery('UpsertObject', (knex) => { @@ -336,6 +337,7 @@ export class StorageKnexDB implements Database { .onConflict(['name', 'bucket_id']) .merge({ metadata: data.metadata, + user_metadata: data.user_metadata, version: data.version, owner: isUuid(data.owner || '') ? data.owner : undefined, owner_id: data.owner, @@ -349,7 +351,7 @@ export class StorageKnexDB implements Database { async updateObject( bucketId: string, name: string, - data: Pick + data: Pick ) { const [object] = await this.runQuery('UpdateObject', (knex) => { return knex @@ -363,6 +365,7 @@ export class StorageKnexDB implements Database { owner: isUuid(data.owner || '') ? data.owner : undefined, owner_id: data.owner, metadata: data.metadata, + user_metadata: data.user_metadata, version: data.version, }, '*' @@ -376,7 +379,9 @@ export class StorageKnexDB implements Database { return object } - async createObject(data: Pick) { + async createObject( + data: Pick + ) { try { const object = { name: data.name, @@ -385,6 +390,7 @@ export class StorageKnexDB implements Database { bucket_id: data.bucket_id, metadata: data.metadata, version: data.version, + user_metadata: data.user_metadata, } await this.runQuery('CreateObject', (knex) => { return knex.from('objects').insert(object) @@ -589,7 +595,8 @@ export class StorageKnexDB implements Database { objectName: string, version: string, signature: string, - owner?: string + owner?: string, + metadata?: Record ) { return this.runQuery('CreateMultipartUpload', async (knex) => { const multipart = await knex @@ -601,6 +608,7 @@ export class StorageKnexDB implements Database { version, upload_signature: signature, owner_id: owner, + user_metadata: metadata, }) .returning('*') @@ -732,10 +740,6 @@ export class StorageKnexDB implements Database { } timer() throw e - } finally { - if (needsNewTransaction) { - tnx.removeAllListeners() - } } } } diff --git a/src/storage/events/webhook.ts b/src/storage/events/webhook.ts index 38e3e17c..54bae628 100644 --- a/src/storage/events/webhook.ts +++ b/src/storage/events/webhook.ts @@ -46,6 +46,7 @@ const httpAgent = webhookURL?.startsWith('https://') const client = axios.create({ ...httpAgent, + timeout: 4000, headers: { ...(webhookApiKey ? { authorization: `Bearer ${webhookApiKey}` } : {}), }, diff --git a/src/storage/object.ts b/src/storage/object.ts index 48ce9f3e..6a239ffa 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -27,6 +27,20 @@ export interface UploadObjectOptions { const { requestUrlLengthLimit, storageS3Bucket } = getConfig() +interface CopyObjectParams { + sourceKey: string + destinationBucket: string + destinationKey: string + owner?: string + copyMetadata?: boolean + conditions?: { + ifMatch?: string + ifNoneMatch?: string + ifModifiedSince?: Date + ifUnmodifiedSince?: Date + } +} + /** * ObjectStorage * interact with remote objects and database state @@ -257,19 +271,16 @@ export class ObjectStorage { * @param destinationKey * @param owner * @param conditions + * @param copyMetadata */ - async copyObject( - sourceKey: string, - destinationBucket: string, - destinationKey: string, - owner?: string, - conditions?: { - ifMatch?: string - ifNoneMatch?: string - ifModifiedSince?: Date - ifUnmodifiedSince?: Date - } - ) { + async copyObject({ + sourceKey, + destinationBucket, + destinationKey, + owner, + conditions, + copyMetadata, + }: CopyObjectParams) { mustBeValidKey(destinationKey) const newVersion = randomUUID() @@ -282,7 +293,7 @@ export class ObjectStorage { const originObject = await this.db.findObject( this.bucketId, sourceKey, - 'bucket_id,metadata,version' + 'bucket_id,metadata,user_metadata,version' ) if (s3SourceKey === s3DestinationKey) { @@ -320,6 +331,7 @@ export class ObjectStorage { name: destinationKey, owner, metadata, + user_metadata: copyMetadata ? originObject.user_metadata : undefined, version: newVersion, }) @@ -383,7 +395,7 @@ export class ObjectStorage { const sourceObj = await this.db .asSuperUser() - .findObject(this.bucketId, sourceObjectName, 'id, version') + .findObject(this.bucketId, sourceObjectName, 'id, version,user_metadata') if (s3SourceKey === s3DestinationKey) { return { @@ -410,7 +422,7 @@ export class ObjectStorage { const sourceObject = await db.findObject( this.bucketId, sourceObjectName, - 'id,version,metadata', + 'id,version,metadata,user_metadata', { forUpdate: true, dontErrorOnEmpty: false, @@ -423,6 +435,7 @@ export class ObjectStorage { version: newVersion, owner: owner, metadata, + user_metadata: sourceObj.user_metadata, }) await ObjectAdminDelete.send({ diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index 82c3e84d..84a8fcd2 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -469,7 +469,15 @@ export class S3ProtocolHandler { const signature = this.uploadSignature({ in_progress_size: 0 }) await this.storage.db .asSuperUser() - .createMultipartUpload(uploadId, Bucket, Key, version, signature, this.owner) + .createMultipartUpload( + uploadId, + Bucket, + Key, + version, + signature, + this.owner, + command.Metadata + ) return { responseBody: { @@ -506,7 +514,7 @@ export class S3ProtocolHandler { const multiPartUpload = await this.storage.db .asSuperUser() - .findMultipartUpload(UploadId, 'id,version') + .findMultipartUpload(UploadId, 'id,version,user_metadata') const parts = command.MultipartUpload?.Parts || [] @@ -545,6 +553,7 @@ export class S3ProtocolHandler { uploadType: 's3', objectMetadata: metadata, owner: this.owner, + userMetadata: multiPartUpload.user_metadata || undefined, }) await this.storage.db.asSuperUser().deleteMultipartUpload(UploadId) @@ -695,6 +704,7 @@ export class S3ProtocolHandler { uploadType: 's3', fileSizeLimit: bucket.file_size_limit, allowedMimeTypes: bucket.allowed_mime_types, + metadata: command.Metadata, }) return { @@ -768,12 +778,20 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('Bucket') } - const object = await this.storage.from(Bucket).findObject(Key, 'metadata,created_at,updated_at') + const object = await this.storage + .from(Bucket) + .findObject(Key, 'metadata,user_metadata,created_at,updated_at') if (!object) { throw ERRORS.NoSuchKey(Key) } + let metadataHeaders: Record = {} + + if (object.user_metadata) { + metadataHeaders = toAwsMeatadataHeaders(object.user_metadata) + } + return { headers: { 'created-at': (object.created_at as string) || '', @@ -783,6 +801,7 @@ export class S3ProtocolHandler { 'content-type': (object.metadata?.mimetype as string) || '', etag: (object.metadata?.eTag as string) || '', 'last-modified': object.updated_at ? new Date(object.updated_at).toUTCString() || '' : '', + ...metadataHeaders, }, } } @@ -825,7 +844,7 @@ export class S3ProtocolHandler { const bucket = command.Bucket as string const key = command.Key as string - const object = await this.storage.from(bucket).findObject(key, 'version') + const object = await this.storage.from(bucket).findObject(key, 'version,user_metadata') const response = await this.storage.backend.getObject( storageS3Bucket, `${this.tenantId}/${bucket}/${key}`, @@ -836,6 +855,13 @@ export class S3ProtocolHandler { range: command.Range, } ) + + let metadataHeaders: Record = {} + + if (object.user_metadata) { + metadataHeaders = toAwsMeatadataHeaders(object.user_metadata) + } + return { headers: { 'cache-control': response.metadata.cacheControl, @@ -844,6 +870,7 @@ export class S3ProtocolHandler { 'content-type': response.metadata.mimetype, etag: response.metadata.eTag, 'last-modified': response.metadata.lastModified?.toUTCString() || '', + ...metadataHeaders, }, responseBody: response.body, statusCode: command.Range ? 206 : 200, @@ -964,14 +991,19 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('CopySource') } - const copyResult = await this.storage - .from(sourceBucket) - .copyObject(sourceKey, Bucket, Key, this.owner, { + const copyResult = await this.storage.from(sourceBucket).copyObject({ + sourceKey, + destinationBucket: Bucket, + destinationKey: Key, + owner: this.owner, + conditions: { ifMatch: command.CopySourceIfMatch, ifNoneMatch: command.CopySourceIfNoneMatch, ifModifiedSince: command.CopySourceIfModifiedSince, ifUnmodifiedSince: command.CopySourceIfUnmodifiedSince, - }) + }, + copyMetadata: command.MetadataDirective === 'COPY', + }) return { responseBody: { @@ -1208,6 +1240,37 @@ export class S3ProtocolHandler { } } +function toAwsMeatadataHeaders(records: Record) { + const metadataHeaders: Record = {} + let missingCount = 0 + + if (records) { + Object.keys(records).forEach((key) => { + const value = records[key] + if (isUSASCII(value)) { + metadataHeaders['x-amz-meta-' + key.toLowerCase()] = value + } else { + missingCount++ + } + }) + } + + if (missingCount) { + metadataHeaders['x-amz-missing-meta'] = missingCount + } + + return metadataHeaders +} + +function isUSASCII(str: string): boolean { + for (let i = 0; i < str.length; i++) { + if (str.charCodeAt(i) > 127) { + return false + } + } + return true +} + function encodeContinuationToken(name: string) { return Buffer.from(`l:${name}`).toString('base64') } diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index 86bd1fb5..7d933f08 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -1,6 +1,5 @@ import crypto from 'crypto' import { ERRORS } from '@internal/errors' -import { signatureV4 } from '../../../http/plugins' interface SignatureV4Options { enforceRegion: boolean diff --git a/src/storage/renderer/head.ts b/src/storage/renderer/head.ts index 2f9c78cd..973fd1e1 100644 --- a/src/storage/renderer/head.ts +++ b/src/storage/renderer/head.ts @@ -1,22 +1,28 @@ import { AssetResponse, Renderer, RenderOptions } from './renderer' import { FastifyReply, FastifyRequest } from 'fastify' -import { ObjectMetadata, StorageBackendAdapter } from '../backend' +import { ObjectMetadata } from '../backend' import { ImageRenderer, TransformOptions } from './image' +import { ERRORS } from '@internal/errors' /** * HeadRenderer * is a special renderer that only outputs metadata information with an empty content */ export class HeadRenderer extends Renderer { - constructor(private readonly backend: StorageBackendAdapter) { - super() - } - async getAsset(request: FastifyRequest, options: RenderOptions): Promise { - const metadata = await this.backend.headObject(options.bucket, options.key, options.version) + const { object } = options + + if (!object) { + throw ERRORS.NoSuchKey(`${options.bucket}/${options.key}/${options.version}`) + } + + const metadata = object.metadata ? { ...object.metadata } : {} + if (metadata.lastModified) { + metadata.lastModified = new Date(metadata.lastModified as string) + } return { - metadata, + metadata: metadata as ObjectMetadata, transformations: ImageRenderer.applyTransformation(request.query as TransformOptions), } } diff --git a/src/storage/renderer/info.ts b/src/storage/renderer/info.ts new file mode 100644 index 00000000..efe06cbe --- /dev/null +++ b/src/storage/renderer/info.ts @@ -0,0 +1,31 @@ +import { HeadRenderer } from './head' +import { FastifyRequest } from 'fastify' +import { AssetResponse, RenderOptions } from './renderer' +import { Obj } from '@storage/schemas' +import { FastifyReply } from 'fastify/types/reply' + +/** + * HeadRenderer + * is a special renderer that only outputs metadata information with an empty content + */ +export class InfoRenderer extends HeadRenderer { + async getAsset(request: FastifyRequest, options: RenderOptions): Promise { + const headAsset = await super.getAsset(request, options) + + const obj = options.object as Obj + + return { + ...headAsset, + body: obj, + } + } + + protected setHeaders( + request: FastifyRequest, + response: FastifyReply, + data: AssetResponse, + options: RenderOptions + ) { + // no-op + } +} diff --git a/src/storage/renderer/renderer.ts b/src/storage/renderer/renderer.ts index 40e90c62..529f497e 100644 --- a/src/storage/renderer/renderer.ts +++ b/src/storage/renderer/renderer.ts @@ -2,6 +2,7 @@ import { FastifyReply, FastifyRequest } from 'fastify' import { ObjectMetadata } from '../backend' import { Readable } from 'stream' import { getConfig } from '../../config' +import { Obj } from '../schemas' export interface RenderOptions { bucket: string @@ -9,10 +10,11 @@ export interface RenderOptions { version: string | undefined download?: string expires?: string + object?: Obj } export interface AssetResponse { - body?: Readable | ReadableStream | Blob | Buffer + body?: Readable | ReadableStream | Blob | Buffer | Record metadata: ObjectMetadata transformations?: string[] } diff --git a/src/storage/schemas/multipart.ts b/src/storage/schemas/multipart.ts index df0796aa..1ca6eb29 100644 --- a/src/storage/schemas/multipart.ts +++ b/src/storage/schemas/multipart.ts @@ -12,6 +12,9 @@ export const multipartUploadSchema = { version: { type: 'string' }, owner_id: { type: 'string' }, created_at: { type: 'string' }, + user_metadata: { + anyOf: [{ type: 'object', additionalProperties: true }, { type: 'null' }], + }, }, required: [ 'id', diff --git a/src/storage/schemas/object.ts b/src/storage/schemas/object.ts index e0ddbf23..c7805aab 100644 --- a/src/storage/schemas/object.ts +++ b/src/storage/schemas/object.ts @@ -17,6 +17,9 @@ export const objectSchema = { metadata: { anyOf: [{ type: 'object', additionalProperties: true }, { type: 'null' }], }, + user_metadata: { + anyOf: [{ type: 'object', additionalProperties: true }, { type: 'null' }], + }, buckets: bucketSchema, }, required: ['name'], diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 54d2b7d0..9589eea4 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -5,6 +5,7 @@ import { AssetRenderer, HeadRenderer, ImageRenderer } from './renderer' import { getFileSizeLimit, mustBeValidBucketName, parseFileSizeToBytes } from './limits' import { getConfig } from '../config' import { ObjectStorage } from './object' +import { InfoRenderer } from '@storage/renderer/info' const { requestUrlLengthLimit, storageS3Bucket } = getConfig() @@ -38,14 +39,16 @@ export class Storage { * Creates a renderer type * @param type */ - renderer(type: 'asset' | 'head' | 'image') { + renderer(type: 'asset' | 'head' | 'image' | 'info') { switch (type) { case 'asset': return new AssetRenderer(this.backend) case 'head': - return new HeadRenderer(this.backend) + return new HeadRenderer() case 'image': return new ImageRenderer(this.backend) + case 'info': + return new InfoRenderer() } throw new Error(`renderer of type "${type}" not supported`) diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index 86ae602f..2b07ef07 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -14,6 +14,7 @@ import { logger, logSchema } from '@internal/monitoring' interface UploaderOptions extends UploadObjectOptions { fileSizeLimit?: number | null allowedMimeTypes?: string[] | null + metadata?: Record } const { storageS3Bucket, uploadFileSizeLimitStandard } = getConfig() @@ -26,6 +27,8 @@ export interface UploadObjectOptions { uploadType?: 'standard' | 's3' | 'resumable' } +const MAX_CUSTOM_METADATA_SIZE = 1024 * 1024 + /** * Uploader * Handles the upload of a multi-part request or binary body @@ -114,6 +117,7 @@ export class Uploader { ...options, version, objectMetadata: objectMetadata, + userMetadata: { ...file.userMetadata, ...(options.metadata || {}) }, }) } catch (e) { await ObjectAdminDelete.send({ @@ -135,11 +139,13 @@ export class Uploader { objectMetadata, uploadType, isUpsert, + userMetadata, }: UploadObjectOptions & { objectMetadata: ObjectMetadata version: string emitEvent?: boolean uploadType?: 'standard' | 's3' | 'resumable' + userMetadata?: Record }) { try { return await this.db.asSuperUser().withTransaction(async (db) => { @@ -159,6 +165,7 @@ export class Uploader { bucket_id: bucketId, name: objectName, metadata: objectMetadata, + user_metadata: userMetadata, version, owner, }) @@ -271,6 +278,7 @@ export class Uploader { ) let body: NodeJS.ReadableStream + let userMetadata: Record | undefined let mimeType: string let isTruncated: () => boolean @@ -289,9 +297,23 @@ export class Uploader { body = formData.file /* @ts-expect-error: https://github.com/aws/aws-sdk-js-v3/issues/2085 */ + const customMd = formData.fields.userMetadata?.value + /* @ts-expect-error: https://github.com/aws/aws-sdk-js-v3/issues/2085 */ mimeType = formData.fields.contentType?.value || formData.mimetype cacheControl = cacheTime ? `max-age=${cacheTime}` : 'no-cache' isTruncated = () => formData.file.truncated + + if (typeof customMd === 'string') { + if (Buffer.byteLength(customMd, 'utf8') > MAX_CUSTOM_METADATA_SIZE) { + throw ERRORS.EntityTooLarge(undefined, 'user_metadata') + } + + try { + userMetadata = JSON.parse(customMd) + } catch (e) { + // no-op + } + } } catch (e) { throw ERRORS.NoContentProvided(e as Error) } @@ -300,6 +322,20 @@ export class Uploader { body = request.raw mimeType = request.headers['content-type'] || 'application/octet-stream' cacheControl = request.headers['cache-control'] ?? 'no-cache' + + const customMd = request.headers['x-metadata'] + + if (typeof customMd === 'string') { + if (userMetadata && Buffer.byteLength(customMd, 'utf8') > MAX_CUSTOM_METADATA_SIZE) { + throw ERRORS.EntityTooLarge(undefined, 'metadata') + } + + try { + userMetadata = JSON.parse(customMd) + } catch (e) { + // no-op + } + } isTruncated = () => { // @todo more secure to get this from the stream or from s3 in the next step return Number(request.headers['content-length']) > fileSizeLimit @@ -311,6 +347,7 @@ export class Uploader { mimeType, cacheControl, isTruncated, + userMetadata, } } } diff --git a/src/test/db/02-dummy-data.sql b/src/test/db/02-dummy-data.sql index e9b10b79..45241e49 100644 --- a/src/test/db/02-dummy-data.sql +++ b/src/test/db/02-dummy-data.sql @@ -23,31 +23,31 @@ INSERT INTO "storage"."buckets" ("id", "name", "owner", "created_at", "updated_a -- insert objects -INSERT INTO "storage"."objects" ("id", "bucket_id", "name", "owner", "created_at", "updated_at", "last_accessed_at", "metadata") VALUES -('03e458f9-892f-4db2-8cb9-d3401a689e25', 'bucket2', 'public/sadcat-upload23.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-04 08:26:08.553748+00', '2021-03-04 08:26:08.553748+00', '2021-03-04 08:26:08.553748+00', '{"mimetype": "image/svg+xml", "size": 1234}'), -('070825af-a11d-44fe-9f1d-abdc76f686f2', 'bucket2', 'public/sadcat-upload.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '{"mimetype": "image/png", "size": 1234}'), -('0cac5609-11e1-4f21-b486-d0eeb60909f6', 'bucket2', 'curlimage.jpg', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-23 11:05:16.625075+00', '2021-02-23 11:05:16.625075+00', '2021-02-23 11:05:16.625075+00', '{"size": 1234}'), -('147c6795-94d5-4008-9d81-f7ba3b4f8a9f', 'bucket2', 'folder/only_uid.jpg', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:36:01.504227+00', '2021-02-17 11:03:03.049618+00', '2021-02-17 10:36:01.504227+00', '{"size": 1234}'), -('65a3aa9c-0ff2-4adc-85d0-eab673c27443', 'bucket2', 'authenticated/casestudy.png', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:42:19.366559+00', '2021-02-17 11:03:30.025116+00', '2021-02-17 10:42:19.366559+00', '{"size": 1234}'), -('10ABE273-D77A-4BDA-B410-6FC0CA3E6ADC', 'bucket2', 'authenticated/cat.jpg', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:42:19.366559+00', '2021-02-17 11:03:30.025116+00', '2021-02-17 10:42:19.366559+00', '{"size": 1234}'), -('1edccac7-0876-4e9f-89da-a08d2a5f654b', 'bucket2', 'authenticated/delete.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '{"mimetype": "image/png", "size": 1234}'), -('1a911f3c-8c1d-4661-93c1-8e065e4d757e', 'bucket2', 'authenticated/delete1.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('372d5d74-e24d-49dc-abe8-47d7eb226a2e', 'bucket2', 'authenticated/delete-multiple1.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('34811c1b-85e5-4eb6-a5e3-d607b2f6986e', 'bucket2', 'authenticated/delete-multiple2.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('45950ff2-d3a8-4add-8e49-bafc01198340', 'bucket2', 'authenticated/delete-multiple3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('469b0216-5419-41f6-9a37-2abfd7fad29c', 'bucket2', 'authenticated/delete-multiple4.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('55930619-a668-4dbc-aea3-b93dfe101e7f', 'bucket2', 'authenticated/delete-multiple7.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('D1CE4E4F-03E2-473D-858B-301D7989B581', 'bucket2', 'authenticated/move-orig.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('222b3d1e-bc17-414c-b336-47894aa4d697', 'bucket2', 'authenticated/move-orig-2.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('8f7d643d-1e82-4d39-ae39-d9bd6b0cfe9c', 'bucket2', 'authenticated/move-orig-3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('24f70210-62aa-4daa-9909-693b3febd8fd', 'bucket2', 'authenticated/move-orig-4.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('18dc5e3b-4fb1-45a7-bfa4-d99b0784be31', 'bucket2', 'authenticated/move-orig-5.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}'), -('8377527d-3518-4dc8-8290-c6926470e795', 'bucket2', 'folder/subfolder/public-all-permissions.png', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:26:42.791214+00', '2021-02-17 11:03:30.025116+00', '2021-02-17 10:26:42.791214+00', '{"size": 1234}'), -('b39ae4ab-802b-4c42-9271-3f908c34363c', 'bucket2', 'private/sadcat-upload3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}'), -('8098E1AC-C744-4368-86DF-71B60CCDE221', 'bucket3', 'sadcat-upload3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}'), -('D3EB488E-94F4-46CD-86D3-242C13B95BAC', 'bucket3', 'sadcat-upload2.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}'), -('746180e8-8029-4134-8a21-48ab35485d81', 'public-bucket', 'favicon.ico', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}'), -('ea2e2806-9ded-4882-8c26-e172a29ed063', 'public-bucket-2', 'favicon.ico', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}'); +INSERT INTO "storage"."objects" ("id", "bucket_id", "name", "owner", "created_at", "updated_at", "last_accessed_at", "metadata", "user_metadata") VALUES +('03e458f9-892f-4db2-8cb9-d3401a689e25', 'bucket2', 'public/sadcat-upload23.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-04 08:26:08.553748+00', '2021-03-04 08:26:08.553748+00', '2021-03-04 08:26:08.553748+00', '{"mimetype": "image/svg+xml", "size": 1234}', NULL), +('070825af-a11d-44fe-9f1d-abdc76f686f2', 'bucket2', 'public/sadcat-upload.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('0cac5609-11e1-4f21-b486-d0eeb60909f6', 'bucket2', 'curlimage.jpg', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-23 11:05:16.625075+00', '2021-02-23 11:05:16.625075+00', '2021-02-23 11:05:16.625075+00', '{"size": 1234}', NULL), +('147c6795-94d5-4008-9d81-f7ba3b4f8a9f', 'bucket2', 'folder/only_uid.jpg', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:36:01.504227+00', '2021-02-17 11:03:03.049618+00', '2021-02-17 10:36:01.504227+00', '{"size": 1234}', NULL), +('65a3aa9c-0ff2-4adc-85d0-eab673c27443', 'bucket2', 'authenticated/casestudy.png', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:42:19.366559+00', '2021-02-17 11:03:30.025116+00', '2021-02-17 10:42:19.366559+00', '{"size": 1234, "eTag": "abc", "lastModified": "Wed, 12 Oct 2022 11:17:02 GMT", "contentLength": 3746, "cacheControl": "no-cache"}', '{"test1": 1234}'), +('10ABE273-D77A-4BDA-B410-6FC0CA3E6ADC', 'bucket2', 'authenticated/cat.jpg', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:42:19.366559+00', '2021-02-17 11:03:30.025116+00', '2021-02-17 10:42:19.366559+00', '{"size": 1234}', NULL), +('1edccac7-0876-4e9f-89da-a08d2a5f654b', 'bucket2', 'authenticated/delete.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '2021-03-02 16:31:11.115996+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('1a911f3c-8c1d-4661-93c1-8e065e4d757e', 'bucket2', 'authenticated/delete1.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('372d5d74-e24d-49dc-abe8-47d7eb226a2e', 'bucket2', 'authenticated/delete-multiple1.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('34811c1b-85e5-4eb6-a5e3-d607b2f6986e', 'bucket2', 'authenticated/delete-multiple2.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('45950ff2-d3a8-4add-8e49-bafc01198340', 'bucket2', 'authenticated/delete-multiple3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('469b0216-5419-41f6-9a37-2abfd7fad29c', 'bucket2', 'authenticated/delete-multiple4.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('55930619-a668-4dbc-aea3-b93dfe101e7f', 'bucket2', 'authenticated/delete-multiple7.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('D1CE4E4F-03E2-473D-858B-301D7989B581', 'bucket2', 'authenticated/move-orig.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('222b3d1e-bc17-414c-b336-47894aa4d697', 'bucket2', 'authenticated/move-orig-2.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('8f7d643d-1e82-4d39-ae39-d9bd6b0cfe9c', 'bucket2', 'authenticated/move-orig-3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('24f70210-62aa-4daa-9909-693b3febd8fd', 'bucket2', 'authenticated/move-orig-4.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('18dc5e3b-4fb1-45a7-bfa4-d99b0784be31', 'bucket2', 'authenticated/move-orig-5.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-02-22 22:29:15.14732+00', '2021-02-22 22:29:15.14732+00', '2021-03-02 09:32:17.116+00', '{"mimetype": "image/png", "size": 1234}', NULL), +('8377527d-3518-4dc8-8290-c6926470e795', 'bucket2', 'folder/subfolder/public-all-permissions.png', 'd8c7bce9-cfeb-497b-bd61-e66ce2cbdaa2', '2021-02-17 10:26:42.791214+00', '2021-02-17 11:03:30.025116+00', '2021-02-17 10:26:42.791214+00', '{"size": 1234}', NULL), +('b39ae4ab-802b-4c42-9271-3f908c34363c', 'bucket2', 'private/sadcat-upload3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}', NULL), +('8098E1AC-C744-4368-86DF-71B60CCDE221', 'bucket3', 'sadcat-upload3.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}', NULL), +('D3EB488E-94F4-46CD-86D3-242C13B95BAC', 'bucket3', 'sadcat-upload2.png', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}', NULL), +('746180e8-8029-4134-8a21-48ab35485d81', 'public-bucket', 'favicon.ico', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"mimetype": "image/svg+xml", "size": 1234}', NULL), +('ea2e2806-9ded-4882-8c26-e172a29ed063', 'public-bucket-2', 'favicon.ico', '317eadce-631a-4429-a0bb-f19a7a517b4a', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '2021-03-01 08:53:29.567975+00', '{"size": 1234, "mimetype": "image/svg+xml", "eTag": "abc", "lastModified": "Wed, 12 Oct 2022 11:17:02 GMT", "contentLength": 3746, "cacheControl": "no-cache"}', NULL); ; INSERT INTO "storage"."objects" ("id", "bucket_id", "name", "owner", "created_at", "updated_at", "last_accessed_at", "metadata") SELECT gen_random_uuid(), 'bucket2', 'authenticated/' || i, '317eadce-631a-4429-a0bb-f19a7a517b4a', now(), now(), now(), '{"size": 1234}' diff --git a/src/test/object.test.ts b/src/test/object.test.ts index d380c70e..c1958c73 100644 --- a/src/test/object.test.ts +++ b/src/test/object.test.ts @@ -97,7 +97,6 @@ describe('testing GET object', () => { expect(response.headers['last-modified']).toBe('Wed, 12 Oct 2022 11:17:02 GMT') expect(response.headers['content-length']).toBe(3746) expect(response.headers['cache-control']).toBe('no-cache') - expect(S3Backend.prototype.headObject).toBeCalled() }) test('get public object info', async () => { @@ -113,7 +112,6 @@ describe('testing GET object', () => { expect(response.headers['last-modified']).toBe('Wed, 12 Oct 2022 11:17:02 GMT') expect(response.headers['content-length']).toBe(3746) expect(response.headers['cache-control']).toBe('no-cache') - expect(S3Backend.prototype.headObject).toBeCalled() }) test('force downloading file with default name', async () => { @@ -356,6 +354,87 @@ describe('testing POST object via multipart upload', () => { expect(S3Backend.prototype.uploadObject).toHaveBeenCalled() }) + test('successfully uploading an object with custom metadata', async () => { + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + form.append( + 'userMetadata', + JSON.stringify({ + test1: 'test1', + test2: 'test2', + }) + ) + const headers = Object.assign({}, form.getHeaders(), { + authorization: `Bearer ${serviceKey}`, + 'x-upsert': 'true', + ...form.getHeaders(), + }) + + const response = await app().inject({ + method: 'POST', + url: '/object/bucket2/sadcat-upload3012.png', + headers, + payload: form, + }) + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.uploadObject).toHaveBeenCalled() + + const client = await getSuperuserPostgrestClient() + + const object = await client + .table('objects') + .select('*') + .where('name', 'sadcat-upload3012.png') + .where('bucket_id', 'bucket2') + .first() + + expect(object).not.toBeFalsy() + expect(object?.user_metadata).toEqual({ + test1: 'test1', + test2: 'test2', + }) + }) + + test('fetch object metadata', async () => { + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + form.append( + 'userMetadata', + JSON.stringify({ + test1: 'test1', + test2: 'test2', + }) + ) + const headers = Object.assign({}, form.getHeaders(), { + authorization: `Bearer ${serviceKey}`, + 'x-upsert': 'true', + }) + + const uploadResponse = await app().inject({ + method: 'POST', + url: '/object/bucket2/sadcat-upload3019.png', + headers: { + ...headers, + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const response = await app().inject({ + method: 'GET', + url: '/object/info/bucket2/sadcat-upload3019.png', + headers, + }) + + const data = await response.json() + + expect(data.user_metadata).toEqual({ + test1: 'test1', + test2: 'test2', + }) + }) + test('return 422 when uploading an object with a not allowed mime-type', async () => { const form = new FormData() form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) @@ -1047,6 +1126,70 @@ describe('testing copy object', () => { expect(response.body).toBe(`{"Key":"bucket3/authenticated/casestudy11.png"}`) }) + test('can copy objects keeping their metadata', async () => { + const copiedKey = 'casestudy-2349.png' + const response = await app().inject({ + method: 'POST', + url: '/object/copy', + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + payload: { + bucketId: 'bucket2', + sourceKey: 'authenticated/casestudy.png', + destinationKey: `authenticated/${copiedKey}`, + copyMetadata: true, + }, + }) + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.copyObject).toBeCalled() + expect(response.body).toBe(`{"Key":"bucket2/authenticated/${copiedKey}"}`) + + const conn = await getSuperuserPostgrestClient() + const object = await conn + .table('objects') + .select('*') + .where('bucket_id', 'bucket2') + .where('name', `authenticated/${copiedKey}`) + .first() + + expect(object).not.toBeFalsy() + expect(object.user_metadata).toEqual({ + test1: 1234, + }) + }) + + test('can copy objects excluding their metadata', async () => { + const copiedKey = 'casestudy-2450.png' + const response = await app().inject({ + method: 'POST', + url: '/object/copy', + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + payload: { + bucketId: 'bucket2', + sourceKey: 'authenticated/casestudy.png', + destinationKey: `authenticated/${copiedKey}`, + copyMetadata: false, + }, + }) + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.copyObject).toBeCalled() + expect(response.body).toBe(`{"Key":"bucket2/authenticated/${copiedKey}"}`) + + const conn = await getSuperuserPostgrestClient() + const object = await conn + .table('objects') + .select('*') + .where('bucket_id', 'bucket2') + .where('name', `authenticated/${copiedKey}`) + .first() + + expect(object).not.toBeFalsy() + expect(object.user_metadata).toBeNull() + }) + test('cannot copy objects across buckets when RLS dont allow it', async () => { const response = await app().inject({ method: 'POST', @@ -1978,7 +2121,7 @@ describe('testing list objects', () => { }) expect(response.statusCode).toBe(200) const responseJSON = JSON.parse(response.body) - expect(responseJSON).toHaveLength(6) + expect(responseJSON).toHaveLength(8) const names = responseJSON.map((ele: any) => ele.name) expect(names).toContain('curlimage.jpg') expect(names).toContain('private') diff --git a/src/test/s3-protocol.test.ts b/src/test/s3-protocol.test.ts index 200b9238..39dc3f9a 100644 --- a/src/test/s3-protocol.test.ts +++ b/src/test/s3-protocol.test.ts @@ -11,6 +11,7 @@ import { GetBucketVersioningCommand, GetObjectCommand, HeadBucketCommand, + HeadObjectCommand, ListBucketsCommand, ListMultipartUploadsCommand, ListObjectsCommand, @@ -497,6 +498,32 @@ describe('S3 Protocol', () => { expect(resp.$metadata.httpStatusCode).toEqual(200) }) + it('upload a file using putObject with custom metadata', async () => { + const bucketName = await createBucket(client) + + const putObject = new PutObjectCommand({ + Bucket: bucketName, + Key: 'test-1-put-object.jpg', + Body: Buffer.alloc(1024 * 1024 * 12), + Metadata: { + nice: '1111', + test2: 'test3', + }, + }) + + const resp = await client.send(putObject) + expect(resp.$metadata.httpStatusCode).toEqual(200) + + const getObject = new HeadObjectCommand({ + Bucket: bucketName, + Key: 'test-1-put-object.jpg', + }) + + const headResp = await client.send(getObject) + expect(headResp.Metadata?.nice).toEqual('1111') + expect(headResp.Metadata?.test2).toEqual('test3') + }) + it('it will not allow to upload a file using putObject when exceeding maxFileSize', async () => { const bucketName = await createBucket(client) diff --git a/src/test/tenant.test.ts b/src/test/tenant.test.ts index d47a8e92..8508fb57 100644 --- a/src/test/tenant.test.ts +++ b/src/test/tenant.test.ts @@ -16,7 +16,7 @@ const payload = { serviceKey: 'd', jwks: { keys: [] }, migrationStatus: 'COMPLETED', - migrationVersion: 'operation-function', + migrationVersion: 'custom-metadata', tracingMode: 'basic', features: { imageTransformation: { @@ -35,7 +35,7 @@ const payload2 = { serviceKey: 'h', jwks: null, migrationStatus: 'COMPLETED', - migrationVersion: 'operation-function', + migrationVersion: 'custom-metadata', tracingMode: 'basic', features: { imageTransformation: { diff --git a/src/test/tus.test.ts b/src/test/tus.test.ts index dc54d48d..2b707738 100644 --- a/src/test/tus.test.ts +++ b/src/test/tus.test.ts @@ -95,6 +95,10 @@ describe('Tus multipart', () => { objectName: objectName, contentType: 'image/jpeg', cacheControl: '3600', + userMetadata: JSON.stringify({ + test1: 'test1', + test2: 'test2', + }), }, onError: function (error) { console.log('Failed because: ' + error) @@ -125,6 +129,10 @@ describe('Tus multipart', () => { mimetype: 'image/jpeg', size: 29526, }, + user_metadata: { + test1: 'test1', + test2: 'test2', + }, name: objectName, owner: null, owner_id: null, @@ -259,6 +267,10 @@ describe('Tus multipart', () => { objectName: objectName, contentType: 'image/jpeg', cacheControl: '3600', + userMetadata: JSON.stringify({ + test1: 'test1', + test3: 'test3', + }), }, onError: function (error) { console.log('Failed because: ' + error) @@ -289,6 +301,10 @@ describe('Tus multipart', () => { mimetype: 'image/jpeg', size: 29526, }, + user_metadata: { + test1: 'test1', + test3: 'test3', + }, name: objectName, owner: null, owner_id: null, @@ -354,6 +370,7 @@ describe('Tus multipart', () => { mimetype: 'image/jpeg', size: 29526, }, + user_metadata: null, name: objectName, owner: null, owner_id: 'some-owner-id',