From f5f0ec42c4c29782d50abd83068ffa7c29e90449 Mon Sep 17 00:00:00 2001 From: fenos Date: Wed, 12 Jun 2024 14:21:21 +0200 Subject: [PATCH] feat: s3 Signed URLs --- src/config.ts | 4 + src/http/plugins/log-request.ts | 14 +- src/http/plugins/signature-v4.ts | 46 ++++--- src/monitoring/logger.ts | 7 +- src/storage/errors.ts | 5 +- src/storage/protocols/s3/signature-v4.ts | 165 ++++++++++++++++++----- src/test/s3-protocol.test.ts | 62 +++++++-- 7 files changed, 239 insertions(+), 64 deletions(-) diff --git a/src/config.ts b/src/config.ts index 83d45668..016f4182 100644 --- a/src/config.ts +++ b/src/config.ts @@ -103,6 +103,7 @@ type StorageConfigType = { s3ProtocolEnforceRegion: boolean s3ProtocolAccessKeyId?: string s3ProtocolAccessKeySecret?: string + s3ProtocolNonCanonicalHostHeader?: string tracingMode?: string } @@ -232,6 +233,9 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { s3ProtocolEnforceRegion: getOptionalConfigFromEnv('S3_PROTOCOL_ENFORCE_REGION') === 'true', s3ProtocolAccessKeyId: getOptionalConfigFromEnv('S3_PROTOCOL_ACCESS_KEY_ID'), s3ProtocolAccessKeySecret: getOptionalConfigFromEnv('S3_PROTOCOL_ACCESS_KEY_SECRET'), + s3ProtocolNonCanonicalHostHeader: getOptionalConfigFromEnv( + 'S3_PROTOCOL_NON_CANONICAL_HOST_HEADER' + ), // Storage storageBackendType: getOptionalConfigFromEnv('STORAGE_BACKEND') as StorageBackendType, diff --git a/src/http/plugins/log-request.ts b/src/http/plugins/log-request.ts index 3d09931a..c943385c 100644 --- a/src/http/plugins/log-request.ts +++ b/src/http/plugins/log-request.ts @@ -52,7 +52,12 @@ export const logRequest = (options: RequestLoggerOptions) => } const rMeth = req.method - const rUrl = redactQueryParamFromRequest(req, ['token']) + const rUrl = redactQueryParamFromRequest(req, [ + 'token', + 'X-Amz-Credential', + 'X-Amz-Signature', + 'X-Amz-Security-Token', + ]) const uAgent = req.headers['user-agent'] const rId = req.id const cIP = req.ip @@ -78,7 +83,12 @@ export const logRequest = (options: RequestLoggerOptions) => } const rMeth = req.method - const rUrl = redactQueryParamFromRequest(req, ['token']) + const rUrl = redactQueryParamFromRequest(req, [ + 'token', + 'X-Amz-Credential', + 'X-Amz-Signature', + 'X-Amz-Security-Token', + ]) const uAgent = req.headers['user-agent'] const rId = req.id const cIP = req.ip diff --git a/src/http/plugins/signature-v4.ts b/src/http/plugins/signature-v4.ts index c28a1613..8e6bb9ca 100644 --- a/src/http/plugins/signature-v4.ts +++ b/src/http/plugins/signature-v4.ts @@ -18,25 +18,22 @@ const { s3ProtocolEnforceRegion, s3ProtocolAccessKeyId, s3ProtocolAccessKeySecret, + s3ProtocolNonCanonicalHostHeader, } = getConfig() -export const signatureV4 = fastifyPlugin(async function (fastify: FastifyInstance) { - fastify.addHook('preHandler', async (request: FastifyRequest) => { - if (typeof request.headers.authorization !== 'string') { - throw ERRORS.AccessDenied('Missing authorization header') - } +type AWSRequest = FastifyRequest<{ Querystring: { 'X-Amz-Credential'?: string } }> - const clientCredentials = SignatureV4.parseAuthorizationHeader(request.headers.authorization) +export const signatureV4 = fastifyPlugin(async function (fastify: FastifyInstance) { + fastify.addHook('preHandler', async (request: AWSRequest) => { + const clientSignature = extractSignature(request) - const sessionToken = request.headers['x-amz-security-token'] as string | undefined + const sessionToken = clientSignature.sessionToken const { signature: signatureV4, claims, token, - } = await createSignature(request.tenantId, clientCredentials, { - sessionToken: sessionToken, - }) + } = await createServerSignature(request.tenantId, clientSignature) const isVerified = signatureV4.verify({ url: request.url, @@ -45,9 +42,7 @@ export const signatureV4 = fastifyPlugin(async function (fastify: FastifyInstanc method: request.method, query: request.query as Record, prefix: s3ProtocolPrefix, - credentials: clientCredentials.credentials, - signature: clientCredentials.signature, - signedHeaders: clientCredentials.signedHeaders, + clientSignature: clientSignature, }) if (!isVerified && !sessionToken) { @@ -94,15 +89,23 @@ export const signatureV4 = fastifyPlugin(async function (fastify: FastifyInstanc }) }) -async function createSignature( - tenantId: string, - clientSignature: ClientSignature, - session?: { sessionToken?: string } -) { +function extractSignature(req: AWSRequest) { + if (typeof req.headers.authorization === 'string') { + return SignatureV4.parseAuthorizationHeader(req.headers) + } + + if (typeof req.query['X-Amz-Credential'] === 'string') { + return SignatureV4.parseQuerySignature(req.query) + } + + throw ERRORS.AccessDenied('Missing signature') +} + +async function createServerSignature(tenantId: string, clientSignature: ClientSignature) { const awsRegion = storageS3Region const awsService = 's3' - if (session?.sessionToken) { + if (clientSignature?.sessionToken) { const tenantAnonKey = isMultitenant ? (await getTenantConfig(tenantId)).anonKey : anonKey if (!tenantAnonKey) { @@ -112,6 +115,7 @@ async function createSignature( const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: tenantId, secretKey: tenantAnonKey, @@ -120,7 +124,7 @@ async function createSignature( }, }) - return { signature, claims: undefined, token: session.sessionToken } + return { signature, claims: undefined, token: clientSignature.sessionToken } } if (isMultitenant) { @@ -132,6 +136,7 @@ async function createSignature( const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: credential.accessKey, secretKey: credential.secretKey, @@ -152,6 +157,7 @@ async function createSignature( const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: s3ProtocolAccessKeyId, secretKey: s3ProtocolAccessKeySecret, diff --git a/src/monitoring/logger.ts b/src/monitoring/logger.ts index 63ed422c..8eb7dd2e 100644 --- a/src/monitoring/logger.ts +++ b/src/monitoring/logger.ts @@ -23,7 +23,12 @@ export const logger = pino({ region, traceId: request.id, method: request.method, - url: redactQueryParamFromRequest(request, ['token']), + url: redactQueryParamFromRequest(request, [ + 'token', + 'X-Amz-Credential', + 'X-Amz-Signature', + 'X-Amz-Security-Token', + ]), headers: whitelistHeaders(request.headers), hostname: request.hostname, remoteAddress: request.ip, diff --git a/src/storage/errors.ts b/src/storage/errors.ts index 60632f05..141bf71a 100644 --- a/src/storage/errors.ts +++ b/src/storage/errors.ts @@ -27,6 +27,7 @@ export enum ErrorCode { BucketAlreadyExists = 'BucketAlreadyExists', DatabaseTimeout = 'DatabaseTimeout', InvalidSignature = 'InvalidSignature', + ExpiredToken = 'ExpiredToken', SignatureDoesNotMatch = 'SignatureDoesNotMatch', AccessDenied = 'AccessDenied', ResourceLocked = 'ResourceLocked', @@ -145,9 +146,9 @@ export const ERRORS = { ExpiredSignature: (e?: Error) => new StorageBackendError({ - code: ErrorCode.InvalidSignature, + code: ErrorCode.ExpiredToken, httpStatusCode: 400, - message: 'Expired signature', + message: 'The provided token has expired.', originalError: e, }), diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index f54dc147..169cd731 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -4,6 +4,7 @@ import { ERRORS } from '../../errors' interface SignatureV4Options { enforceRegion: boolean allowForwardedHeader?: boolean + nonCanonicalForwardedHost?: string credentials: Omit & { secretKey: string } } @@ -11,6 +12,9 @@ export interface ClientSignature { credentials: Credentials signature: string signedHeaders: string[] + sessionToken?: string + longDate: string + contentSha?: string } interface SignatureRequest { @@ -20,9 +24,7 @@ interface SignatureRequest { method: string query?: Record prefix?: string - credentials: Credentials - signature: string - signedHeaders: string[] + clientSignature: ClientSignature } interface Credentials { @@ -53,24 +55,36 @@ export const ALWAYS_UNSIGNABLE_HEADERS = { 'x-amzn-trace-id': true, } +export const ALWAYS_UNSIGNABLE_QUERY_PARAMS = { + 'X-Amz-Signature': true, +} + export class SignatureV4 { public readonly serverCredentials: SignatureV4Options['credentials'] enforceRegion: boolean allowForwardedHeader?: boolean + nonCanonicalForwardedHost?: string constructor(options: SignatureV4Options) { this.serverCredentials = options.credentials this.enforceRegion = options.enforceRegion this.allowForwardedHeader = options.allowForwardedHeader + this.nonCanonicalForwardedHost = options.nonCanonicalForwardedHost } - static parseAuthorizationHeader(header: string) { - const parts = header.split(' ') + static parseAuthorizationHeader(headers: Record) { + const clientSignature = headers.authorization + + if (typeof clientSignature !== 'string') { + throw ERRORS.InvalidSignature('Missing authorization header') + } + + const parts = clientSignature.split(' ') if (parts[0] !== 'AWS4-HMAC-SHA256') { throw ERRORS.InvalidSignature('Unsupported authorization type') } - const params = header + const params = clientSignature .replace('AWS4-HMAC-SHA256 ', '') .split(',') .reduce((values, value) => { @@ -81,14 +95,18 @@ export class SignatureV4 { const credentialPart = params.get('Credential') const signedHeadersPart = params.get('SignedHeaders') - const signaturePart = params.get('Signature') + const signature = params.get('Signature') + const longDate = headers['x-amz-date'] + const contentSha = headers['x-amz-content-sha256'] + const sessionToken = headers['x-amz-security-token'] - if (!credentialPart || !signedHeadersPart || !signaturePart) { + if (!validateTypeOfStrings(credentialPart, signedHeadersPart, signature, longDate)) { throw ERRORS.InvalidSignature('Invalid signature format') } - const signedHeaders = signedHeadersPart.split(';') || [] - const credentialsPart = credentialPart.split('/') + const signedHeaders = signedHeadersPart?.split(';') || [] + + const credentialsPart = credentialPart?.split('/') || [] if (credentialsPart.length !== 5) { throw ERRORS.InvalidSignature('Invalid credentials') @@ -104,7 +122,74 @@ export class SignatureV4 { service, }, signedHeaders, - signature: signaturePart, + signature, + longDate, + contentSha, + sessionToken, + } + } + + static parseQuerySignature(query: Record) { + const credentialPart = query['X-Amz-Credential'] + const signedHeaders = query['X-Amz-SignedHeaders'] + const signature = query['X-Amz-Signature'] + const longDate = query['X-Amz-Date'] + const contentSha = query['X-Amz-Content-Sha256'] + const sessionToken = query['X-Amz-Security-Token'] + const expires = query['X-Amz-Expires'] + + if (!validateTypeOfStrings(credentialPart, signedHeaders, signature)) { + throw ERRORS.InvalidSignature('Invalid signature format') + } + + if (expires) { + const expiresSec = parseInt(expires, 10) + if (isNaN(expiresSec) || expiresSec < 0) { + throw ERRORS.InvalidSignature('Invalid expiration') + } + + if (typeof longDate !== 'string') { + throw ERRORS.InvalidSignature('Invalid date') + } + + const isoLongDate = longDate.replace( + /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/, + '$1-$2-$3T$4:$5:$6Z' + ) + + const expirationDate = new Date(isoLongDate) + + if (isNaN(expirationDate.getTime())) { + throw ERRORS.InvalidSignature('Invalid date') + } + expirationDate.setSeconds(expirationDate.getSeconds() + expiresSec) + + const isExpired = expirationDate < new Date() + if (isExpired) { + throw ERRORS.ExpiredSignature() + } + } + + const credentialsPart = credentialPart.split('/') + + if (credentialsPart.length !== 5) { + throw ERRORS.InvalidSignature('Invalid credentials') + } + + const [accessKey, shortDate, region, service] = credentialsPart + + return { + credentials: { + accessKey, + shortDate, + region, + service, + }, + signedHeaders: signedHeaders.split(';'), + signature, + longDate, + contentSha, + sessionToken, } } @@ -115,25 +200,23 @@ export class SignatureV4 { } sign(request: SignatureRequest) { - const authorizationHeader = this.getHeader(request, 'authorization') - if (!authorizationHeader) { - throw ERRORS.AccessDenied('Missing authorization header') - } - - if (request.credentials.accessKey !== this.serverCredentials.accessKey) { + if (request.clientSignature.credentials.accessKey !== this.serverCredentials.accessKey) { throw ERRORS.AccessDenied('Invalid Access Key') } // Ensure the region and service match the expected values - if (this.enforceRegion && request.credentials.region !== this.serverCredentials.region) { + if ( + this.enforceRegion && + request.clientSignature.credentials.region !== this.serverCredentials.region + ) { throw ERRORS.AccessDenied('Invalid Region') } - if (request.credentials.service !== this.serverCredentials.service) { + if (request.clientSignature.credentials.service !== this.serverCredentials.service) { throw ERRORS.AccessDenied('Invalid Service') } - const longDate = request.headers['x-amz-date'] as string + const longDate = request.clientSignature.longDate if (!longDate) { throw ERRORS.AccessDenied('No date header provided') } @@ -144,20 +227,25 @@ export class SignatureV4 { // - the region set in the env if ( !this.enforceRegion && - !['auto', 'us-east-1', this.serverCredentials.region, ''].includes(request.credentials.region) + !['auto', 'us-east-1', this.serverCredentials.region, ''].includes( + request.clientSignature.credentials.region + ) ) { throw ERRORS.AccessDenied('Invalid Region') } const selectedRegion = this.enforceRegion ? this.serverCredentials.region - : request.credentials.region + : request.clientSignature.credentials.region // Construct the Canonical Request and String to Sign - const canonicalRequest = this.constructCanonicalRequest(request, request.signedHeaders) + const canonicalRequest = this.constructCanonicalRequest( + request, + request.clientSignature.signedHeaders + ) const stringToSign = this.constructStringToSign( longDate, - request.credentials.shortDate, + request.clientSignature.credentials.shortDate, selectedRegion, this.serverCredentials.service, canonicalRequest @@ -165,25 +253,22 @@ export class SignatureV4 { const signingKey = this.signingKey( this.serverCredentials.secretKey, - request.credentials.shortDate, + request.clientSignature.credentials.shortDate, selectedRegion, this.serverCredentials.service ) return { - clientSignature: request.signature, + clientSignature: request.clientSignature.signature, serverSignature: this.hmac(signingKey, stringToSign).toString('hex'), } } getPayloadHash(request: SignatureRequest) { - const headers = request.headers const body = request.body - for (const headerName of Object.keys(headers)) { - if (headerName.toLowerCase() === 'x-amz-content-sha256') { - return headers[headerName] - } + if (request.clientSignature.contentSha) { + return request.clientSignature.contentSha } const contentLenght = parseInt(this.getHeader(request, 'content-length') || '0', 10) @@ -242,6 +327,7 @@ export class SignatureV4 { .pathname const canonicalQueryString = Object.keys((request.query as object) || {}) + .filter((key) => !(key in ALWAYS_UNSIGNABLE_QUERY_PARAMS)) .sort() .map( (key) => @@ -269,6 +355,17 @@ export class SignatureV4 { } } + if (this.nonCanonicalForwardedHost) { + const xForwardedHost = this.getHeader( + request, + this.nonCanonicalForwardedHost.toLowerCase() + ) + + if (xForwardedHost) { + return `host:${xForwardedHost.toLowerCase()}` + } + } + const xForwardedHost = this.getHeader(request, 'x-forwarded-host') if (xForwardedHost) { return `host:${xForwardedHost.toLowerCase()}` @@ -296,3 +393,9 @@ export class SignatureV4 { return item } } + +function validateTypeOfStrings(...values: any[]) { + return values.every((value) => { + return typeof value === 'string' + }) +} diff --git a/src/test/s3-protocol.test.ts b/src/test/s3-protocol.test.ts index 558daefb..3a450110 100644 --- a/src/test/s3-protocol.test.ts +++ b/src/test/s3-protocol.test.ts @@ -28,15 +28,9 @@ import { FastifyInstance } from 'fastify' import { Upload } from '@aws-sdk/lib-storage' import { ReadableStreamBuffer } from 'stream-buffers' import { randomUUID } from 'crypto' +import { getSignedUrl } from '@aws-sdk/s3-request-presigner' -const { - s3ProtocolAccessKeySecret, - s3ProtocolAccessKeyId, - storageS3Region, - tenantId, - anonKey, - serviceKey, -} = getConfig() +const { s3ProtocolAccessKeySecret, s3ProtocolAccessKeyId, storageS3Region } = getConfig() async function createBucket(client: S3Client, name?: string, publicRead = true) { let bucketName: string @@ -1083,5 +1077,57 @@ describe('S3 Protocol', () => { expect(parts.Parts?.length).toBe(1) }) }) + + describe('S3 Presigned URL', () => { + it('can call a simple method with presigned url', async () => { + const bucket = await createBucket(client) + const bucketVersioningCommand = new GetBucketVersioningCommand({ + Bucket: bucket, + }) + const signedUrl = await getSignedUrl(client, bucketVersioningCommand, { expiresIn: 100 }) + const resp = await fetch(signedUrl) + + expect(resp.ok).toBeTruthy() + }) + + it('cannot request a presigned url if expired', async () => { + const bucket = await createBucket(client) + const bucketVersioningCommand = new GetBucketVersioningCommand({ + Bucket: bucket, + }) + const signedUrl = await getSignedUrl(client, bucketVersioningCommand, { expiresIn: 1 }) + await new Promise((resolve) => setTimeout(resolve, 1500)) + const resp = await fetch(signedUrl) + + expect(resp.ok).toBeFalsy() + expect(resp.status).toBe(400) + }) + + it('can upload with presigned URL', async () => { + const bucket = await createBucket(client) + const key = 'test-1.jpg' + const body = Buffer.alloc(1024 * 1024 * 2) + + const uploadUrl = await getSignedUrl( + client, + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + { expiresIn: 100 } + ) + + const resp = await fetch(uploadUrl, { + method: 'PUT', + body: body, + headers: { + 'Content-Length': body.length.toString(), + }, + }) + + expect(resp.ok).toBeTruthy() + }) + }) }) })