Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix s3 file driver xattr #453

Merged
merged 1 commit into from
Apr 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@
"xml2js": "^0.6.2"
},
"devDependencies": {
"@types/ajv": "^1.0.0",
"@types/async-retry": "^1.4.5",
"@types/busboy": "^1.3.0",
"@types/crypto-js": "^4.1.1",
Expand Down
9 changes: 7 additions & 2 deletions src/http/plugins/xml.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@ import xml from 'xml2js'
// @ts-ignore
import xmlBodyParser from 'fastify-xml-body-parser'

export const jsonToXml = fastifyPlugin(async function (fastify: FastifyInstance) {
export const jsonToXml = fastifyPlugin(async function (
fastify: FastifyInstance,
opts: { disableContentParser?: boolean }
) {
fastify.register(accepts)

fastify.register(xmlBodyParser)
if (!opts.disableContentParser) {
fastify.register(xmlBodyParser)
}
fastify.addHook('preSerialization', async (req, res, payload) => {
const accept = req.accepts()
if (
Expand Down
48 changes: 29 additions & 19 deletions src/http/routes/s3/commands/upload-part.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,25 +58,35 @@ const UploadPartInput = {
} as const

export default function UploadPart(s3Router: S3Router) {
s3Router.put('/:Bucket/*?uploadId&partNumber', UploadPartInput, (req, ctx) => {
const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
s3Router.put(
'/:Bucket/*?uploadId&partNumber',
UploadPartInput,
(req, ctx) => {
const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)

return s3Protocol.uploadPart({
Body: ctx.req.raw,
UploadId: req.Querystring?.uploadId,
Bucket: req.Params.Bucket,
Key: req.Params['*'],
PartNumber: req.Querystring?.partNumber,
ContentLength: req.Headers?.['content-length'],
})
})
return s3Protocol.uploadPart({
Body: ctx.req.raw,
UploadId: req.Querystring?.uploadId,
Bucket: req.Params.Bucket,
Key: req.Params['*'],
PartNumber: req.Querystring?.partNumber,
ContentLength: req.Headers?.['content-length'],
})
},
{ disableContentTypeParser: true }
)

s3Router.put('/:Bucket/*', PutObjectInput, (req, ctx) => {
const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
return s3Protocol.putObject({
Body: ctx.req as any,
Bucket: req.Params.Bucket,
Key: req.Params['*'],
})
})
s3Router.put(
'/:Bucket/*',
PutObjectInput,
(req, ctx) => {
const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
return s3Protocol.putObject({
Body: ctx.req as any,
Bucket: req.Params.Bucket,
Key: req.Params['*'],
})
},
{ disableContentTypeParser: true }
)
}
57 changes: 37 additions & 20 deletions src/http/routes/s3/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,6 @@ import { s3ErrorHandler } from './error-handler'

export default async function routes(fastify: FastifyInstance) {
fastify.register(async (fastify) => {
fastify.register(jsonToXml)
fastify.register(signatureV4)
fastify.register(db)
fastify.register(storage)

const s3Router = getRouter()
const s3Routes = s3Router.routes()

Expand Down Expand Up @@ -74,28 +69,50 @@ export default async function routes(fastify: FastifyInstance) {
return reply.status(404).send()
}

fastify[method](
routePath,
{
validatorCompiler: () => () => true,
exposeHeadRoute: false,
errorHandler: s3ErrorHandler,
},
routeHandler
)

// handle optional trailing slash
if (!routePath.endsWith('*') && !routePath.endsWith('/')) {
fastify[method](
routePath + '/',
fastify.register(async (localFastify) => {
const disableContentParser = routesByMethod?.some(
(route) => route.disableContentTypeParser
)

if (disableContentParser) {
localFastify.addContentTypeParser(
['application/json', 'text/plain', 'application/xml'],
function (request, payload, done) {
done(null)
}
)
}

fastify.register(jsonToXml, {
disableContentParser,
})
fastify.register(signatureV4)
fastify.register(db)
fastify.register(storage)

localFastify[method](
routePath,
{
validatorCompiler: () => () => true,
exposeHeadRoute: false,
errorHandler: s3ErrorHandler,
},
routeHandler
)
}

// handle optional trailing slash
if (!routePath.endsWith('*') && !routePath.endsWith('/')) {
localFastify[method](
routePath + '/',
{
validatorCompiler: () => () => true,
exposeHeadRoute: false,
errorHandler: s3ErrorHandler,
},
routeHandler
)
}
})
})
})
})
Expand Down
49 changes: 38 additions & 11 deletions src/http/routes/s3/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,14 @@ type Route<S extends Schema, Context> = {
headersMatches: string[]
handler?: Handler<S, Context>
schema: S
disableContentTypeParser?: boolean
compiledSchema: () => ValidateFunction<JTDDataType<S>>
}

interface RouteOptions {
disableContentTypeParser?: boolean
}

export class Router<Context = unknown, S extends Schema = Schema> {
protected _routes: Map<string, Route<S, Context>[]> = new Map<string, Route<S, Context>[]>()

Expand All @@ -117,7 +122,8 @@ export class Router<Context = unknown, S extends Schema = Schema> {
method: HTTPMethod,
url: string,
schema: R,
handler: Handler<R, Context>
handler: Handler<R, Context>,
options?: { disableContentTypeParser?: boolean }
) {
const { query, headers } = this.parseQueryString(url)
const normalizedUrl = url.split('?')[0].split('|')[0]
Expand Down Expand Up @@ -160,6 +166,7 @@ export class Router<Context = unknown, S extends Schema = Schema> {
schema: schema,
compiledSchema: () => this.ajv.getSchema(method + url) as ValidateFunction<JTDDataType<R>>,
handler: handler as Handler<R, Context>,
disableContentTypeParser: options?.disableContentTypeParser,
} as const

if (!existingPath) {
Expand All @@ -171,24 +178,44 @@ export class Router<Context = unknown, S extends Schema = Schema> {
this._routes.set(normalizedUrl, existingPath)
}

get<R extends S>(url: string, schema: R, handler: Handler<R, Context>) {
this.registerRoute('get', url, schema, handler as any)
get<R extends S>(url: string, schema: R, handler: Handler<R, Context>, options?: RouteOptions) {
this.registerRoute('get', url, schema, handler as any, options)
}

post<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
this.registerRoute('post', url, schema, handler as any)
post<R extends S = S>(
url: string,
schema: R,
handler: Handler<R, Context>,
options?: RouteOptions
) {
this.registerRoute('post', url, schema, handler as any, options)
}

put<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
this.registerRoute('put', url, schema, handler as any)
put<R extends S = S>(
url: string,
schema: R,
handler: Handler<R, Context>,
options?: RouteOptions
) {
this.registerRoute('put', url, schema, handler as any, options)
}

delete<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
this.registerRoute('delete', url, schema, handler as any)
delete<R extends S = S>(
url: string,
schema: R,
handler: Handler<R, Context>,
options?: RouteOptions
) {
this.registerRoute('delete', url, schema, handler as any, options)
}

head<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
this.registerRoute('head', url, schema, handler as any)
head<R extends S = S>(
url: string,
schema: R,
handler: Handler<R, Context>,
options?: RouteOptions
) {
this.registerRoute('head', url, schema, handler as any, options)
}

parseQueryMatch(query: string) {
Expand Down
9 changes: 6 additions & 3 deletions src/storage/backend/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ const METADATA_ATTR_KEYS = {
darwin: {
'cache-control': 'com.apple.metadata.supabase.cache-control',
'content-type': 'com.apple.metadata.supabase.content-type',
etag: 'com.apple.metadata.supabase.etag',
},
linux: {
'cache-control': 'user.supabase.cache-control',
'content-type': 'user.supabase.content-type',
etag: 'user.supabase.content-type',
},
}

Expand Down Expand Up @@ -304,8 +306,8 @@ export class FileBackend implements StorageBackendAdapter {
await pipeline(body, writeStream)

const etag = await fileChecksum(multipartFile)

await this.setMetadataAttr(multipartFile, 'etag', etag)
const platform = process.platform == 'darwin' ? 'darwin' : 'linux'
await this.setMetadataAttr(multipartFile, METADATA_ATTR_KEYS[platform]['etag'], etag)

return { ETag: etag }
}
Expand Down Expand Up @@ -336,7 +338,8 @@ export class FileBackend implements StorageBackendAdapter {
const partExists = await fsExtra.pathExists(partFilePath)

if (partExists) {
const etag = await this.getMetadataAttr(partFilePath, 'etag')
const platform = process.platform == 'darwin' ? 'darwin' : 'linux'
const etag = await this.getMetadataAttr(partFilePath, METADATA_ATTR_KEYS[platform]['etag'])
if (etag === part.ETag) {
return partFilePath
}
Expand Down
4 changes: 2 additions & 2 deletions src/storage/object.ts
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ export class ObjectStorage {
'bucket_id,metadata,version'
)

if (sourceKey === destinationKey) {
if (s3SourceKey === s3DestinationKey) {
return {
destObject: originObject,
httpStatusCode: 200,
Expand Down Expand Up @@ -374,7 +374,7 @@ export class ObjectStorage {
.asSuperUser()
.findObject(this.bucketId, sourceObjectName, 'id, version')

if (sourceObjectName === destinationObjectName) {
if (s3SourceKey === s3DestinationKey) {
return {
destObject: sourceObj,
}
Expand Down
Loading