fix: s3 file driver xattr

package.json

@@ -71,7 +71,6 @@
     "xml2js": "^0.6.2"
   },
   "devDependencies": {
-    "@types/ajv": "^1.0.0",
     "@types/async-retry": "^1.4.5",
     "@types/busboy": "^1.3.0",
     "@types/crypto-js": "^4.1.1",

src/http/plugins/xml.ts

@@ -8,10 +8,15 @@ import xml from 'xml2js'
 // @ts-ignore
 import xmlBodyParser from 'fastify-xml-body-parser'
 
-export const jsonToXml = fastifyPlugin(async function (fastify: FastifyInstance) {
+export const jsonToXml = fastifyPlugin(async function (
+  fastify: FastifyInstance,
+  opts: { disableContentParser?: boolean }
+) {
   fastify.register(accepts)
 
-  fastify.register(xmlBodyParser)
+  if (!opts.disableContentParser) {
+    fastify.register(xmlBodyParser)
+  }
   fastify.addHook('preSerialization', async (req, res, payload) => {
     const accept = req.accepts()
     if (

src/http/routes/s3/commands/upload-part.ts

@@ -58,25 +58,35 @@ const UploadPartInput = {
 } as const
 
 export default function UploadPart(s3Router: S3Router) {
-  s3Router.put('/:Bucket/*?uploadId&partNumber', UploadPartInput, (req, ctx) => {
-    const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
+  s3Router.put(
+    '/:Bucket/*?uploadId&partNumber',
+    UploadPartInput,
+    (req, ctx) => {
+      const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
 
-    return s3Protocol.uploadPart({
-      Body: ctx.req.raw,
-      UploadId: req.Querystring?.uploadId,
-      Bucket: req.Params.Bucket,
-      Key: req.Params['*'],
-      PartNumber: req.Querystring?.partNumber,
-      ContentLength: req.Headers?.['content-length'],
-    })
-  })
+      return s3Protocol.uploadPart({
+        Body: ctx.req.raw,
+        UploadId: req.Querystring?.uploadId,
+        Bucket: req.Params.Bucket,
+        Key: req.Params['*'],
+        PartNumber: req.Querystring?.partNumber,
+        ContentLength: req.Headers?.['content-length'],
+      })
+    },
+    { disableContentTypeParser: true }
+  )
 
-  s3Router.put('/:Bucket/*', PutObjectInput, (req, ctx) => {
-    const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
-    return s3Protocol.putObject({
-      Body: ctx.req as any,
-      Bucket: req.Params.Bucket,
-      Key: req.Params['*'],
-    })
-  })
+  s3Router.put(
+    '/:Bucket/*',
+    PutObjectInput,
+    (req, ctx) => {
+      const s3Protocol = new S3ProtocolHandler(ctx.storage, ctx.tenantId, ctx.owner)
+      return s3Protocol.putObject({
+        Body: ctx.req as any,
+        Bucket: req.Params.Bucket,
+        Key: req.Params['*'],
+      })
+    },
+    { disableContentTypeParser: true }
+  )
 }

src/http/routes/s3/index.ts

@@ -5,11 +5,6 @@ import { s3ErrorHandler } from './error-handler'
 
 export default async function routes(fastify: FastifyInstance) {
   fastify.register(async (fastify) => {
-    fastify.register(jsonToXml)
-    fastify.register(signatureV4)
-    fastify.register(db)
-    fastify.register(storage)
-
     const s3Router = getRouter()
     const s3Routes = s3Router.routes()
 
@@ -74,28 +69,50 @@ export default async function routes(fastify: FastifyInstance) {
           return reply.status(404).send()
         }
 
-        fastify[method](
-          routePath,
-          {
-            validatorCompiler: () => () => true,
-            exposeHeadRoute: false,
-            errorHandler: s3ErrorHandler,
-          },
-          routeHandler
-        )
-
-        // handle optional trailing slash
-        if (!routePath.endsWith('*') && !routePath.endsWith('/')) {
-          fastify[method](
-            routePath + '/',
+        fastify.register(async (localFastify) => {
+          const disableContentParser = routesByMethod?.some(
+            (route) => route.disableContentTypeParser
+          )
+
+          fastify.register(jsonToXml, {
+            disableContentParser,
+          })
+          fastify.register(signatureV4)
+          fastify.register(db)
+          fastify.register(storage)
+
+          if (disableContentParser) {
+            localFastify.addContentTypeParser(
+              ['application/json', 'text/plain', 'application/xml'],
+              function (request, payload, done) {
+                done(null)
+              }
+            )
+          }
+
+          localFastify[method](
+            routePath,
             {
               validatorCompiler: () => () => true,
               exposeHeadRoute: false,
               errorHandler: s3ErrorHandler,
             },
             routeHandler
           )
-        }
+
+          // handle optional trailing slash
+          if (!routePath.endsWith('*') && !routePath.endsWith('/')) {
+            localFastify[method](
+              routePath + '/',
+              {
+                validatorCompiler: () => () => true,
+                exposeHeadRoute: false,
+                errorHandler: s3ErrorHandler,
+              },
+              routeHandler
+            )
+          }
+        })
       })
     })
   })

src/http/routes/s3/router.ts

@@ -98,9 +98,14 @@ type Route<S extends Schema, Context> = {
   headersMatches: string[]
   handler?: Handler<S, Context>
   schema: S
+  disableContentTypeParser?: boolean
   compiledSchema: () => ValidateFunction<JTDDataType<S>>
 }
 
+interface RouteOptions {
+  disableContentTypeParser?: boolean
+}
+
 export class Router<Context = unknown, S extends Schema = Schema> {
   protected _routes: Map<string, Route<S, Context>[]> = new Map<string, Route<S, Context>[]>()
 
@@ -117,7 +122,8 @@ export class Router<Context = unknown, S extends Schema = Schema> {
     method: HTTPMethod,
     url: string,
     schema: R,
-    handler: Handler<R, Context>
+    handler: Handler<R, Context>,
+    options?: { disableContentTypeParser?: boolean }
   ) {
     const { query, headers } = this.parseQueryString(url)
     const normalizedUrl = url.split('?')[0].split('|')[0]
@@ -160,6 +166,7 @@ export class Router<Context = unknown, S extends Schema = Schema> {
       schema: schema,
       compiledSchema: () => this.ajv.getSchema(method + url) as ValidateFunction<JTDDataType<R>>,
       handler: handler as Handler<R, Context>,
+      disableContentTypeParser: options?.disableContentTypeParser,
     } as const
 
     if (!existingPath) {
@@ -171,24 +178,44 @@ export class Router<Context = unknown, S extends Schema = Schema> {
     this._routes.set(normalizedUrl, existingPath)
   }
 
-  get<R extends S>(url: string, schema: R, handler: Handler<R, Context>) {
-    this.registerRoute('get', url, schema, handler as any)
+  get<R extends S>(url: string, schema: R, handler: Handler<R, Context>, options?: RouteOptions) {
+    this.registerRoute('get', url, schema, handler as any, options)
   }
 
-  post<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
-    this.registerRoute('post', url, schema, handler as any)
+  post<R extends S = S>(
+    url: string,
+    schema: R,
+    handler: Handler<R, Context>,
+    options?: RouteOptions
+  ) {
+    this.registerRoute('post', url, schema, handler as any, options)
   }
 
-  put<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
-    this.registerRoute('put', url, schema, handler as any)
+  put<R extends S = S>(
+    url: string,
+    schema: R,
+    handler: Handler<R, Context>,
+    options?: RouteOptions
+  ) {
+    this.registerRoute('put', url, schema, handler as any, options)
   }
 
-  delete<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
-    this.registerRoute('delete', url, schema, handler as any)
+  delete<R extends S = S>(
+    url: string,
+    schema: R,
+    handler: Handler<R, Context>,
+    options?: RouteOptions
+  ) {
+    this.registerRoute('delete', url, schema, handler as any, options)
   }
 
-  head<R extends S = S>(url: string, schema: R, handler: Handler<R, Context>) {
-    this.registerRoute('head', url, schema, handler as any)
+  head<R extends S = S>(
+    url: string,
+    schema: R,
+    handler: Handler<R, Context>,
+    options?: RouteOptions
+  ) {
+    this.registerRoute('head', url, schema, handler as any, options)
   }
 
   parseQueryMatch(query: string) {

src/storage/backend/file.ts

@@ -29,10 +29,12 @@ const METADATA_ATTR_KEYS = {
   darwin: {
     'cache-control': 'com.apple.metadata.supabase.cache-control',
     'content-type': 'com.apple.metadata.supabase.content-type',
+    etag: 'com.apple.metadata.supabase.etag',
   },
   linux: {
     'cache-control': 'user.supabase.cache-control',
     'content-type': 'user.supabase.content-type',
+    etag: 'user.supabase.content-type',
   },
 }
 
@@ -187,6 +189,7 @@ export class FileBackend implements StorageBackendAdapter {
     destinationVersion: string
   ): Promise<Pick<ObjectMetadata, 'httpStatusCode' | 'eTag' | 'lastModified'>> {
     const srcFile = path.resolve(this.filePath, withOptionalVersion(`${bucket}/${source}`, version))
+    console.log('source file', srcFile)
     const destFile = path.resolve(
       this.filePath,
       withOptionalVersion(`${bucket}/${destination}`, destinationVersion)
@@ -304,8 +307,8 @@ export class FileBackend implements StorageBackendAdapter {
     await pipeline(body, writeStream)
 
     const etag = await fileChecksum(multipartFile)
-
-    await this.setMetadataAttr(multipartFile, 'etag', etag)
+    const platform = process.platform == 'darwin' ? 'darwin' : 'linux'
+    await this.setMetadataAttr(multipartFile, METADATA_ATTR_KEYS[platform]['etag'], etag)
 
     return { ETag: etag }
   }
@@ -336,7 +339,8 @@ export class FileBackend implements StorageBackendAdapter {
       const partExists = await fsExtra.pathExists(partFilePath)
 
       if (partExists) {
-        const etag = await this.getMetadataAttr(partFilePath, 'etag')
+        const platform = process.platform == 'darwin' ? 'darwin' : 'linux'
+        const etag = await this.getMetadataAttr(partFilePath, METADATA_ATTR_KEYS[platform]['etag'])
         if (etag === part.ETag) {
           return partFilePath
         }

src/storage/object.ts

@@ -275,7 +275,7 @@ export class ObjectStorage {
         'bucket_id,metadata,version'
       )
 
-      if (sourceKey === destinationKey) {
+      if (s3SourceKey === s3DestinationKey) {
         return {
           destObject: originObject,
           httpStatusCode: 200,
@@ -374,7 +374,7 @@ export class ObjectStorage {
       .asSuperUser()
       .findObject(this.bucketId, sourceObjectName, 'id, version')
 
-    if (sourceObjectName === destinationObjectName) {
+    if (s3SourceKey === s3DestinationKey) {
       return {
         destObject: sourceObj,
       }