Merge pull request #29 from supabase/feat/support-binary-and-stream-u…

…ploads

Feat/support binary and stream uploads

src/app.ts

@@ -23,6 +23,10 @@ const build = (opts: buildOpts = {}): FastifyInstance => {
     throwFileSizeLimit: false,
   })
 
+  app.addContentTypeParser('*', function (request, payload, done) {
+    done(null)
+  })
+
   // kong should take care of cors
   // app.register(fastifyCors)
 

src/routes/object/createObject.ts

@@ -1,6 +1,7 @@
-import { FastifyInstance } from 'fastify'
+import { ServiceOutputTypes } from '@aws-sdk/client-s3'
+import { FastifyInstance, RequestGenericInterface } from 'fastify'
 import { FromSchema } from 'json-schema-to-ts'
-import { AuthenticatedRequest, Obj, ObjectMetadata } from '../../types/types'
+import { Obj, ObjectMetadata } from '../../types/types'
 import { getOwner, getPostgrestClient, isValidKey, transformPostgrestError } from '../../utils'
 import { getConfig } from '../../utils/config'
 import { createDefaultSchema, createResponse } from '../../utils/generic-routes'
@@ -27,8 +28,13 @@ const successResponseSchema = {
   },
   required: ['Key'],
 }
-interface createObjectRequestInterface extends AuthenticatedRequest {
+interface createObjectRequestInterface extends RequestGenericInterface {
   Params: FromSchema<typeof createObjectParamsSchema>
+  Headers: {
+    authorization: string
+    'content-type': string
+    'cache-control'?: string
+  }
 }
 
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
@@ -50,16 +56,16 @@ export default async function routes(fastify: FastifyInstance) {
       // check if the user is able to insert that row
       const authHeader = request.headers.authorization
       const jwt = authHeader.substring('Bearer '.length)
-      const data = await request.file()
 
-      // Can't seem to get the typing to work properly
-      // https://github.com/fastify/fastify-multipart/issues/162
-      /* @ts-expect-error: https://github.com/aws/aws-sdk-js-v3/issues/2085 */
-      const cacheTime = data.fields.cacheControl?.value
-      const cacheControl: string = cacheTime ? `max-age=${cacheTime}` : 'no-cache'
+      const contentType = request.headers['content-type']
+      request.log.info(`content-type is ${contentType}`)
 
       const { bucketName } = request.params
       const objectName = request.params['*']
+      const path = `${bucketName}/${objectName}`
+      const s3Key = `${projectRef}/${path}`
+      let mimeType: string, cacheControl: string, isTruncated: boolean
+      let uploadResult: ServiceOutputTypes
 
       if (!isValidKey(objectName) || !isValidKey(bucketName)) {
         return response
@@ -105,22 +111,45 @@ export default async function routes(fastify: FastifyInstance) {
       request.log.info({ results }, 'results')
 
       // if successfully inserted, upload to s3
-      const path = `${bucketName}/${objectName}`
-      const s3Key = `${projectRef}/${path}`
-      const uploadResult = await uploadObject(
-        client,
-        globalS3Bucket,
-        s3Key,
-        data.file,
-        data.mimetype,
-        cacheControl
-      )
-
-      // since we are using streams, fastify can't throw the error reliably
-      // busboy sets the truncated property on streams if limit was exceeded
-      // https://github.com/fastify/fastify-multipart/issues/196#issuecomment-782847791
-      /* @ts-expect-error: busboy doesn't export proper types */
-      const isTruncated = data.file.truncated
+      if (contentType?.startsWith('multipart/form-data')) {
+        const data = await request.file()
+
+        // Can't seem to get the typing to work properly
+        // https://github.com/fastify/fastify-multipart/issues/162
+        /* @ts-expect-error: https://github.com/aws/aws-sdk-js-v3/issues/2085 */
+        const cacheTime = data.fields.cacheControl?.value
+        cacheControl = cacheTime ? `max-age=${cacheTime}` : 'no-cache'
+        mimeType = data.mimetype
+        uploadResult = await uploadObject(
+          client,
+          globalS3Bucket,
+          s3Key,
+          data.file,
+          mimeType,
+          cacheControl
+        )
+        // since we are using streams, fastify can't throw the error reliably
+        // busboy sets the truncated property on streams if limit was exceeded
+        // https://github.com/fastify/fastify-multipart/issues/196#issuecomment-782847791
+        /* @ts-expect-error: busboy doesn't export proper types */
+        isTruncated = data.file.truncated
+      } else {
+        // just assume its a binary file
+        mimeType = request.headers['content-type']
+        cacheControl = request.headers['cache-control'] ?? 'no-cache'
+
+        uploadResult = await uploadObject(
+          client,
+          globalS3Bucket,
+          s3Key,
+          request.raw,
+          mimeType,
+          cacheControl
+        )
+        const { fileSizeLimit } = getConfig()
+        // @todo more secure to get this from the stream or from s3 in the next step
+        isTruncated = Number(request.headers['content-length']) > fileSizeLimit
+      }
       if (isTruncated) {
         // undo operations as super user
         await superUserPostgrest
@@ -148,7 +177,7 @@ export default async function routes(fastify: FastifyInstance) {
       const objectMetadata = await headObject(client, globalS3Bucket, s3Key)
       // update content-length as super user since user may not have update permissions
       const metadata: ObjectMetadata = {
-        mimetype: data.mimetype,
+        mimetype: mimeType,
         cacheControl,
         size: objectMetadata.ContentLength,
       }

src/test/object.test.ts

@@ -150,8 +150,9 @@ describe('testing GET object', () => {
 })
 /*
  * POST /object/:id
+ * multipart upload
  */
-describe('testing POST object', () => {
+describe('testing POST object via multipart upload', () => {
   test('check if RLS policies are respected: authenticated user is able to upload authenticated resource', async () => {
     const form = new FormData()
     form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`))
@@ -243,6 +244,119 @@ describe('testing POST object', () => {
   })
 })
 
+/*
+ * POST /object/:id
+ * binary upload
+ */
+describe('testing POST object via binary upload', () => {
+  test('check if RLS policies are respected: authenticated user is able to upload authenticated resource', async () => {
+    const path = './src/test/assets/sadcat.jpg'
+    const { size } = fs.statSync(path)
+
+    const headers = {
+      authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`,
+      'Content-Length': size,
+      'Content-Type': 'image/jpeg',
+    }
+
+    const response = await app().inject({
+      method: 'POST',
+      url: '/object/bucket2/authenticated/binary-casestudy1.png',
+      headers,
+      payload: fs.createReadStream(path),
+    })
+    expect(response.statusCode).toBe(200)
+    expect(mockUploadObject).toBeCalled()
+    expect(response.body).toBe(`{"Key":"bucket2/authenticated/binary-casestudy1.png"}`)
+  })
+
+  test('check if RLS policies are respected: anon user is not able to upload authenticated resource', async () => {
+    const path = './src/test/assets/sadcat.jpg'
+    const { size } = fs.statSync(path)
+
+    const headers = {
+      authorization: `Bearer ${anonKey}`,
+      'Content-Length': size,
+      'Content-Type': 'image/jpeg',
+    }
+
+    const response = await app().inject({
+      method: 'POST',
+      url: '/object/bucket2/authenticated/binary-casestudy.png',
+      headers,
+      payload: fs.createReadStream(path),
+    })
+    expect(response.statusCode).toBe(400)
+    expect(mockUploadObject).not.toHaveBeenCalled()
+    expect(response.body).toBe(
+      JSON.stringify({
+        statusCode: '42501',
+        error: '',
+        message: 'new row violates row-level security policy for table "objects"',
+      })
+    )
+  })
+
+  test('check if RLS policies are respected: user is not able to upload a resource without Auth header', async () => {
+    const path = './src/test/assets/sadcat.jpg'
+    const { size } = fs.statSync(path)
+
+    const headers = {
+      'Content-Length': size,
+      'Content-Type': 'image/jpeg',
+    }
+
+    const response = await app().inject({
+      method: 'POST',
+      url: '/object/bucket2/authenticated/binary-casestudy1.png',
+      headers,
+      payload: fs.createReadStream(path),
+    })
+    expect(response.statusCode).toBe(400)
+    expect(mockUploadObject).not.toHaveBeenCalled()
+  })
+
+  test('return 400 when uploading to a non existent bucket', async () => {
+    const path = './src/test/assets/sadcat.jpg'
+    const { size } = fs.statSync(path)
+
+    const headers = {
+      authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`,
+      'Content-Length': size,
+      'Content-Type': 'image/jpeg',
+    }
+
+    const response = await app().inject({
+      method: 'POST',
+      url: '/object/notfound/authenticated/binary-casestudy1.png',
+      headers,
+      payload: fs.createReadStream(path),
+    })
+    expect(response.statusCode).toBe(400)
+    expect(mockUploadObject).not.toHaveBeenCalled()
+  })
+
+  test('return 400 when uploading to duplicate object', async () => {
+    const path = './src/test/assets/sadcat.jpg'
+    const { size } = fs.statSync(path)
+
+    const headers = {
+      authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`,
+      'Content-Length': size,
+      'Content-Type': 'image/jpeg',
+    }
+
+    const response = await app().inject({
+      method: 'POST',
+      url: '/object/bucket2/public/sadcat-upload23.png',
+      headers,
+      payload: fs.createReadStream(path),
+    })
+    expect(response.statusCode).toBe(400)
+    expect(mockUploadObject).not.toHaveBeenCalled()
+  })
+})
+
 /**
  * PUT /object/:id
  */