feat: stable S3 Protocol

.github/workflows/ci.yml

@@ -56,16 +56,13 @@ jobs:
           SERVICE_KEY: ${{ secrets.SERVICE_KEY }}
           TENANT_ID: ${{ secrets.TENANT_ID }}
           REGION: ${{ secrets.REGION }}
-          POSTGREST_URL: ${{ secrets.POSTGREST_URL }}
           GLOBAL_S3_BUCKET: ${{ secrets.GLOBAL_S3_BUCKET }}
           PGRST_JWT_SECRET: ${{ secrets.PGRST_JWT_SECRET }}
           AUTHENTICATED_KEY: ${{ secrets.AUTHENTICATED_KEY }}
           DATABASE_URL: postgresql://postgres:[email protected]/postgres
-          PGOPTIONS: -c search_path=storage,public
           FILE_SIZE_LIMIT: '52428800'
           STORAGE_BACKEND: s3
           MULTITENANT_DATABASE_URL: postgresql://postgres:[email protected]:5433/postgres
-          POSTGREST_URL_SUFFIX: /rest/v1
           ADMIN_API_KEYS: apikey
           ENABLE_IMAGE_TRANSFORMATION: true
           IMGPROXY_URL: http://127.0.0.1:50020

README.md

@@ -6,11 +6,19 @@ A scalable, light-weight object storage service.
 
 > Read [this post](https://supabase.io/blog/2021/03/30/supabase-storage) on why we decided to build a new object storage service.
 
+- Multi-protocol support (HTTP, TUS, S3)
 - Uses Postgres as its datastore for storing metadata
 - Authorization rules are written as Postgres Row Level Security policies
-- Integrates with S3 as the storage backend (with more in the pipeline!)
+- Integrates with S3 Compatible Storages
 - Extremely lightweight and performant
 
+
+**Supported Protocols**
+
+- [x] HTTP/REST
+- [x] TUS Resumable Upload
+- [x] S3 Compatible API
+
 ![Architecture](./static/architecture.png?raw=true 'Architecture')
 
 ## Documentation

docker-compose-multi-tenant.yml

@@ -2,56 +2,56 @@
 
 version: '3'
 services:
-#  storage:
-#    image: supabase/storage-api:latest
-#    ports:
-#      - '5000:5000'
-#      - '5001:5001'
-#    depends_on:
-#      tenant_db:
-#        condition: service_healthy
-#      multitenant_db:
-#        condition: service_healthy
-#      supavisor:
-#          condition: service_started
-#      minio_setup:
-#          condition: service_completed_successfully
-#    environment:
-#      # Server
-#      SERVER_PORT: 5000
-#      SERVER_REGION: local
-#      # Auth
-#      AUTH_JWT_SECRET: f023d3db-39dc-4ac9-87b2-b2be72e9162b
-#      AUTH_JWT_ALGORITHM: HS256
-#      AUTH_ENCRYPTION_KEY: encryptionkey
-#      # Multi tenant Mode
-#      MULTI_TENANT: true
-#      DATABASE_MULTITENANT_URL: postgresql://postgres:postgres@multitenant_db:5432/postgres
-#      SERVER_ADMIN_API_KEYS: apikey
-#      SERVER_ADMIN_PORT: 5001
-#      REQUEST_X_FORWARDED_HOST_REGEXP: "^([a-z]{20}).local.(?:com|dev)$"
-#      # Migrations
-#      DB_INSTALL_ROLES: true # set to false if you want to manage roles yourself
-#      # Storage
-#      STORAGE_BACKEND: s3
-#      STORAGE_S3_BUCKET: supa-storage-bucket  # name of s3 bucket where you want to store objects
-#      STORAGE_S3_ENDPOINT: http://minio:9000
-#      STORAGE_S3_FORCE_PATH_STYLE: "true"
-#      STORAGE_S3_REGION: us-east-1
-#      AWS_ACCESS_KEY_ID: supa-storage
-#      AWS_SECRET_ACCESS_KEY: secret1234
-#      # Upload
-#      UPLOAD_FILE_SIZE_LIMIT: 524288000
-#      UPLOAD_FILE_SIZE_LIMIT_STANDARD: 52428800
-#      UPLOAD_SIGNED_URL_EXPIRATION_TIME: 120
-#      TUS_URL_PATH: /upload/resumable
-#      TUS_URL_EXPIRY_MS: 3600000
-#      # Image Tranformation
-#      IMAGE_TRANSFORMATION_ENABLED: "true"
-#      IMGPROXY_URL: http://imgproxy:8080
-#      IMGPROXY_REQUEST_TIMEOUT: 15
-#
-#      PG_QUEUE_ENABLE: "true"
+  storage:
+    image: supabase/storage-api:latest
+    ports:
+      - '5000:5000'
+      - '5001:5001'
+    depends_on:
+      tenant_db:
+        condition: service_healthy
+      multitenant_db:
+        condition: service_healthy
+      supavisor:
+          condition: service_started
+      minio_setup:
+          condition: service_completed_successfully
+    environment:
+      # Server
+      SERVER_PORT: 5000
+      SERVER_REGION: local
+      # Auth
+      AUTH_JWT_SECRET: f023d3db-39dc-4ac9-87b2-b2be72e9162b
+      AUTH_JWT_ALGORITHM: HS256
+      AUTH_ENCRYPTION_KEY: encryptionkey
+      # Multi tenant Mode
+      MULTI_TENANT: true
+      DATABASE_MULTITENANT_URL: postgresql://postgres:postgres@multitenant_db:5432/postgres
+      SERVER_ADMIN_API_KEYS: apikey
+      SERVER_ADMIN_PORT: 5001
+      REQUEST_X_FORWARDED_HOST_REGEXP: "^([a-z]{20}).local.(?:com|dev)$"
+      # Migrations
+      DB_INSTALL_ROLES: true # set to false if you want to manage roles yourself
+      # Storage
+      STORAGE_BACKEND: s3
+      STORAGE_S3_BUCKET: supa-storage-bucket  # name of s3 bucket where you want to store objects
+      STORAGE_S3_ENDPOINT: http://minio:9000
+      STORAGE_S3_FORCE_PATH_STYLE: "true"
+      STORAGE_S3_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: supa-storage
+      AWS_SECRET_ACCESS_KEY: secret1234
+      # Upload
+      UPLOAD_FILE_SIZE_LIMIT: 524288000
+      UPLOAD_FILE_SIZE_LIMIT_STANDARD: 52428800
+      UPLOAD_SIGNED_URL_EXPIRATION_TIME: 120
+      TUS_URL_PATH: /upload/resumable
+      TUS_URL_EXPIRY_MS: 3600000
+      # Image Tranformation
+      IMAGE_TRANSFORMATION_ENABLED: "true"
+      IMGPROXY_URL: http://imgproxy:8080
+      IMGPROXY_REQUEST_TIMEOUT: 15
+
+      PG_QUEUE_ENABLE: "true"
 
   tenant_db:
     extends:

jest-setup.ts

@@ -1,3 +1,7 @@
 import { getConfig, setEnvPaths } from './src/config'
 
 setEnvPaths(['.env.test', '.env'])
+
+beforeEach(() => {
+  getConfig({ reload: true })
+})

jest.config.js

@@ -4,7 +4,7 @@ module.exports = {
   transform: {
     '^.+\\.(t|j)sx?$': 'ts-jest',
   },
-  setupFiles: ['<rootDir>/jest-setup.ts'],
+  setupFilesAfterEnv: ['<rootDir>/jest-setup.ts'],
   testEnvironment: 'node',
   testPathIgnorePatterns: ['node_modules', 'dist'],
   coverageProvider: 'v8',

jest.sequencer.js

@@ -11,15 +11,21 @@ const isTusTest = (test) => {
   return test.path.includes('tus')
 }
 
+const isS3Test = (test) => {
+  return test.path.includes('s3')
+}
+
 class CustomSequencer extends Sequencer {
   sort(tests) {
     const copyTests = Array.from(tests)
     const normalTests = copyTests.filter((t) => !isRLSTest(t) && !isTusTest(t))
     const tusTests = copyTests.filter((t) => isTusTest(t))
+    const s3Tests = copyTests.filter((t) => isS3Test(t))
     const rlsTests = copyTests.filter((t) => isRLSTest(t))
     return super
       .sort(normalTests)
       .concat(tusTests)
+      .concat(s3Tests)
       .concat(rlsTests.sort((a, b) => (a.path > b.path ? 1 : -1)))
   }
 }

migrations/tenant/0020-list-objects-with-delimiter.sql

@@ -0,0 +1,39 @@
+
+
+CREATE OR REPLACE FUNCTION storage.list_objects_with_delimiter(bucket_id text, prefix_param text, delimiter_param text, max_keys integer default 100, next_token text DEFAULT '')
+    RETURNS TABLE (name text, id uuid, metadata jsonb, updated_at timestamptz) AS
+$$
+BEGIN
+    RETURN QUERY EXECUTE
+        'SELECT DISTINCT ON(name COLLATE "C") * from (
+            SELECT
+                CASE
+                    WHEN position($2 IN substring(name from length($1) + 1)) > 0 THEN
+                        substring(name from 1 for length($1) + position($2 IN substring(name from length($1) + 1)))
+                    ELSE
+                        name
+                END AS name, id, metadata, updated_at
+            FROM
+                storage.objects
+            WHERE
+                bucket_id = $5 AND
+                name ILIKE $1 || ''%'' AND
+                CASE
+                    WHEN $4 != '''' THEN
+                        CASE
+                            WHEN position($2 IN substring(name from length($1) + 1)) > 0 THEN
+                                substring(name from 1 for length($1) + position($2 IN substring(name from length($1) + 1))) COLLATE "C" > $4
+                            ELSE
+                                name COLLATE "C" > $4
+                            END
+                    ELSE
+                        true
+                END
+            ORDER BY
+                name COLLATE "C" ASC) as e order by name COLLATE "C" LIMIT $3'
+        USING prefix_param, delimiter_param, max_keys, next_token, bucket_id;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE INDEX idx_objects_bucket_id_name
+    ON storage.objects (bucket_id, (name COLLATE "C"));

migrations/tenant/0021-s3-multipart-uploads.sql

@@ -0,0 +1,66 @@
+
+CREATE TABLE IF NOT EXISTS storage._s3_multipart_uploads (
+    id text PRIMARY KEY,
+    in_progress_size int NOT NULL default 0,
+    upload_signature text NOT NULL,
+    bucket_id text NOT NULL references storage.buckets(id),
+    key text COLLATE "C" NOT NULL ,
+    version text NOT NULL,
+    created_at timestamptz NOT NULL default now()
+);
+
+CREATE TABLE IF NOT EXISTS storage._s3_multipart_uploads_parts (
+     id uuid PRIMARY KEY default gen_random_uuid(),
+     upload_id text NOT NULL references storage._s3_multipart_uploads(id) ON DELETE CASCADE,
+     size int NOT NULL default 0,
+     part_number int NOT NULL,
+     bucket_id text NOT NULL references storage.buckets(id),
+     key text COLLATE "C" NOT NULL,
+     etag text NOT NULL,
+     version text NOT NULL,
+     created_at timestamptz NOT NULL default now()
+);
+
+CREATE INDEX idx_multipart_uploads_list
+    ON storage._s3_multipart_uploads (bucket_id, (key COLLATE "C"), created_at ASC);
+
+CREATE OR REPLACE FUNCTION storage.list_multipart_uploads_with_delimiter(bucket_id text, prefix_param text, delimiter_param text, max_keys integer default 100, next_key_token text DEFAULT '', next_upload_token text default '')
+    RETURNS TABLE (key text, id text, created_at timestamptz) AS
+$$
+BEGIN
+    RETURN QUERY EXECUTE
+        'SELECT DISTINCT ON(key COLLATE "C") * from (
+            SELECT
+                CASE
+                    WHEN position($2 IN substring(key from length($1) + 1)) > 0 THEN
+                        substring(key from 1 for length($1) + position($2 IN substring(key from length($1) + 1)))
+                    ELSE
+                        key
+                END AS key, id, created_at
+            FROM
+                storage._s3_multipart_uploads
+            WHERE
+                bucket_id = $5 AND
+                key ILIKE $1 || ''%'' AND
+                CASE
+                    WHEN $4 != '''' AND $6 = '''' THEN
+                        CASE
+                            WHEN position($2 IN substring(key from length($1) + 1)) > 0 THEN
+                                substring(key from 1 for length($1) + position($2 IN substring(key from length($1) + 1))) COLLATE "C" > $4
+                            ELSE
+                                key COLLATE "C" > $4
+                            END
+                    ELSE
+                        true
+                END AND
+                CASE
+                    WHEN $6 != '''' THEN
+                        id COLLATE "C" > $6
+                    ELSE
+                        true
+                    END
+            ORDER BY
+                key COLLATE "C" ASC, created_at ASC) as e order by key COLLATE "C" LIMIT $3'
+        USING prefix_param, delimiter_param, max_keys, next_key_token, bucket_id, next_upload_token;
+END;
+$$ LANGUAGE plpgsql;