feat: set up S3 connector to get presigned PUT URL

.env.example

@@ -29,18 +29,12 @@ POSTMAN_API_KEY=
 SENDGRID_API_KEY=
 SENDGRID_FROM_ADDRESS=
 
-# Image upload settings
+# Assets upload settings
 # =====================
-# R2 API Keys for S3-compatible image upload
-# If set to true, enables image upload
-NEXT_PUBLIC_ENABLE_STORAGE=
-R2_ACCOUNT_ID=
-R2_BUCKET_NAME=
-R2_AVATARS_DIRECTORY=avatars
-R2_IMAGES_DIRECTORY=images
-R2_ACCESS_KEY_ID=
-R2_SECRET_ACCESS_KEY=
-R2_PUBLIC_HOSTNAME=
+S3_PUBLIC_ASSETS_DOMAIN_NAME=user-content.example.com
+S3_UNSAFE_ASSETS_DOMAIN_NAME=user-unsafe-content.example.com
+S3_PUBLIC_ASSETS_BUCKET_NAME=public-assets-bucket
+S3_UNSAFE_ASSETS_BUCKET_NAME=unsafe-assets-bucket
 
 # SGID settings
 # =============
@@ -54,4 +48,4 @@ SGID_PRIVATE_KEY=
 # Other settings
 # ================
 # NEXT_PUBLIC_APP_NAME= # Uncomment to set application name
-# NEXT_PUBLIC_APP_URL= # Uncomment to set application URL
+# NEXT_PUBLIC_APP_URL= # Uncomment to set application URL

.env.test

@@ -3,3 +3,7 @@
 NODE_ENV=test
 DATABASE_URL=postgres://root:root@localhost:5432/test
 SESSION_SECRET=random_session_secret_that_is_at_least_32_characters
+S3_PUBLIC_ASSETS_DOMAIN_NAME=user-content.example.com
+S3_UNSAFE_ASSETS_DOMAIN_NAME=user-unsafe-content.example.com
+S3_PUBLIC_ASSETS_BUCKET_NAME=public-assets-bucket
+S3_UNSAFE_ASSETS_BUCKET_NAME=unsafe-assets-bucket

.github/dependabot.yml

@@ -5,10 +5,24 @@ updates:
     schedule:
       interval: daily
     groups:
+      aws-sdk:
+        applies-to: version-updates
+        patterns:
+          - "@aws-sdk/*"
       babel:
         applies-to: version-updates
         patterns:
           - "@babel/*"
+      lodash:
+        applies-to: version-updates
+        patterns:
+          - "lodash"
+          - "@types/lodash"
+      pg:
+        applies-to: version-updates
+        patterns:
+          - "pg"
+          - "@types/pg"
       prisma:
         applies-to: version-updates
         patterns:
@@ -19,6 +33,12 @@ updates:
         patterns:
           - "react"
           - "react-dom"
+          - "@types/react"
+      react-query:
+        applies-to: version-updates
+        patterns:
+          - "@tanstack/react-query"
+          - "@tanstack/react-query-devtools"
       storybook:
         applies-to: version-updates
         patterns:
@@ -32,3 +52,8 @@ updates:
         applies-to: version-updates
         patterns:
           - "@trpc/*"
+      validator:
+        applies-to: version-updates
+        patterns:
+          - "validator"
+          - "@types/validator"

apps/studio/next.config.mjs

@@ -25,12 +25,7 @@ const ContentSecurityPolicy = `
   object-src 'none';
   script-src 'self' 'unsafe-eval';
   style-src 'self' https: 'unsafe-inline';
-  connect-src 'self' https://schema.isomer.gov.sg https://browser-intake-datadoghq.com https://*.browser-intake-datadoghq.com https://vitals.vercel-insights.com/v1/vitals ${
-    // For POSTing presigned URLs to R2 storage.
-    env.R2_ACCOUNT_ID
-      ? `https://*.${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`
-      : ""
-  };
+  connect-src 'self' https://schema.isomer.gov.sg https://browser-intake-datadoghq.com https://*.browser-intake-datadoghq.com https://vitals.vercel-insights.com/v1/vitals;
   worker-src 'self' blob:;
   ${env.NODE_ENV === "production" ? "upgrade-insecure-requests" : ""}
 `
@@ -52,7 +47,10 @@ const config = {
   /** We run eslint as a separate task in CI */
   eslint: { ignoreDuringBuilds: true },
   images: {
-    domains: [env.R2_PUBLIC_HOSTNAME ?? ""].filter((d) => d),
+    domains: [
+      env.S3_UNSAFE_ASSETS_DOMAIN_NAME ?? "",
+      env.S3_PUBLIC_ASSETS_DOMAIN_NAME ?? "",
+    ].filter((d) => d),
   },
   async headers() {
     return [

apps/studio/src/env.mjs

@@ -23,30 +23,14 @@ const client = z.object({
 })
 
 /** Feature flags */
-const baseR2Schema = z.object({
-  R2_ACCESS_KEY_ID: z.string().optional(),
-  R2_ACCOUNT_ID: z.string().optional(),
-  R2_SECRET_ACCESS_KEY: z.string().optional(),
-  R2_PUBLIC_HOSTNAME: z.string().optional(),
-  R2_BUCKET_NAME: z.string().optional(),
-  R2_AVATARS_DIRECTORY: z.string().optional(),
-  R2_IMAGES_DIRECTORY: z.string().optional(),
+const s3Schema = z.object({
+  S3_REGION: z.string().default("us-east-1"),
+  S3_PUBLIC_ASSETS_DOMAIN_NAME: z.string(),
+  S3_UNSAFE_ASSETS_DOMAIN_NAME: z.string(),
+  S3_PUBLIC_ASSETS_BUCKET_NAME: z.string(),
+  S3_UNSAFE_ASSETS_BUCKET_NAME: z.string(),
 })
 
-const r2ServerSchema = z.discriminatedUnion("NEXT_PUBLIC_ENABLE_STORAGE", [
-  baseR2Schema.extend({
-    NEXT_PUBLIC_ENABLE_STORAGE: z.literal(true),
-    // Add required keys if flag is enabled.
-    R2_ACCESS_KEY_ID: z.string().min(1),
-    R2_ACCOUNT_ID: z.string().min(1),
-    R2_SECRET_ACCESS_KEY: z.string().min(1),
-    R2_PUBLIC_HOSTNAME: z.string().min(1),
-  }),
-  baseR2Schema.extend({
-    NEXT_PUBLIC_ENABLE_STORAGE: z.literal(false),
-  }),
-])
-
 const baseSgidSchema = z.object({
   SGID_CLIENT_ID: z.string().optional(),
   SGID_CLIENT_SECRET: z.string().optional(),
@@ -86,25 +70,12 @@ const server = z
     ]),
     SESSION_SECRET: z.string().min(32),
   })
+  .merge(s3Schema)
   // Add on schemas as needed that requires conditional validation.
-  .merge(baseR2Schema)
   .merge(baseSgidSchema)
   .merge(client)
   // Add on refinements as needed for conditional environment variables
   // .superRefine((val, ctx) => ...)
-  .superRefine((val, ctx) => {
-    const parse = r2ServerSchema.safeParse(val)
-    if (!parse.success) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        path: ["NEXT_PUBLIC_ENABLE_STORAGE"],
-        message: "R2 environment variables are missing",
-      })
-      parse.error.issues.forEach((issue) => {
-        ctx.addIssue(issue)
-      })
-    }
-  })
   .superRefine((val, ctx) => {
     const parse = sgidServerSchema.safeParse(val)
     if (!parse.success) {
@@ -139,13 +110,11 @@ const processEnv = {
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   SENDGRID_FROM_ADDRESS: process.env.SENDGRID_FROM_ADDRESS,
   SESSION_SECRET: process.env.SESSION_SECRET,
-  R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID,
-  R2_ACCOUNT_ID: process.env.R2_ACCOUNT_ID,
-  R2_BUCKET_NAME: process.env.R2_BUCKET_NAME,
-  R2_AVATARS_DIRECTORY: process.env.R2_AVATARS_DIRECTORY,
-  R2_IMAGES_DIRECTORY: process.env.R2_IMAGES_DIRECTORY,
-  R2_PUBLIC_HOSTNAME: process.env.R2_PUBLIC_HOSTNAME,
-  R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY,
+  S3_REGION: process.env.S3_REGION,
+  S3_PUBLIC_ASSETS_DOMAIN_NAME: process.env.S3_PUBLIC_ASSETS_DOMAIN_NAME,
+  S3_UNSAFE_ASSETS_DOMAIN_NAME: process.env.S3_UNSAFE_ASSETS_DOMAIN_NAME,
+  S3_PUBLIC_ASSETS_BUCKET_NAME: process.env.S3_PUBLIC_ASSETS_BUCKET_NAME,
+  S3_UNSAFE_ASSETS_BUCKET_NAME: process.env.S3_UNSAFE_ASSETS_BUCKET_NAME,
   SGID_CLIENT_ID: process.env.SGID_CLIENT_ID,
   SGID_CLIENT_SECRET: process.env.SGID_CLIENT_SECRET,
   SGID_PRIVATE_KEY: process.env.SGID_PRIVATE_KEY,

apps/studio/src/features/editing-experience/components/form-builder/renderers/controls/JsonFormsImageControl.tsx

@@ -40,7 +40,7 @@ export function JsonFormsImageControl({
       <FormControl isRequired={required} isInvalid={!pendingFile || !!errors}>
         <FormLabel description={description}>{label}</FormLabel>
         <Attachment
-          isRequired
+          isRequired={required}
           name="image-upload"
           imagePreview="large"
           multiple={false}

apps/studio/src/lib/s3.ts

@@ -0,0 +1,26 @@
+import type { PutObjectCommandInput } from "@aws-sdk/client-s3"
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
+
+import { env } from "~/env.mjs"
+
+const { S3_REGION, S3_UNSAFE_ASSETS_BUCKET_NAME } = env
+
+export const storage = new S3Client({
+  region: S3_REGION,
+})
+
+export const generateSignedPutUrl = async ({
+  Key,
+}: Pick<PutObjectCommandInput, "Key">): Promise<string> => {
+  return getSignedUrl(
+    storage,
+    new PutObjectCommand({
+      Bucket: S3_UNSAFE_ASSETS_BUCKET_NAME,
+      Key,
+    }),
+    {
+      expiresIn: 60 * 5, // 5 minutes
+    },
+  )
+}

apps/studio/src/schemas/asset.ts

@@ -0,0 +1,8 @@
+import { z } from "zod"
+
+export const getPresignedPutUrlSchema = z.object({
+  siteId: z.number().min(1),
+  fileName: z.string({
+    required_error: "Missing file name",
+  }),
+})

apps/studio/src/server/modules/_app.ts

@@ -2,6 +2,7 @@
  * This file contains the root router of your tRPC-backend
  */
 import { publicProcedure, router } from "../trpc"
+import { assetRouter } from "./asset/asset.router"
 import { authRouter } from "./auth/auth.router"
 import { folderRouter } from "./folder/folder.router"
 import { meRouter } from "./me/me.router"
@@ -12,6 +13,7 @@ export const appRouter = router({
   healthcheck: publicProcedure.query(() => "yay!"),
   me: meRouter,
   auth: authRouter,
+  asset: assetRouter,
   page: pageRouter,
   folder: folderRouter,
   site: siteRouter,

apps/studio/src/server/modules/asset/asset.router.ts

@@ -0,0 +1,33 @@
+import { getPresignedPutUrlSchema } from "~/schemas/asset"
+import { protectedProcedure, router } from "~/server/trpc"
+import { getFileKey, getPresignedPutUrl } from "./asset.service"
+
+export const assetRouter = router({
+  getPresignedPutUrl: protectedProcedure
+    .input(getPresignedPutUrlSchema)
+    .mutation(async ({ ctx, input: { siteId, fileName } }) => {
+      const fileKey = getFileKey({
+        siteId,
+        fileName,
+      })
+
+      const presignedPutUrl = await getPresignedPutUrl({
+        key: fileKey,
+      })
+
+      ctx.logger.info(
+        `Generated presigned PUT URL for ${fileKey} for site ${siteId}`,
+        {
+          userId: ctx.session?.userId,
+          siteId,
+          fileName,
+          fileKey,
+        },
+      )
+
+      return {
+        fileKey,
+        presignedPutUrl,
+      }
+    }),
+})

apps/studio/src/server/modules/asset/asset.service.ts

@@ -0,0 +1,21 @@
+import { randomUUID } from "crypto"
+import type { z } from "zod"
+
+import type { getPresignedPutUrlSchema } from "~/schemas/asset"
+import { generateSignedPutUrl } from "~/lib/s3"
+
+export const getFileKey = ({
+  siteId,
+  fileName,
+}: z.infer<typeof getPresignedPutUrlSchema>) => {
+  // NOTE: We're using a random folder name to prevent collisions
+  const folderName = randomUUID()
+
+  return `${siteId}/${folderName}/${fileName}`
+}
+
+export const getPresignedPutUrl = async ({ key }: { key: string }) => {
+  return generateSignedPutUrl({
+    Key: key,
+  })
+}

turbo.json

@@ -2,7 +2,15 @@
   "$schema": "https://turbo.build/schema.json",
   "ui": "tui",
   "globalDependencies": ["**/.env", "tsconfig.json"],
-  "globalEnv": ["NODE_ENV", "DATABASE_URL", "SESSION_SECRET"],
+  "globalEnv": [
+    "NODE_ENV",
+    "DATABASE_URL",
+    "SESSION_SECRET",
+    "S3_PUBLIC_ASSETS_DOMAIN_NAME",
+    "S3_UNSAFE_ASSETS_DOMAIN_NAME",
+    "S3_PUBLIC_ASSETS_BUCKET_NAME",
+    "S3_UNSAFE_ASSETS_BUCKET_NAME"
+  ],
   "tasks": {
     "build": {
       "dependsOn": ["^build"],