Skip to content

Commit

Permalink
feat: set up S3 connector to get presigned PUT URL (#439)
Browse files Browse the repository at this point in the history
* feat: set up S3 connector to get presigned PUT URL

* fix: add more env vars and remove unused schema

* fix: add asset router to app router

* fix: switch to use mutation and perform logging

* chore: add new env vars to turbo.json

* chore: add more groups to dependabot
  • Loading branch information
dcshzj authored Aug 7, 2024
1 parent f017263 commit 76f8f40
Show file tree
Hide file tree
Showing 14 changed files with 152 additions and 99 deletions.
18 changes: 6 additions & 12 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,12 @@ POSTMAN_API_KEY=
SENDGRID_API_KEY=
SENDGRID_FROM_ADDRESS=

# Image upload settings
# Assets upload settings
# =====================
# R2 API Keys for S3-compatible image upload
# If set to true, enables image upload
NEXT_PUBLIC_ENABLE_STORAGE=
R2_ACCOUNT_ID=
R2_BUCKET_NAME=
R2_AVATARS_DIRECTORY=avatars
R2_IMAGES_DIRECTORY=images
R2_ACCESS_KEY_ID=
R2_SECRET_ACCESS_KEY=
R2_PUBLIC_HOSTNAME=
S3_PUBLIC_ASSETS_DOMAIN_NAME=user-content.example.com
S3_UNSAFE_ASSETS_DOMAIN_NAME=user-unsafe-content.example.com
S3_PUBLIC_ASSETS_BUCKET_NAME=public-assets-bucket
S3_UNSAFE_ASSETS_BUCKET_NAME=unsafe-assets-bucket

# SGID settings
# =============
Expand All @@ -54,4 +48,4 @@ SGID_PRIVATE_KEY=
# Other settings
# ================
# NEXT_PUBLIC_APP_NAME= # Uncomment to set application name
# NEXT_PUBLIC_APP_URL= # Uncomment to set application URL
# NEXT_PUBLIC_APP_URL= # Uncomment to set application URL
4 changes: 4 additions & 0 deletions .env.test
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,7 @@
NODE_ENV=test
DATABASE_URL=postgres://root:root@localhost:5432/test
SESSION_SECRET=random_session_secret_that_is_at_least_32_characters
S3_PUBLIC_ASSETS_DOMAIN_NAME=user-content.example.com
S3_UNSAFE_ASSETS_DOMAIN_NAME=user-unsafe-content.example.com
S3_PUBLIC_ASSETS_BUCKET_NAME=public-assets-bucket
S3_UNSAFE_ASSETS_BUCKET_NAME=unsafe-assets-bucket
25 changes: 25 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,24 @@ updates:
schedule:
interval: daily
groups:
aws-sdk:
applies-to: version-updates
patterns:
- "@aws-sdk/*"
babel:
applies-to: version-updates
patterns:
- "@babel/*"
lodash:
applies-to: version-updates
patterns:
- "lodash"
- "@types/lodash"
pg:
applies-to: version-updates
patterns:
- "pg"
- "@types/pg"
prisma:
applies-to: version-updates
patterns:
Expand All @@ -19,6 +33,12 @@ updates:
patterns:
- "react"
- "react-dom"
- "@types/react"
react-query:
applies-to: version-updates
patterns:
- "@tanstack/react-query"
- "@tanstack/react-query-devtools"
storybook:
applies-to: version-updates
patterns:
Expand All @@ -32,3 +52,8 @@ updates:
applies-to: version-updates
patterns:
- "@trpc/*"
validator:
applies-to: version-updates
patterns:
- "validator"
- "@types/validator"
12 changes: 5 additions & 7 deletions apps/studio/next.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,7 @@ const ContentSecurityPolicy = `
object-src 'none';
script-src 'self' 'unsafe-eval';
style-src 'self' https: 'unsafe-inline';
connect-src 'self' https://schema.isomer.gov.sg https://browser-intake-datadoghq.com https://*.browser-intake-datadoghq.com https://vitals.vercel-insights.com/v1/vitals ${
// For POSTing presigned URLs to R2 storage.
env.R2_ACCOUNT_ID
? `https://*.${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`
: ""
};
connect-src 'self' https://schema.isomer.gov.sg https://browser-intake-datadoghq.com https://*.browser-intake-datadoghq.com https://vitals.vercel-insights.com/v1/vitals;
worker-src 'self' blob:;
${env.NODE_ENV === "production" ? "upgrade-insecure-requests" : ""}
`
Expand All @@ -52,7 +47,10 @@ const config = {
/** We run eslint as a separate task in CI */
eslint: { ignoreDuringBuilds: true },
images: {
domains: [env.R2_PUBLIC_HOSTNAME ?? ""].filter((d) => d),
domains: [
env.S3_UNSAFE_ASSETS_DOMAIN_NAME ?? "",
env.S3_PUBLIC_ASSETS_DOMAIN_NAME ?? "",
].filter((d) => d),
},
async headers() {
return [
Expand Down
55 changes: 12 additions & 43 deletions apps/studio/src/env.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,30 +23,14 @@ const client = z.object({
})

/** Feature flags */
const baseR2Schema = z.object({
R2_ACCESS_KEY_ID: z.string().optional(),
R2_ACCOUNT_ID: z.string().optional(),
R2_SECRET_ACCESS_KEY: z.string().optional(),
R2_PUBLIC_HOSTNAME: z.string().optional(),
R2_BUCKET_NAME: z.string().optional(),
R2_AVATARS_DIRECTORY: z.string().optional(),
R2_IMAGES_DIRECTORY: z.string().optional(),
const s3Schema = z.object({
S3_REGION: z.string().default("us-east-1"),
S3_PUBLIC_ASSETS_DOMAIN_NAME: z.string(),
S3_UNSAFE_ASSETS_DOMAIN_NAME: z.string(),
S3_PUBLIC_ASSETS_BUCKET_NAME: z.string(),
S3_UNSAFE_ASSETS_BUCKET_NAME: z.string(),
})

const r2ServerSchema = z.discriminatedUnion("NEXT_PUBLIC_ENABLE_STORAGE", [
baseR2Schema.extend({
NEXT_PUBLIC_ENABLE_STORAGE: z.literal(true),
// Add required keys if flag is enabled.
R2_ACCESS_KEY_ID: z.string().min(1),
R2_ACCOUNT_ID: z.string().min(1),
R2_SECRET_ACCESS_KEY: z.string().min(1),
R2_PUBLIC_HOSTNAME: z.string().min(1),
}),
baseR2Schema.extend({
NEXT_PUBLIC_ENABLE_STORAGE: z.literal(false),
}),
])

const baseSgidSchema = z.object({
SGID_CLIENT_ID: z.string().optional(),
SGID_CLIENT_SECRET: z.string().optional(),
Expand Down Expand Up @@ -86,25 +70,12 @@ const server = z
]),
SESSION_SECRET: z.string().min(32),
})
.merge(s3Schema)
// Add on schemas as needed that requires conditional validation.
.merge(baseR2Schema)
.merge(baseSgidSchema)
.merge(client)
// Add on refinements as needed for conditional environment variables
// .superRefine((val, ctx) => ...)
.superRefine((val, ctx) => {
const parse = r2ServerSchema.safeParse(val)
if (!parse.success) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ["NEXT_PUBLIC_ENABLE_STORAGE"],
message: "R2 environment variables are missing",
})
parse.error.issues.forEach((issue) => {
ctx.addIssue(issue)
})
}
})
.superRefine((val, ctx) => {
const parse = sgidServerSchema.safeParse(val)
if (!parse.success) {
Expand Down Expand Up @@ -139,13 +110,11 @@ const processEnv = {
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
SENDGRID_FROM_ADDRESS: process.env.SENDGRID_FROM_ADDRESS,
SESSION_SECRET: process.env.SESSION_SECRET,
R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID,
R2_ACCOUNT_ID: process.env.R2_ACCOUNT_ID,
R2_BUCKET_NAME: process.env.R2_BUCKET_NAME,
R2_AVATARS_DIRECTORY: process.env.R2_AVATARS_DIRECTORY,
R2_IMAGES_DIRECTORY: process.env.R2_IMAGES_DIRECTORY,
R2_PUBLIC_HOSTNAME: process.env.R2_PUBLIC_HOSTNAME,
R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY,
S3_REGION: process.env.S3_REGION,
S3_PUBLIC_ASSETS_DOMAIN_NAME: process.env.S3_PUBLIC_ASSETS_DOMAIN_NAME,
S3_UNSAFE_ASSETS_DOMAIN_NAME: process.env.S3_UNSAFE_ASSETS_DOMAIN_NAME,
S3_PUBLIC_ASSETS_BUCKET_NAME: process.env.S3_PUBLIC_ASSETS_BUCKET_NAME,
S3_UNSAFE_ASSETS_BUCKET_NAME: process.env.S3_UNSAFE_ASSETS_BUCKET_NAME,
SGID_CLIENT_ID: process.env.SGID_CLIENT_ID,
SGID_CLIENT_SECRET: process.env.SGID_CLIENT_SECRET,
SGID_PRIVATE_KEY: process.env.SGID_PRIVATE_KEY,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export function JsonFormsImageControl({
<FormControl isRequired={required} isInvalid={!pendingFile || !!errors}>
<FormLabel description={description}>{label}</FormLabel>
<Attachment
isRequired
isRequired={required}
name="image-upload"
imagePreview="large"
multiple={false}
Expand Down
22 changes: 0 additions & 22 deletions apps/studio/src/lib/r2.ts

This file was deleted.

26 changes: 26 additions & 0 deletions apps/studio/src/lib/s3.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import type { PutObjectCommandInput } from "@aws-sdk/client-s3"
import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
import { getSignedUrl } from "@aws-sdk/s3-request-presigner"

import { env } from "~/env.mjs"

const { S3_REGION, S3_UNSAFE_ASSETS_BUCKET_NAME } = env

export const storage = new S3Client({
region: S3_REGION,
})

export const generateSignedPutUrl = async ({
Key,
}: Pick<PutObjectCommandInput, "Key">): Promise<string> => {
return getSignedUrl(
storage,
new PutObjectCommand({
Bucket: S3_UNSAFE_ASSETS_BUCKET_NAME,
Key,
}),
{
expiresIn: 60 * 5, // 5 minutes
},
)
}
8 changes: 8 additions & 0 deletions apps/studio/src/schemas/asset.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { z } from "zod"

export const getPresignedPutUrlSchema = z.object({
siteId: z.number().min(1),
fileName: z.string({
required_error: "Missing file name",
}),
})
13 changes: 0 additions & 13 deletions apps/studio/src/schemas/presign.ts

This file was deleted.

2 changes: 2 additions & 0 deletions apps/studio/src/server/modules/_app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* This file contains the root router of your tRPC-backend
*/
import { publicProcedure, router } from "../trpc"
import { assetRouter } from "./asset/asset.router"
import { authRouter } from "./auth/auth.router"
import { folderRouter } from "./folder/folder.router"
import { meRouter } from "./me/me.router"
Expand All @@ -12,6 +13,7 @@ export const appRouter = router({
healthcheck: publicProcedure.query(() => "yay!"),
me: meRouter,
auth: authRouter,
asset: assetRouter,
page: pageRouter,
folder: folderRouter,
site: siteRouter,
Expand Down
33 changes: 33 additions & 0 deletions apps/studio/src/server/modules/asset/asset.router.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { getPresignedPutUrlSchema } from "~/schemas/asset"
import { protectedProcedure, router } from "~/server/trpc"
import { getFileKey, getPresignedPutUrl } from "./asset.service"

export const assetRouter = router({
getPresignedPutUrl: protectedProcedure
.input(getPresignedPutUrlSchema)
.mutation(async ({ ctx, input: { siteId, fileName } }) => {
const fileKey = getFileKey({
siteId,
fileName,
})

const presignedPutUrl = await getPresignedPutUrl({
key: fileKey,
})

ctx.logger.info(
`Generated presigned PUT URL for ${fileKey} for site ${siteId}`,
{
userId: ctx.session?.userId,
siteId,
fileName,
fileKey,
},
)

return {
fileKey,
presignedPutUrl,
}
}),
})
21 changes: 21 additions & 0 deletions apps/studio/src/server/modules/asset/asset.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { randomUUID } from "crypto"
import type { z } from "zod"

import type { getPresignedPutUrlSchema } from "~/schemas/asset"
import { generateSignedPutUrl } from "~/lib/s3"

export const getFileKey = ({
siteId,
fileName,
}: z.infer<typeof getPresignedPutUrlSchema>) => {
// NOTE: We're using a random folder name to prevent collisions
const folderName = randomUUID()

return `${siteId}/${folderName}/${fileName}`
}

export const getPresignedPutUrl = async ({ key }: { key: string }) => {
return generateSignedPutUrl({
Key: key,
})
}
10 changes: 9 additions & 1 deletion turbo.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,15 @@
"$schema": "https://turbo.build/schema.json",
"ui": "tui",
"globalDependencies": ["**/.env", "tsconfig.json"],
"globalEnv": ["NODE_ENV", "DATABASE_URL", "SESSION_SECRET"],
"globalEnv": [
"NODE_ENV",
"DATABASE_URL",
"SESSION_SECRET",
"S3_PUBLIC_ASSETS_DOMAIN_NAME",
"S3_UNSAFE_ASSETS_DOMAIN_NAME",
"S3_PUBLIC_ASSETS_BUCKET_NAME",
"S3_UNSAFE_ASSETS_BUCKET_NAME"
],
"tasks": {
"build": {
"dependsOn": ["^build"],
Expand Down

0 comments on commit 76f8f40

Please sign in to comment.