diff --git a/.env.example b/.env.example index 11427de059..8c578c43ef 100644 --- a/.env.example +++ b/.env.example @@ -29,18 +29,12 @@ POSTMAN_API_KEY= SENDGRID_API_KEY= SENDGRID_FROM_ADDRESS= -# Image upload settings +# Assets upload settings # ===================== -# R2 API Keys for S3-compatible image upload -# If set to true, enables image upload -NEXT_PUBLIC_ENABLE_STORAGE= -R2_ACCOUNT_ID= -R2_BUCKET_NAME= -R2_AVATARS_DIRECTORY=avatars -R2_IMAGES_DIRECTORY=images -R2_ACCESS_KEY_ID= -R2_SECRET_ACCESS_KEY= -R2_PUBLIC_HOSTNAME= +S3_PUBLIC_ASSETS_DOMAIN_NAME=user-content.example.com +S3_UNSAFE_ASSETS_DOMAIN_NAME=user-unsafe-content.example.com +S3_PUBLIC_ASSETS_BUCKET_NAME=public-assets-bucket +S3_UNSAFE_ASSETS_BUCKET_NAME=unsafe-assets-bucket # SGID settings # ============= @@ -54,4 +48,4 @@ SGID_PRIVATE_KEY= # Other settings # ================ # NEXT_PUBLIC_APP_NAME= # Uncomment to set application name -# NEXT_PUBLIC_APP_URL= # Uncomment to set application URL \ No newline at end of file +# NEXT_PUBLIC_APP_URL= # Uncomment to set application URL diff --git a/.env.test b/.env.test index 121c4097d3..d23ac89f87 100644 --- a/.env.test +++ b/.env.test @@ -3,3 +3,7 @@ NODE_ENV=test DATABASE_URL=postgres://root:root@localhost:5432/test SESSION_SECRET=random_session_secret_that_is_at_least_32_characters +S3_PUBLIC_ASSETS_DOMAIN_NAME=user-content.example.com +S3_UNSAFE_ASSETS_DOMAIN_NAME=user-unsafe-content.example.com +S3_PUBLIC_ASSETS_BUCKET_NAME=public-assets-bucket +S3_UNSAFE_ASSETS_BUCKET_NAME=unsafe-assets-bucket diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 621fc6d55d..a146422ac2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,10 +5,24 @@ updates: schedule: interval: daily groups: + aws-sdk: + applies-to: version-updates + patterns: + - "@aws-sdk/*" babel: applies-to: version-updates patterns: - "@babel/*" + lodash: + applies-to: version-updates + patterns: + - "lodash" + - "@types/lodash" + pg: + applies-to: version-updates + patterns: + - "pg" + - "@types/pg" prisma: applies-to: version-updates patterns: @@ -19,6 +33,12 @@ updates: patterns: - "react" - "react-dom" + - "@types/react" + react-query: + applies-to: version-updates + patterns: + - "@tanstack/react-query" + - "@tanstack/react-query-devtools" storybook: applies-to: version-updates patterns: @@ -32,3 +52,8 @@ updates: applies-to: version-updates patterns: - "@trpc/*" + validator: + applies-to: version-updates + patterns: + - "validator" + - "@types/validator" diff --git a/apps/studio/next.config.mjs b/apps/studio/next.config.mjs index dc6bc85c45..ed2a67d20b 100644 --- a/apps/studio/next.config.mjs +++ b/apps/studio/next.config.mjs @@ -25,12 +25,7 @@ const ContentSecurityPolicy = ` object-src 'none'; script-src 'self' 'unsafe-eval'; style-src 'self' https: 'unsafe-inline'; - connect-src 'self' https://schema.isomer.gov.sg https://browser-intake-datadoghq.com https://*.browser-intake-datadoghq.com https://vitals.vercel-insights.com/v1/vitals ${ - // For POSTing presigned URLs to R2 storage. - env.R2_ACCOUNT_ID - ? `https://*.${env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com` - : "" - }; + connect-src 'self' https://schema.isomer.gov.sg https://browser-intake-datadoghq.com https://*.browser-intake-datadoghq.com https://vitals.vercel-insights.com/v1/vitals; worker-src 'self' blob:; ${env.NODE_ENV === "production" ? "upgrade-insecure-requests" : ""} ` @@ -52,7 +47,10 @@ const config = { /** We run eslint as a separate task in CI */ eslint: { ignoreDuringBuilds: true }, images: { - domains: [env.R2_PUBLIC_HOSTNAME ?? ""].filter((d) => d), + domains: [ + env.S3_UNSAFE_ASSETS_DOMAIN_NAME ?? "", + env.S3_PUBLIC_ASSETS_DOMAIN_NAME ?? "", + ].filter((d) => d), }, async headers() { return [ diff --git a/apps/studio/src/env.mjs b/apps/studio/src/env.mjs index f9f1329259..5955708442 100644 --- a/apps/studio/src/env.mjs +++ b/apps/studio/src/env.mjs @@ -23,30 +23,14 @@ const client = z.object({ }) /** Feature flags */ -const baseR2Schema = z.object({ - R2_ACCESS_KEY_ID: z.string().optional(), - R2_ACCOUNT_ID: z.string().optional(), - R2_SECRET_ACCESS_KEY: z.string().optional(), - R2_PUBLIC_HOSTNAME: z.string().optional(), - R2_BUCKET_NAME: z.string().optional(), - R2_AVATARS_DIRECTORY: z.string().optional(), - R2_IMAGES_DIRECTORY: z.string().optional(), +const s3Schema = z.object({ + S3_REGION: z.string().default("us-east-1"), + S3_PUBLIC_ASSETS_DOMAIN_NAME: z.string(), + S3_UNSAFE_ASSETS_DOMAIN_NAME: z.string(), + S3_PUBLIC_ASSETS_BUCKET_NAME: z.string(), + S3_UNSAFE_ASSETS_BUCKET_NAME: z.string(), }) -const r2ServerSchema = z.discriminatedUnion("NEXT_PUBLIC_ENABLE_STORAGE", [ - baseR2Schema.extend({ - NEXT_PUBLIC_ENABLE_STORAGE: z.literal(true), - // Add required keys if flag is enabled. - R2_ACCESS_KEY_ID: z.string().min(1), - R2_ACCOUNT_ID: z.string().min(1), - R2_SECRET_ACCESS_KEY: z.string().min(1), - R2_PUBLIC_HOSTNAME: z.string().min(1), - }), - baseR2Schema.extend({ - NEXT_PUBLIC_ENABLE_STORAGE: z.literal(false), - }), -]) - const baseSgidSchema = z.object({ SGID_CLIENT_ID: z.string().optional(), SGID_CLIENT_SECRET: z.string().optional(), @@ -86,25 +70,12 @@ const server = z ]), SESSION_SECRET: z.string().min(32), }) + .merge(s3Schema) // Add on schemas as needed that requires conditional validation. - .merge(baseR2Schema) .merge(baseSgidSchema) .merge(client) // Add on refinements as needed for conditional environment variables // .superRefine((val, ctx) => ...) - .superRefine((val, ctx) => { - const parse = r2ServerSchema.safeParse(val) - if (!parse.success) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ["NEXT_PUBLIC_ENABLE_STORAGE"], - message: "R2 environment variables are missing", - }) - parse.error.issues.forEach((issue) => { - ctx.addIssue(issue) - }) - } - }) .superRefine((val, ctx) => { const parse = sgidServerSchema.safeParse(val) if (!parse.success) { @@ -139,13 +110,11 @@ const processEnv = { SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, SENDGRID_FROM_ADDRESS: process.env.SENDGRID_FROM_ADDRESS, SESSION_SECRET: process.env.SESSION_SECRET, - R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID, - R2_ACCOUNT_ID: process.env.R2_ACCOUNT_ID, - R2_BUCKET_NAME: process.env.R2_BUCKET_NAME, - R2_AVATARS_DIRECTORY: process.env.R2_AVATARS_DIRECTORY, - R2_IMAGES_DIRECTORY: process.env.R2_IMAGES_DIRECTORY, - R2_PUBLIC_HOSTNAME: process.env.R2_PUBLIC_HOSTNAME, - R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY, + S3_REGION: process.env.S3_REGION, + S3_PUBLIC_ASSETS_DOMAIN_NAME: process.env.S3_PUBLIC_ASSETS_DOMAIN_NAME, + S3_UNSAFE_ASSETS_DOMAIN_NAME: process.env.S3_UNSAFE_ASSETS_DOMAIN_NAME, + S3_PUBLIC_ASSETS_BUCKET_NAME: process.env.S3_PUBLIC_ASSETS_BUCKET_NAME, + S3_UNSAFE_ASSETS_BUCKET_NAME: process.env.S3_UNSAFE_ASSETS_BUCKET_NAME, SGID_CLIENT_ID: process.env.SGID_CLIENT_ID, SGID_CLIENT_SECRET: process.env.SGID_CLIENT_SECRET, SGID_PRIVATE_KEY: process.env.SGID_PRIVATE_KEY, diff --git a/apps/studio/src/features/editing-experience/components/form-builder/renderers/controls/JsonFormsImageControl.tsx b/apps/studio/src/features/editing-experience/components/form-builder/renderers/controls/JsonFormsImageControl.tsx index 21a7edcbb3..e2856dd335 100644 --- a/apps/studio/src/features/editing-experience/components/form-builder/renderers/controls/JsonFormsImageControl.tsx +++ b/apps/studio/src/features/editing-experience/components/form-builder/renderers/controls/JsonFormsImageControl.tsx @@ -40,7 +40,7 @@ export function JsonFormsImageControl({ {label} ): Promise => { + return getSignedUrl( + storage, + new PutObjectCommand({ + Bucket: S3_UNSAFE_ASSETS_BUCKET_NAME, + Key, + }), + { + expiresIn: 60 * 5, // 5 minutes + }, + ) +} diff --git a/apps/studio/src/schemas/asset.ts b/apps/studio/src/schemas/asset.ts new file mode 100644 index 0000000000..468cb73c41 --- /dev/null +++ b/apps/studio/src/schemas/asset.ts @@ -0,0 +1,8 @@ +import { z } from "zod" + +export const getPresignedPutUrlSchema = z.object({ + siteId: z.number().min(1), + fileName: z.string({ + required_error: "Missing file name", + }), +}) diff --git a/apps/studio/src/schemas/presign.ts b/apps/studio/src/schemas/presign.ts deleted file mode 100644 index f4e357732a..0000000000 --- a/apps/studio/src/schemas/presign.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { z } from "zod" - -import { ACCEPTED_FILE_TYPES } from "~/utils/image" - -export const presignImageInputSchema = z.object({ - fileContentType: z.enum(ACCEPTED_FILE_TYPES), -}) -export const presignImageOutputSchema = z - .object({ - url: z.string(), - key: z.string(), - }) - .or(z.null()) diff --git a/apps/studio/src/server/modules/_app.ts b/apps/studio/src/server/modules/_app.ts index 21ed1ffaef..cb79e85f46 100644 --- a/apps/studio/src/server/modules/_app.ts +++ b/apps/studio/src/server/modules/_app.ts @@ -2,6 +2,7 @@ * This file contains the root router of your tRPC-backend */ import { publicProcedure, router } from "../trpc" +import { assetRouter } from "./asset/asset.router" import { authRouter } from "./auth/auth.router" import { folderRouter } from "./folder/folder.router" import { meRouter } from "./me/me.router" @@ -12,6 +13,7 @@ export const appRouter = router({ healthcheck: publicProcedure.query(() => "yay!"), me: meRouter, auth: authRouter, + asset: assetRouter, page: pageRouter, folder: folderRouter, site: siteRouter, diff --git a/apps/studio/src/server/modules/asset/asset.router.ts b/apps/studio/src/server/modules/asset/asset.router.ts new file mode 100644 index 0000000000..4ac7f38ef1 --- /dev/null +++ b/apps/studio/src/server/modules/asset/asset.router.ts @@ -0,0 +1,33 @@ +import { getPresignedPutUrlSchema } from "~/schemas/asset" +import { protectedProcedure, router } from "~/server/trpc" +import { getFileKey, getPresignedPutUrl } from "./asset.service" + +export const assetRouter = router({ + getPresignedPutUrl: protectedProcedure + .input(getPresignedPutUrlSchema) + .mutation(async ({ ctx, input: { siteId, fileName } }) => { + const fileKey = getFileKey({ + siteId, + fileName, + }) + + const presignedPutUrl = await getPresignedPutUrl({ + key: fileKey, + }) + + ctx.logger.info( + `Generated presigned PUT URL for ${fileKey} for site ${siteId}`, + { + userId: ctx.session?.userId, + siteId, + fileName, + fileKey, + }, + ) + + return { + fileKey, + presignedPutUrl, + } + }), +}) diff --git a/apps/studio/src/server/modules/asset/asset.service.ts b/apps/studio/src/server/modules/asset/asset.service.ts new file mode 100644 index 0000000000..99faae6e93 --- /dev/null +++ b/apps/studio/src/server/modules/asset/asset.service.ts @@ -0,0 +1,21 @@ +import { randomUUID } from "crypto" +import type { z } from "zod" + +import type { getPresignedPutUrlSchema } from "~/schemas/asset" +import { generateSignedPutUrl } from "~/lib/s3" + +export const getFileKey = ({ + siteId, + fileName, +}: z.infer) => { + // NOTE: We're using a random folder name to prevent collisions + const folderName = randomUUID() + + return `${siteId}/${folderName}/${fileName}` +} + +export const getPresignedPutUrl = async ({ key }: { key: string }) => { + return generateSignedPutUrl({ + Key: key, + }) +} diff --git a/turbo.json b/turbo.json index a8b1fb44c1..fa822092ea 100644 --- a/turbo.json +++ b/turbo.json @@ -2,7 +2,15 @@ "$schema": "https://turbo.build/schema.json", "ui": "tui", "globalDependencies": ["**/.env", "tsconfig.json"], - "globalEnv": ["NODE_ENV", "DATABASE_URL", "SESSION_SECRET"], + "globalEnv": [ + "NODE_ENV", + "DATABASE_URL", + "SESSION_SECRET", + "S3_PUBLIC_ASSETS_DOMAIN_NAME", + "S3_UNSAFE_ASSETS_DOMAIN_NAME", + "S3_PUBLIC_ASSETS_BUCKET_NAME", + "S3_UNSAFE_ASSETS_BUCKET_NAME" + ], "tasks": { "build": { "dependsOn": ["^build"],