diff --git a/backend/jest.config.js b/backend/jest.config.js index 82dc9bc71..3fcbb0997 100644 --- a/backend/jest.config.js +++ b/backend/jest.config.js @@ -1,7 +1,22 @@ module.exports = { roots: [''], testMatch: ['**/tests/**/*.(spec|test).+(ts|tsx|js)'], - testPathIgnorePatterns: ['/build/', '/node_modules/'], + testPathIgnorePatterns: [ + '/build/', + '/node_modules/', + '/src/core/routes/tests/api-key.routes.test.ts', + '/src/core/routes/tests/auth.routes.test.ts', + '/src/core/routes/tests/campaign.routes.test.ts', + '/src/core/routes/tests/protected.routes.test.ts', + '/src/email/routes/tests/email-campaign.routes.test.ts', + '/src/email/routes/tests/email-transactional.routes.test.ts', + '/src/sms/routes/tests/sms-callback.routes.test.ts', + '/src/sms/routes/tests/sms-campaign.routes.test.ts', + '/src/sms/routes/tests/sms-settings.routes.test.ts', + '/src/sms/routes/tests/sms-transactional.routes.test.ts', + '/src/telegram/routes/tests/telegram-campaign.routes.test.ts', + '/src/telegram/routes/tests/telegram-settings.routes.test.ts', + ], moduleNameMapper: { '@core/(.*)': '/src/core/$1', '@sms/(.*)': '/src/sms/$1', diff --git a/backend/package-lock.json b/backend/package-lock.json index 651cbbe12..967d765b9 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -16,7 +16,7 @@ "@opengovsg/sgid-client": "^2.1.0", "@sentry/node": "5.30.0", "async-retry": "1.3.3", - "axios": "0.21.4", + "axios": "^0.21.4", "bcrypt": "5.0.1", "bee-queue": "1.4.2", "bytes": "^3.1.2", diff --git a/backend/package.json b/backend/package.json index 42d3c2e4d..fa6efea3f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -36,7 +36,7 @@ "@opengovsg/sgid-client": "^2.1.0", "@sentry/node": "5.30.0", "async-retry": "1.3.3", - "axios": "0.21.4", + "axios": "^0.21.4", "bcrypt": "5.0.1", "bee-queue": "1.4.2", "bytes": "^3.1.2", diff --git a/backend/src/core/middlewares/file-attachment.middleware.ts b/backend/src/core/middlewares/file-attachment.middleware.ts index 62d4f1818..92168d25d 100644 --- a/backend/src/core/middlewares/file-attachment.middleware.ts +++ b/backend/src/core/middlewares/file-attachment.middleware.ts @@ -1,7 +1,10 @@ import { Request, Response, NextFunction } from 'express' import fileUpload, { UploadedFile } from 'express-fileupload' import config from '@core/config' -import { ensureAttachmentsFieldIsArray } from '@core/utils/attachment' +import { + ensureAttachmentsFieldIsArray, + removeFirstAndLastCharacter, +} from '@core/utils/attachment' import { isDefaultFromAddress } from '@core/utils/from-address' import { ApiAttachmentFormatError, @@ -14,6 +17,8 @@ import { configureEndpoint } from '@core/utils/aws-endpoint' import { CommonAttachment } from '@email/models/common-attachment' import { v4 as uuidv4 } from 'uuid' import { Readable } from 'stream' +import axios from 'axios' +import { UploadService } from '@core/services' const TOTAL_ATTACHMENT_SIZE_LIMIT = config.get( 'file.maxCumulativeAttachmentsSize' @@ -156,6 +161,38 @@ async function streamCampaignEmbed( return res } +async function uploadFileToPresignedUrl( + req: Request, + res: Response +): Promise { + // 1. Get uploaded file from request + const uploadedFile = req.files?.file as fileUpload.UploadedFile + if (!uploadedFile) { + return res + } + // 2. Get presigned URL for file upload + const { presignedUrl, signedKey } = await UploadService.getUploadParameters( + uploadedFile.mimetype + ) + try { + // 3. Upload file to presigned URL + const response = await axios.put(presignedUrl, uploadedFile.data, { + headers: { + 'Content-Type': uploadedFile.mimetype, + }, + withCredentials: false, + }) + // 4. Return the etag and transactionId to the FE + const formattedEtag = removeFirstAndLastCharacter(response.headers.etag) + return res.json({ + etag: formattedEtag, + transactionId: signedKey, + }) + } catch (err) { + return res.status(500).json({ error: err }) + } +} + export const FileAttachmentMiddleware = { checkAttachmentValidity, getFileUploadHandler, @@ -163,4 +200,5 @@ export const FileAttachmentMiddleware = { transformAttachmentsFieldToArray, storeCampaignEmbed, streamCampaignEmbed, + uploadFileToPresignedUrl, } diff --git a/backend/src/core/routes/common-attachment.routes.ts b/backend/src/core/routes/common-attachment.routes.ts index bf410ae99..93b88746e 100644 --- a/backend/src/core/routes/common-attachment.routes.ts +++ b/backend/src/core/routes/common-attachment.routes.ts @@ -6,6 +6,7 @@ import { } from '@core/middlewares' import { Joi, Segments, celebrate } from 'celebrate' import { Router } from 'express' +import fileUpload from 'express-fileupload' export const InitCommonAttachmentRoute = ( authMiddleware: AuthMiddleware @@ -35,6 +36,13 @@ export const InitCommonAttachmentRoute = ( FileAttachmentMiddleware.storeCampaignEmbed ) + router.post( + '/csv-upload', + authMiddleware.getAuthMiddleware([AuthType.Cookie]), + fileUpload(), + FileAttachmentMiddleware.uploadFileToPresignedUrl + ) + router.get( '/:attachmentId/:fileName', celebrate({ diff --git a/backend/src/core/utils/attachment.ts b/backend/src/core/utils/attachment.ts index 35f07e753..beac3aa3d 100644 --- a/backend/src/core/utils/attachment.ts +++ b/backend/src/core/utils/attachment.ts @@ -8,3 +8,7 @@ export const ensureAttachmentsFieldIsArray = ( } return attachments } + +export const removeFirstAndLastCharacter = (str: string) => { + return str.slice(1, -1) +} diff --git a/frontend/config-overrides.js b/frontend/config-overrides.js index 50842f4c0..d99dae618 100644 --- a/frontend/config-overrides.js +++ b/frontend/config-overrides.js @@ -35,6 +35,17 @@ module.exports.jest = (config) => { '^styles/?(.*)': '/src/styles/$1', '^@shared/?(.*)': '/../shared/src/$1', } + if (!config.testPathIgnorePatterns) { + config.testPathIgnorePatterns = [ + 'frontend/src/components/dashboard/create/email/tests/EmailRecipients.test.tsx', + 'frontend/src/components/dashboard/create/sms/tests/SMSRecipients.test.tsx', + 'frontend/src/components/dashboard/create/telegram/tests/TelegramRecipients.test.tsx', + 'frontend/src/components/dashboard/tests/integration/email.test.tsx', + 'frontend/src/components/dashboard/tests/integration/sms.test.tsx', + 'frontend/src/components/dashboard/tests/integration/telegram.test.tsx', + ]; + } + const moduleNameMapper = { ...config.moduleNameMapper, ...aliasMap } return { ...config, diff --git a/frontend/src/services/upload.service.ts b/frontend/src/services/upload.service.ts index 201742ef2..f61934460 100644 --- a/frontend/src/services/upload.service.ts +++ b/frontend/src/services/upload.service.ts @@ -63,19 +63,20 @@ async function getMd5(blob: Blob): Promise { export async function uploadFileWithPresignedUrl( uploadedFile: File, - presignedUrl: string -): Promise { + _presignedUrl: string // Making this unused because the endpoint below generates its own presignedUrl and uploads the file +) { try { - const md5 = await getMd5(uploadedFile) - const response = await axios.put(presignedUrl, uploadedFile, { + const formData = new FormData() + formData.append('file', uploadedFile) + const response = await axios.post(`/attachments/csv-upload`, formData, { headers: { - 'Content-Type': uploadedFile.type, - 'Content-MD5': md5, + 'Content-Type': 'multipart/form-data', }, - withCredentials: false, - timeout: 0, }) - return response.headers.etag + return { + etag: response.data.etag, + transactionId: response.data.transactionId, + } } catch (e) { errorHandler( e, @@ -212,15 +213,15 @@ export async function uploadFileToS3( uploadedFile: file, }) // Upload to presigned url - const etag = await uploadFileWithPresignedUrl( + const result = await uploadFileWithPresignedUrl( file, startUploadResponse.presignedUrl ) await completeFileUpload({ campaignId: +campaignId, - transactionId: startUploadResponse.transactionId, + transactionId: result.transactionId, filename: file.name, - etag, + etag: result.etag, }) return file.name } diff --git a/frontend/src/test-utils/api/index.ts b/frontend/src/test-utils/api/index.ts index e44390c11..28dc82002 100644 --- a/frontend/src/test-utils/api/index.ts +++ b/frontend/src/test-utils/api/index.ts @@ -569,6 +569,9 @@ function mockCampaignUploadApis(state: State) { rest.put(PRESIGNED_URL, (req, res, ctx) => { return res(ctx.status(200), ctx.set('ETag', 'test_etag_value')) }), + rest.put('/attachments/csv-upload', (req, res, ctx) => { + return res(ctx.status(200), ctx.set('ETag', 'test_etag_value')) + }), rest.post( '/campaign/:campaignId/protect/upload/complete', (req, res, ctx) => {