feat: download s3 clean file

src/app/modules/submission/encrypt-submission/encrypt-submission.errors.ts

@@ -69,3 +69,11 @@ export class JsonParseFailedError extends ApplicationError {
     super(message)
   }
 }
+
+export class DownloadCleanFileFailedError extends ApplicationError {
+  constructor(
+    message = 'Attempt to download clean file failed. Please try again.',
+  ) {
+    super(message)
+  }
+}

src/app/modules/submission/encrypt-submission/encrypt-submission.middleware.ts

@@ -3,7 +3,7 @@ import { celebrate, Joi, Segments } from 'celebrate'
 import { NextFunction } from 'express'
 import { StatusCodes } from 'http-status-codes'
 import { chain, omit } from 'lodash'
-import { okAsync, ResultAsync } from 'neverthrow'
+import { okAsync, Result, ResultAsync } from 'neverthrow'
 
 import { featureFlags } from '../../../../../shared/constants'
 import {
@@ -15,6 +15,7 @@ import {
   EncryptAttachmentResponse,
   EncryptFormFieldResponse,
   FormLoadedDto,
+  ParsedClearAttachmentResponse,
 } from '../../../../types/api'
 import { paymentConfig } from '../../../config/features/payment.config'
 import formsgSdk from '../../../config/formsg-sdk'
@@ -34,16 +35,19 @@ import {
 import {
   EncryptedPayloadExistsError,
   FormsgReqBodyExistsError,
+  VirusScanFailedError,
 } from './encrypt-submission.errors'
 import {
   checkFormIsEncryptMode,
+  downloadCleanFile,
   triggerVirusScanning,
 } from './encrypt-submission.service'
 import {
   CreateFormsgAndRetrieveFormMiddlewareHandlerRequest,
   CreateFormsgAndRetrieveFormMiddlewareHandlerType,
   EncryptSubmissionMiddlewareHandlerRequest,
   EncryptSubmissionMiddlewareHandlerType,
+  ParseVirusScannerLambdaPayloadOkBody,
   StorageSubmissionMiddlewareHandlerRequest,
   StorageSubmissionMiddlewareHandlerType,
   ValidateSubmissionMiddlewareHandlerRequest,
@@ -225,13 +229,26 @@ export const scanAndRetrieveAttachments = async (
   // have virus scanning enabled.
 
   // Step 3: Trigger lambda to scan attachments.
-  const triggerLambdaResult = await ResultAsync.combine(
+  const triggerLambdaResult: Result<
+    (
+      | false
+      | {
+          response: ParsedClearAttachmentResponse
+          cleanFile: ParseVirusScannerLambdaPayloadOkBody
+        }
+    )[],
+    VirusScanFailedError
+  > = await ResultAsync.combine(
     req.body.responses.map((response) => {
       if (isQuarantinedAttachmentResponse(response)) {
-        return triggerVirusScanning(response.answer)
+        return triggerVirusScanning(response.filename).map((result) => ({
+          response,
+          cleanFile: result.body,
+        }))
       }
+
       // If response is not an attachment, return ok.
-      return okAsync(true)
+      return okAsync(false as const)
     }),
   )
 
@@ -243,13 +260,69 @@ export const scanAndRetrieveAttachments = async (
       error: triggerLambdaResult.error,
     })
 
-    return res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({
-      message:
-        'Something went wrong while scanning your attachments. Please try again.',
+    const { statusCode, errorMessage } = mapRouteError(
+      triggerLambdaResult.error,
+    )
+    return res.status(statusCode).json({
+      message: errorMessage,
     })
   }
 
-  // TODO(FRM-1302): Step 4: Retrieve attachments from the clean bucket.
+  // const getBody = (response: GetObjectCommandOutput) => {
+  //   return response.Body && (response.Body as Readable);
+  // }
+
+  // const bodyToBuffer = (body: S3.Body) => {
+  //   let buffer: Buffer
+  //   if (body instanceof Uint8Array || body instanceof Blob) {
+  //     buffer = Buffer.from(body)
+  //   } else if (
+  //     typeof body === 'string' ||
+  //     body instanceof Readable ||
+  //     body instanceof ReadableStream
+  //   ) {
+  //     const chunks: Uint8Array[] = []
+  //     for await (const chunk of body) {
+  //       chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk))
+  //     }
+  //     buffer = Buffer.concat(chunks)
+  //   } else {
+  //     throw new Error('Invalid type for S3 object Body')
+  //   }
+  // }
+
+  // Step 4: Retrieve attachments from the clean bucket.
+  const downloadCleanFilesResult = await ResultAsync.combine(
+    triggerLambdaResult.value.map((isAttachment) => {
+      if (isAttachment) {
+        // Retrieve attachment from clean bucket.
+        return downloadCleanFile(
+          isAttachment.cleanFile.cleanFileKey,
+          isAttachment.cleanFile.destinationVersionId,
+        ).map((result) => {
+          isAttachment.response.content = result
+          return true
+        })
+      }
+
+      return okAsync(false as const)
+    }),
+  )
+
+  if (downloadCleanFilesResult.isErr()) {
+    logger.error({
+      message: 'Error downloading clean attachments',
+      meta: logMeta,
+      error: downloadCleanFilesResult.error,
+    })
+
+    const { statusCode, errorMessage } = mapRouteError(
+      downloadCleanFilesResult.error,
+    )
+    return res.status(statusCode).json({
+      message: errorMessage,
+    })
+  }
 
   return next()
 }
@@ -508,6 +581,8 @@ export const createFormsgAndRetrieveForm = async (
 ) => {
   const { formId } = req.params
 
+  console.log('createFormsgAndRetrieveForm req.body.version', req.body.version)
+
   const logMeta = {
     action: 'createFormsgAndRetrieveForm',
     ...createReqMeta(req),

src/app/modules/submission/encrypt-submission/encrypt-submission.service.ts

@@ -7,7 +7,7 @@ import { StatusCodes } from 'http-status-codes'
 import moment from 'moment'
 import mongoose from 'mongoose'
 import { err, errAsync, ok, okAsync, Result, ResultAsync } from 'neverthrow'
-import { Transform } from 'stream'
+import { Transform, Writable } from 'stream'
 import { validate } from 'uuid'
 
 import {
@@ -63,6 +63,7 @@ import {
 import { PRESIGNED_ATTACHMENT_POST_EXPIRY_SECS } from './encrypt-submission.constants'
 import {
   AttachmentSizeLimitExceededError,
+  DownloadCleanFileFailedError,
   InvalidFieldIdError,
   InvalidQuarantineFileKeyError,
   JsonParseFailedError,
@@ -786,3 +787,70 @@ export const triggerVirusScanning = (
     return errAsync(new VirusScanFailedError())
   })
 }
+
+export const downloadCleanFile = (cleanFileKey: string, versionId: string) => {
+  const logMeta = {
+    action: 'downloadCleanFile',
+    cleanFileKey,
+    versionId,
+  }
+
+  let buffer = Buffer.alloc(0)
+
+  const writeStream = new Writable({
+    write(chunk, encoding, callback) {
+      buffer = Buffer.concat([buffer, chunk])
+      callback()
+    },
+  })
+
+  const readStream = AwsConfig.s3
+    .getObject({
+      Bucket: AwsConfig.virusScannerCleanS3Bucket,
+      Key: cleanFileKey,
+      VersionId: versionId,
+    })
+    .createReadStream()
+
+  readStream.pipe(writeStream)
+
+  return ResultAsync.fromPromise(
+    new Promise<Buffer>((resolve, reject) => {
+      readStream.on('end', () => {
+        resolve(buffer)
+      })
+
+      readStream.on('error', (error) => {
+        reject(error)
+      })
+    }),
+    (error) => {
+      logger.error({
+        message: 'Error encountered when downloading file from clean bucket',
+        meta: logMeta,
+        error,
+      })
+
+      return new DownloadCleanFileFailedError()
+    },
+  )
+
+  // return ResultAsync.fromPromise(
+  //   AwsConfig.s3
+  //     .getObject({
+  //       Bucket: AwsConfig.virusScannerCleanS3Bucket,
+  //       Key: cleanFileKey,
+  //       VersionId: versionId,
+  //     })
+  //     .promise(),
+  //   (error) => {
+  //     logger.error({
+  //       message: 'Error encountered when invoking virus scanning lambda',
+  //       meta: logMeta,
+  //       error,
+  //     })
+
+  //     return new DownloadCleanFileFailedError()
+  //   },
+  // )
+}

src/app/modules/submission/receiver/receiver.service.ts

@@ -152,7 +152,11 @@ export const configureMultipartReceiver = (
         .on('close', () => {
           if (body) {
             handleDuplicatesInAttachments(attachments)
-            addAttachmentToResponses(body.responses, attachments)
+            addAttachmentToResponses(
+              body.responses,
+              attachments,
+              (body.version ?? 0) >= 2.1,
+            )
             return resolve(body)
           } else {
             // if body is not defined, the Promise would have been rejected elsewhere.

src/app/modules/submission/receiver/receiver.types.ts

@@ -4,4 +4,5 @@ import { FieldResponse } from '../../../../types'
 export interface ParsedMultipartForm {
   responses: FieldResponse[]
   responseMetadata: ResponseMetadata
+  version?: number
 }

src/app/modules/submission/receiver/receiver.utils.ts

@@ -66,6 +66,7 @@ const isAttachmentResponseFromMap = (
 export const addAttachmentToResponses = (
   responses: ParsedClearFormFieldResponse[],
   attachments: IAttachmentInfo[],
+  isVirusScannerEnabled: boolean,
 ): void => {
   // Create a map of the attachments with fieldId as keys
   const attachmentMap: Record<IAttachmentInfo['fieldId'], IAttachmentInfo> =
@@ -79,9 +80,11 @@ export const addAttachmentToResponses = (
     responses.forEach((response) => {
       if (isAttachmentResponseFromMap(attachmentMap, response)) {
         const file = attachmentMap[response._id]
-        response.answer = file.filename
         response.filename = file.filename
         response.content = file.content
+        if (!isVirusScannerEnabled) {
+          response.answer = file.filename
+        }
       }
     })
   }