From 87dc2524681d1c0e891a4588513faab8e9472711 Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Thu, 22 Feb 2024 11:37:11 +0800 Subject: [PATCH 01/15] chore(OSS): add FerretDB migration instructions (#7107) chore: add ferretdb migration instructions --- README.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e49b6778a8..d9f7165572 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## Table of Contents - [Contributing](#contributing) - - [IMPORTANT NOTE TO ALL CONTRIBUTORS](#important-note-to-all-contributors) + - [IMPORTANT NOTE TO ALL CONTRIBUTORS](#important-note-to-all-contributors) - [Features](#features) - [Local Development (Docker)](#local-development-docker) - [Prerequisites](#prerequisites) @@ -31,6 +31,7 @@ - [MongoDB Scripts](#mongodb-scripts) - [Support](#support) - [Database Alternatives](#database-alternatives) + - [Migrating from MongoDB to FerretDB](#migrating-from-mongodb-to-ferretdb) - [Migrating from Mongoose ODM to Prisma ORM](#migrating-from-mongoose-odm-to-prisma-orm) - [Replacing MongoDB with CockroachDB](#replacing-mongodb-with-cockroachdb) - [Other Prisma supported DBs](#other-prisma-supported-dbs) @@ -225,6 +226,51 @@ Please contact FormSG (support@form.gov.sg) for any details. ## Database Alternatives +### Migrating from MongoDB to FerretDB +[FerretDB](https://ferretdb.io) is an open source MongoDB alternative built on PostgreSQL. MongoDB can be swapped out of FormSG for FerretDB. In order for this to be done, certain changes to the code should be made as described below: + +- Add postgres to the list of services in the `docker.compose` file e.g. + ``` pg: + image: postgres:15.3-alpine3.18 + environment: + - POSTGRES_USER= + - POSTGRES_PASSWORD= + - POSTGRES_DB= + volumes: + - pgdata:/var/lib/postgresql/data + ports: + - '5432:5432' +- In the same file, change the "database" image from MongoDB to FerretDB and update the database section to include the lines below: + ``` + image: ghcr.io/ferretdb/ferretdb:1.17.0 + environment: + - FERRETDB_TELEMETRY=disable + - FERRETDB_POSTGRESQL_URL=postgres://pg:5432/formsg?user=&password= + ports: + - '8080:8080' + depends_on: + - pg +- Lastly, add the *pgdata* volume + ``` + volumes: + mongodb_data: + driver: local + pgdata: +- FerretDB currently has some limitations and [certain database features are not supported](https://docs.ferretdb.io/reference/supported-commands/), these include TTL, database transactions and some aggregration pipelines which are all features used by FormSG. + + The following changes can be made to mitigate the limitations of FerretDB: + + - Add the *autoRemove: 'interval'* property to the initializing of the session object in the `session.ts` file. + - Remove the unsupported [aggregration pipeline stages](https://docs.ferretdb.io/reference/supported-commands/#aggregation-pipeline-stages) e.g. *lookup* and *project*, in the `submission.server.model.ts` file. + - Replace the *findOneAndUpdate* code block in the `user.server.model.ts` file with code similar to the one below: + ``` + const user = await this.exists({ email: upsertParams.email }) + if (!user) { + await this.create(upsertParams) + } + return this.findOne({ + email: upsertParams.email, + }).populate({... ### Migrating from Mongoose ODM to Prisma ORM FormSG uses Mongoose as the Object-Document Mapping (ODM) to MongoDB. This means that our code is strongly coupled with MongoDB as Mongoose solely supports it. From 7e1b9c86f2d5965ff2168313884bfdd3eca531f4 Mon Sep 17 00:00:00 2001 From: Kathleen Koh <89055608+kathleenkhy@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:57:45 +0800 Subject: [PATCH 02/15] fix: correct alignment issues in preview mode (#7109) * fix: correct alignment issues in preview mode * fix: add different padding for edit vs preview mode --- .../common/components/PreviewFormBanner/PreviewFormBanner.tsx | 2 +- .../FormEndPage/components/PaymentEndPagePreview.tsx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/admin-form/common/components/PreviewFormBanner/PreviewFormBanner.tsx b/frontend/src/features/admin-form/common/components/PreviewFormBanner/PreviewFormBanner.tsx index 53991ee8fe..966aac2551 100644 --- a/frontend/src/features/admin-form/common/components/PreviewFormBanner/PreviewFormBanner.tsx +++ b/frontend/src/features/admin-form/common/components/PreviewFormBanner/PreviewFormBanner.tsx @@ -47,7 +47,7 @@ interface PreviewFormBannerProps { const textProps: TextProps = { textStyle: 'body-2', color: 'white', - ml: '2rem', + mx: '2rem', mt: '0.5rem', mb: '0.5rem', } diff --git a/frontend/src/features/public-form/components/FormEndPage/components/PaymentEndPagePreview.tsx b/frontend/src/features/public-form/components/FormEndPage/components/PaymentEndPagePreview.tsx index 7daa3fb9d3..09a5dac02d 100644 --- a/frontend/src/features/public-form/components/FormEndPage/components/PaymentEndPagePreview.tsx +++ b/frontend/src/features/public-form/components/FormEndPage/components/PaymentEndPagePreview.tsx @@ -31,8 +31,9 @@ export const PaymentEndPagePreview = ({ {isFeedbackSubmitted ? null : ( From ac0c695e2137d11e178a40ff2a8db6ceadbaba36 Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Mon, 26 Feb 2024 14:28:26 +0800 Subject: [PATCH 03/15] feat(mrf): disable unsupported features (#7106) * feat: remove attachment field on mrf builder * feat: disable vfn on mobile no * feat: disable vfn on email * feat: hide feedback section on mrf * chore: remove redundant comments, rename boolean to be more grammatically consistent * feat: disable email cnfrmtn on email field * fix: add missing storybook params * chore: rename props --- .../edit-fieldtype/EditEmail/EditEmail.tsx | 19 +++++++++-- .../edit-fieldtype/EditMobile/EditMobile.tsx | 10 +++++- .../field-panels/BasicFieldPanel.tsx | 32 ++++++++++++++----- .../FormEndPage/FormEndPage.stories.tsx | 4 ++- .../components/FormEndPage/FormEndPage.tsx | 6 ++-- .../FormEndPage/FormEndPageContainer.tsx | 8 ++++- 6 files changed, 63 insertions(+), 16 deletions(-) diff --git a/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/EditFieldDrawer/edit-fieldtype/EditEmail/EditEmail.tsx b/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/EditFieldDrawer/edit-fieldtype/EditEmail/EditEmail.tsx index dd456dfb0a..dfa29143a8 100644 --- a/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/EditFieldDrawer/edit-fieldtype/EditEmail/EditEmail.tsx +++ b/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/EditFieldDrawer/edit-fieldtype/EditEmail/EditEmail.tsx @@ -139,6 +139,18 @@ export const EditEmail = ({ field }: EditEmailProps): JSX.Element => { } }, [isPdfResponseEnabled]) + // vfn is not supported on MRF + const isToggleVfnDisabled = useMemo( + () => form?.responseMode === FormResponseMode.Multirespondent, + [form], + ) + + // email confirmation is not supported on MRF + const isToggleEmailConfirmationDisabled = useMemo( + () => form?.responseMode === FormResponseMode.Multirespondent, + [form], + ) + return ( @@ -158,7 +170,7 @@ export const EditEmail = ({ field }: EditEmailProps): JSX.Element => { - + { )} - + { const { data: freeSmsCount } = useFreeSmsQuota() const isToggleVfnDisabled = useMemo(() => { + // vfn is not supported on MRF + if (form?.responseMode === FormResponseMode.Multirespondent) return true if (!freeSmsCount) return true return ( !field.isVerifiable && !hasTwilioCredentials && freeSmsCount.freeSmsCounts >= freeSmsCount.quota ) - }, [field.isVerifiable, freeSmsCount, hasTwilioCredentials]) + }, [ + field.isVerifiable, + freeSmsCount, + hasTwilioCredentials, + form?.responseMode, + ]) const smsCountsDisclosure = useDisclosure() diff --git a/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/FieldListDrawer/field-panels/BasicFieldPanel.tsx b/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/FieldListDrawer/field-panels/BasicFieldPanel.tsx index 0e9667fc1b..b3c75d58e0 100644 --- a/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/FieldListDrawer/field-panels/BasicFieldPanel.tsx +++ b/frontend/src/features/admin-form/create/builder-and-design/BuilderAndDesignDrawer/FieldListDrawer/field-panels/BasicFieldPanel.tsx @@ -1,6 +1,9 @@ import { Droppable } from 'react-beautiful-dnd' import { Box } from '@chakra-ui/react' +import { BasicField, FormResponseMode } from '~shared/types' + +import { useAdminForm } from '~features/admin-form/common/queries' import { BASIC_FIELDS_ORDERED, CREATE_FIELD_DROP_ID, @@ -13,20 +16,33 @@ import { FieldSection } from './FieldSection' export const BasicFieldPanel = () => { const { isLoading } = useCreateTabForm() + const { data: form } = useAdminForm() return ( {(provided) => ( - {BASIC_FIELDS_ORDERED.map((fieldType, index) => ( - - ))} + {BASIC_FIELDS_ORDERED.map((fieldType, index) => { + let shouldDisableField = isLoading + + // Attachment is not supported on MRF + if ( + fieldType === BasicField.Attachment && + form?.responseMode === FormResponseMode.Multirespondent + ) { + shouldDisableField = true + } + + return ( + + ) + })} {provided.placeholder} diff --git a/frontend/src/features/public-form/components/FormEndPage/FormEndPage.stories.tsx b/frontend/src/features/public-form/components/FormEndPage/FormEndPage.stories.tsx index e33db29525..c135214869 100644 --- a/frontend/src/features/public-form/components/FormEndPage/FormEndPage.stories.tsx +++ b/frontend/src/features/public-form/components/FormEndPage/FormEndPage.stories.tsx @@ -43,6 +43,8 @@ Default.args = { title: 'Thank you for your submission with some super long backstory about how important the submission is to them', paragraph: 'We will get back to you shortly.\n\nOnce again,\r\nthank you.', + paymentTitle: '', + paymentParagraph: '', }, submissionData: { id: 'mockSubmissionId', @@ -84,7 +86,7 @@ ColorThemeOrange.args = { export const FeedbackSubmitted = Template.bind({}) FeedbackSubmitted.args = { ...Default.args, - isFeedbackSubmitted: true, + isFeedbackSectionHidden: true, } export const Mobile = Template.bind({}) diff --git a/frontend/src/features/public-form/components/FormEndPage/FormEndPage.tsx b/frontend/src/features/public-form/components/FormEndPage/FormEndPage.tsx index 798dea40e8..c3eb632e64 100644 --- a/frontend/src/features/public-form/components/FormEndPage/FormEndPage.tsx +++ b/frontend/src/features/public-form/components/FormEndPage/FormEndPage.tsx @@ -13,13 +13,13 @@ export interface FormEndPageProps { endPage: FormDto['endPage'] submissionData: SubmissionData handleSubmitFeedback: (inputs: FeedbackFormInput) => void - isFeedbackSubmitted: boolean + isFeedbackSectionHidden: boolean colorTheme: FormColorTheme } export const FormEndPage = ({ handleSubmitFeedback, - isFeedbackSubmitted, + isFeedbackSectionHidden, colorTheme, ...endPageProps }: FormEndPageProps): JSX.Element => { @@ -40,7 +40,7 @@ export const FormEndPage = ({ {...endPageProps} colorTheme={colorTheme} /> - {isFeedbackSubmitted ? null : ( + {isFeedbackSectionHidden ? null : ( { /> ) } + + const isFeedbackHidden = + // Feedback is not supported on MRF + form.responseMode === FormResponseMode.Multirespondent || + isFeedbackSubmitted + return ( { submissionData={submissionData} formTitle={form.title} endPage={form.endPage} - isFeedbackSubmitted={isFeedbackSubmitted} + isFeedbackSectionHidden={isFeedbackHidden} handleSubmitFeedback={handleSubmitFeedback} /> From c2db633846df241094398f63c55a8a1a5d64da8b Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Tue, 27 Feb 2024 20:22:31 +0800 Subject: [PATCH 04/15] chore(vscode): update code actions value options (#7113) chore: update code actions value options --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9791ff2909..5ab2664b7d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "eslint.validate": ["javascript", "typescript"], "editor.formatOnSave": true, @@ -12,7 +12,7 @@ }, "[html]": { "editor.codeActionsOnSave": { - "source.fixAll.eslint": false + "source.fixAll.eslint": "never" }, "editor.defaultFormatter": "esbenp.prettier-vscode" }, From da55f39ac14b2f1ed27d5979123f2eb10e0b8b72 Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Thu, 29 Feb 2024 23:47:52 +0800 Subject: [PATCH 05/15] chore: remove Anguilla from country listing (#7108) --- shared/constants/countryRegion.ts | 1 - shared/constants/field/myinfo/myinfo-countries.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/shared/constants/countryRegion.ts b/shared/constants/countryRegion.ts index 393f342fc8..66da81c73e 100644 --- a/shared/constants/countryRegion.ts +++ b/shared/constants/countryRegion.ts @@ -5,7 +5,6 @@ export enum CountryRegion { American_Samoa = 'American Samoa', Andorra = 'Andorra', Angola = 'Angola', - Anguilla = 'Anguilla', Antigua = 'Antigua', Argentina = 'Argentina', Armenia = 'Armenia', diff --git a/shared/constants/field/myinfo/myinfo-countries.ts b/shared/constants/field/myinfo/myinfo-countries.ts index cc2e5efec6..d14bff865a 100644 --- a/shared/constants/field/myinfo/myinfo-countries.ts +++ b/shared/constants/field/myinfo/myinfo-countries.ts @@ -5,7 +5,6 @@ export const myInfoCountries = [ 'AMERICAN SAMOA', 'ANDORRA', 'ANGOLA', - 'ANGUILLA', 'ANTIGUA', 'ARGENTINA', 'ARMENIA', From e0d6ac4159a55b7b8e0001b433d3556efff73800 Mon Sep 17 00:00:00 2001 From: LoneRifle Date: Fri, 1 Mar 2024 10:21:05 +0800 Subject: [PATCH 06/15] feat(virus-scanner): allow endpoint to be specified (#7114) For environments that are neither AWS nor local dev, the virus scanner lambda might be hosted elsewhere, eg, on a host running an emulated environment. Grant the flexibility to specify the endpoint address to reach the virus scanner, and thus the discretion to use AWS or some other environment. --- src/app/config/config.ts | 10 ++++++++-- src/app/config/schema.ts | 6 ++++++ src/types/config.ts | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/app/config/config.ts b/src/app/config/config.ts index 524f95ad72..f242b94ec4 100644 --- a/src/app/config/config.ts +++ b/src/app/config/config.ts @@ -97,10 +97,16 @@ const s3 = new aws.S3({ // using aws-sdk v3 (FRM-993) const virusScannerLambda = new Lambda({ region: basicVars.awsConfig.region, - // Endpoint is set for development mode to point to the separate docker container running the lambda function. + // For dev mode or where specified, endpoint is set to point to the separate docker container running the lambda function. // host.docker.internal is a special DNS name which resolves to the internal IP address used by the host. // Reference: https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host - ...(isDev ? { endpoint: 'http://host.docker.internal:9999' } : undefined), + ...(isDev || basicVars.awsConfig.virusScannerLambdaEndpoint + ? { + endpoint: + basicVars.awsConfig.virusScannerLambdaEndpoint || + 'http://host.docker.internal:9999', + } + : undefined), }) const awsConfig: AwsConfig = { diff --git a/src/app/config/schema.ts b/src/app/config/schema.ts index e17f25d980..ee6361f134 100644 --- a/src/app/config/schema.ts +++ b/src/app/config/schema.ts @@ -297,6 +297,12 @@ export const optionalVarsSchema: Schema = { default: '', env: 'VIRUS_SCANNER_LAMBDA_FUNCTION_NAME', }, + virusScannerLambdaEndpoint: { + doc: 'Endpoint address for virus scanner lambda function. Specify this if the lambda is hosted neither on AWS nor your local dev environment.', + format: String, + default: '', + env: 'VIRUS_SCANNER_LAMBDA_ENDPOINT', + }, }, core: { port: { diff --git a/src/types/config.ts b/src/types/config.ts index 0c85b6d056..e76271230b 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -168,6 +168,7 @@ export interface IOptionalVarsSchema { region: string customCloudWatchGroup: string virusScannerLambdaFunctionName: string + virusScannerLambdaEndpoint: string } mail: { from: string From 3747f92e1fed27137f8da7d50e9f7cb96a467701 Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Mon, 4 Mar 2024 10:42:26 +0800 Subject: [PATCH 07/15] feat(fe): update copy, copy btn (#7116) * feat: update copy, copy btn * chore: remove unintended box-flex switch --- .../stripe/StripePaymentElement.tsx | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/frontend/src/features/public-form/components/FormPaymentPage/stripe/StripePaymentElement.tsx b/frontend/src/features/public-form/components/FormPaymentPage/stripe/StripePaymentElement.tsx index 8f5b491b8e..b90a493aa7 100644 --- a/frontend/src/features/public-form/components/FormPaymentPage/stripe/StripePaymentElement.tsx +++ b/frontend/src/features/public-form/components/FormPaymentPage/stripe/StripePaymentElement.tsx @@ -1,6 +1,6 @@ import { useMemo, useState } from 'react' import { useParams } from 'react-router-dom' -import { Box, Center, Container } from '@chakra-ui/react' +import { Box, Center, Container, Flex, Stack, Text } from '@chakra-ui/react' import { Elements, useStripe } from '@stripe/react-stripe-js' import { loadStripe } from '@stripe/stripe-js' @@ -11,6 +11,7 @@ import { } from '~shared/types' import InlineMessage from '~components/InlineMessage' +import { CopyButton } from '~templates/CopyButton' import { useEnv } from '~features/env/queries' @@ -155,9 +156,22 @@ const StripePaymentContainer = ({ {secretEnv === 'production' ? null : ( - Use '4242 4242 4242 4242' as your card number to test payments - on this form. Payments made on this form will only show in - test mode in Stripe. + + + Make a test payment with the card number below! Payments + made on this form will only show in test mode in Stripe. + + + 4242 4242 4242 4242 + + + + + )} From d71e1bf707c8bd10d16266dc49d04979fb2cdfec Mon Sep 17 00:00:00 2001 From: Ken Date: Mon, 4 Mar 2024 12:58:22 +0800 Subject: [PATCH 08/15] chore: bump version to v6.111.0 --- CHANGELOG.md | 15 +++++++++++++++ frontend/package-lock.json | 4 ++-- frontend/package.json | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bed7ebc951..5c7c0fc83b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,22 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v6.111.0](https://github.com/opengovsg/FormSG/compare/v6.110.0...v6.111.0) + +- feat(fe): update copy, copy btn [`#7116`](https://github.com/opengovsg/FormSG/pull/7116) +- feat(virus-scanner): allow endpoint to be specified [`#7114`](https://github.com/opengovsg/FormSG/pull/7114) +- chore: remove Anguilla from country listing [`#7108`](https://github.com/opengovsg/FormSG/pull/7108) +- chore(vscode): update code actions value options [`#7113`](https://github.com/opengovsg/FormSG/pull/7113) +- feat(mrf): disable unsupported features [`#7106`](https://github.com/opengovsg/FormSG/pull/7106) +- fix: correct alignment issues in preview mode [`#7109`](https://github.com/opengovsg/FormSG/pull/7109) +- chore(OSS): add FerretDB migration instructions [`#7107`](https://github.com/opengovsg/FormSG/pull/7107) +- build: merge v6.110.0 back into develop [`#7105`](https://github.com/opengovsg/FormSG/pull/7105) +- build: release v6.110.0 [`#7104`](https://github.com/opengovsg/FormSG/pull/7104) + #### [v6.110.0](https://github.com/opengovsg/FormSG/compare/v6.109.0...v6.110.0) +> 21 February 2024 + - feat(mrf): retain workflow through a submission lifetime [`#7087`](https://github.com/opengovsg/FormSG/pull/7087) - feat(config): allow R2 bucket URLs [`#7097`](https://github.com/opengovsg/FormSG/pull/7097) - chore(deps): update ip package [`#7098`](https://github.com/opengovsg/FormSG/pull/7098) @@ -22,6 +36,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - build: merge release v6.109.0 into develop [`#7085`](https://github.com/opengovsg/FormSG/pull/7085) - chore(dev): update README with clearer virus-scanner install instructions [`#7083`](https://github.com/opengovsg/FormSG/pull/7083) - build: release v6.109.0 [`#7081`](https://github.com/opengovsg/FormSG/pull/7081) +- chore: bump version to v6.110.0 [`102cef3`](https://github.com/opengovsg/FormSG/commit/102cef33d7ae37e95c1b9f93f51712156cef1384) #### [v6.109.0](https://github.com/opengovsg/FormSG/compare/v6.108.0...v6.109.0) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 68219c6d35..fbb2e7dc88 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "form-frontend", - "version": "6.110.0", + "version": "6.111.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "form-frontend", - "version": "6.110.0", + "version": "6.111.0", "hasInstallScript": true, "dependencies": { "@chakra-ui/react": "^1.8.6", diff --git a/frontend/package.json b/frontend/package.json index 45501cd674..b7ba9b626d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "form-frontend", - "version": "6.110.0", + "version": "6.111.0", "homepage": ".", "private": true, "dependencies": { diff --git a/package-lock.json b/package-lock.json index 54fbe0b08e..1ddabfdde1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "FormSG", - "version": "6.110.0", + "version": "6.111.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "FormSG", - "version": "6.110.0", + "version": "6.111.0", "hasInstallScript": true, "dependencies": { "@aws-sdk/client-cloudwatch-logs": "^3.347.1", diff --git a/package.json b/package.json index 6845945fd2..cbb3f9108d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "FormSG", "description": "Form Manager for Government", - "version": "6.110.0", + "version": "6.111.0", "homepage": "https://form.gov.sg", "authors": [ "FormSG " From 0a96b28e7fc728e499600bf37fc61aada9206736 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:01:09 +0000 Subject: [PATCH 09/15] chore(deps-dev): bump json5 from 1.0.1 to 1.0.2 (#7119) Bumps [json5](https://github.com/json5/json5) from 1.0.1 to 1.0.2. - [Release notes](https://github.com/json5/json5/releases) - [Changelog](https://github.com/json5/json5/blob/main/CHANGELOG.md) - [Commits](https://github.com/json5/json5/compare/v1.0.1...v1.0.2) --- updated-dependencies: - dependency-name: json5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1ddabfdde1..25eaed6676 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10490,9 +10490,10 @@ } }, "node_modules/babel-loader/node_modules/json5": { - "version": "2.2.1", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, - "license": "MIT", "bin": { "json5": "lib/cli.js" }, @@ -20694,9 +20695,10 @@ "license": "ISC" }, "node_modules/json5": { - "version": "1.0.1", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "dev": true, - "license": "MIT", "dependencies": { "minimist": "^1.2.0" }, @@ -27363,9 +27365,9 @@ } }, "node_modules/ts-loader/node_modules/json5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, "bin": { "json5": "lib/cli.js" @@ -37610,7 +37612,9 @@ } }, "json5": { - "version": "2.2.1", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true }, "loader-utils": { @@ -44729,7 +44733,9 @@ "version": "5.0.1" }, "json5": { - "version": "1.0.1", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "dev": true, "requires": { "minimist": "^1.2.0" @@ -49467,9 +49473,9 @@ "dev": true }, "json5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true }, "loader-utils": { From 241ed3333751dbfb8f4a60573caa436463abe323 Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Wed, 6 Mar 2024 14:36:18 +0800 Subject: [PATCH 10/15] feat(tracking): wogaa tracking (#7123) * feat: add wogaa controller * feat: add tracking to view, feedback, submit * feat: add wogaa config * feat: add pageUrl to wogaa submit * chore: clean up * chore: add logs --- docker-compose.yml | 5 + package-lock.json | 39 +++++ package.json | 1 + src/app/config/features/wogaa.ts | 39 +++++ src/app/modules/wogaa/wogaa.controller.ts | 160 ++++++++++++++++++ .../v3/forms/public-forms.feedback.routes.ts | 6 +- .../api/v3/forms/public-forms.form.routes.ts | 2 + .../forms/public-forms.submissions.routes.ts | 4 + 8 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 src/app/config/features/wogaa.ts create mode 100644 src/app/modules/wogaa/wogaa.controller.ts diff --git a/docker-compose.yml b/docker-compose.yml index 0dd47c6ca4..ffb6c0124f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,6 +129,11 @@ services: - GROWTHBOOK_CLIENT_KEY # env vars for virus scanner - VIRUS_SCANNER_LAMBDA_FUNCTION_NAME=function + - WOGAA_SECRET_KEY + - WOGAA_START_ENDPOINT + - WOGAA_SUBMIT_ENDPOINT + - WOGAA_FEEDBACK_ENDPOINT + mockpass: build: https://github.com/opengovsg/mockpass.git#v4.0.4 diff --git a/package-lock.json b/package-lock.json index 25eaed6676..8157f24117 100644 --- a/package-lock.json +++ b/package-lock.json @@ -106,6 +106,7 @@ "uid-generator": "^2.0.0", "ulid": "^2.3.0", "uuid": "^9.0.0", + "uuid-by-string": "^4.0.0", "validator": "^13.7.0", "web-streams-polyfill": "^3.2.1", "whatwg-fetch": "^3.6.2", @@ -20309,12 +20310,22 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-md5": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.7.3.tgz", + "integrity": "sha512-ZC41vPSTLKGwIRjqDh8DfXoCrdQIyBgspJVPXHBGu4nZlAEvG3nf+jO9avM9RmLiGakg7vz974ms99nEV0tmTQ==" + }, "node_modules/js-sdsl": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==", "dev": true }, + "node_modules/js-sha1": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/js-sha1/-/js-sha1-0.6.0.tgz", + "integrity": "sha512-01gwBFreYydzmU9BmZxpVk6svJJHrVxEN3IOiGl6VO93bVKYETJ0sIth6DASI6mIFdt7NmfX9UiByRzsYHGU9w==" + }, "node_modules/js-tokens": { "version": "4.0.0", "dev": true, @@ -28122,6 +28133,15 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/uuid-by-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/uuid-by-string/-/uuid-by-string-4.0.0.tgz", + "integrity": "sha512-88ZSfcSkN04juiLqSsuyteqlSrXNFdsEPzSv3urnElDXNsZUXQN0smeTnh99x2DE15SCUQNgqKBfro54CuzHNQ==", + "dependencies": { + "js-md5": "^0.7.3", + "js-sha1": "^0.6.0" + } + }, "node_modules/v8-compile-cache": { "version": "2.1.1", "dev": true, @@ -44450,12 +44470,22 @@ "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz", "integrity": "sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==" }, + "js-md5": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.7.3.tgz", + "integrity": "sha512-ZC41vPSTLKGwIRjqDh8DfXoCrdQIyBgspJVPXHBGu4nZlAEvG3nf+jO9avM9RmLiGakg7vz974ms99nEV0tmTQ==" + }, "js-sdsl": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==", "dev": true }, + "js-sha1": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/js-sha1/-/js-sha1-0.6.0.tgz", + "integrity": "sha512-01gwBFreYydzmU9BmZxpVk6svJJHrVxEN3IOiGl6VO93bVKYETJ0sIth6DASI6mIFdt7NmfX9UiByRzsYHGU9w==" + }, "js-tokens": { "version": "4.0.0", "dev": true @@ -49960,6 +49990,15 @@ "uuid": { "version": "9.0.0" }, + "uuid-by-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/uuid-by-string/-/uuid-by-string-4.0.0.tgz", + "integrity": "sha512-88ZSfcSkN04juiLqSsuyteqlSrXNFdsEPzSv3urnElDXNsZUXQN0smeTnh99x2DE15SCUQNgqKBfro54CuzHNQ==", + "requires": { + "js-md5": "^0.7.3", + "js-sha1": "^0.6.0" + } + }, "v8-compile-cache": { "version": "2.1.1", "dev": true diff --git a/package.json b/package.json index cbb3f9108d..a2386c31b0 100644 --- a/package.json +++ b/package.json @@ -152,6 +152,7 @@ "uid-generator": "^2.0.0", "ulid": "^2.3.0", "uuid": "^9.0.0", + "uuid-by-string": "^4.0.0", "validator": "^13.7.0", "web-streams-polyfill": "^3.2.1", "whatwg-fetch": "^3.6.2", diff --git a/src/app/config/features/wogaa.ts b/src/app/config/features/wogaa.ts new file mode 100644 index 0000000000..e5777ed153 --- /dev/null +++ b/src/app/config/features/wogaa.ts @@ -0,0 +1,39 @@ +import convict, { Schema } from 'convict' + +export interface IWogaa { + wogaaSecretKey: string + wogaaStartEndpoint: string + wogaaSubmitEndpoint: string + wogaaFeedbackEndpoint: string +} + +const wogaaSchema: Schema = { + wogaaSecretKey: { + doc: 'Wogaa shared secret key', + format: String, + default: '', + env: 'WOGAA_SECRET_KEY', + }, + wogaaStartEndpoint: { + doc: 'Wogaa endpoint when a form is loaded', + format: String, + default: '', + env: 'WOGAA_START_ENDPOINT', + }, + wogaaSubmitEndpoint: { + doc: 'Wogaa endpoint when a form is loaded', + format: String, + default: '', + env: 'WOGAA_SUBMIT_ENDPOINT', + }, + wogaaFeedbackEndpoint: { + doc: 'Wogaa endpoint when a form is loaded', + format: String, + default: '', + env: 'WOGAA_FEEDBACK_ENDPOINT', + }, +} + +export const wogaaConfig = convict(wogaaSchema) + .validate({ allowed: 'strict' }) + .getProperties() diff --git a/src/app/modules/wogaa/wogaa.controller.ts b/src/app/modules/wogaa/wogaa.controller.ts new file mode 100644 index 0000000000..1d755fcaf3 --- /dev/null +++ b/src/app/modules/wogaa/wogaa.controller.ts @@ -0,0 +1,160 @@ +import Axios from 'axios' +import * as crypto from 'crypto' +import uuidGen from 'uuid-by-string' + +import { wogaaConfig } from '../../config/features/wogaa' +import { createLoggerWithLabel } from '../../config/logger' +import { ControllerHandler } from '../core/core.types' + +const logger = createLoggerWithLabel(module) +const generateSignature = (payload: Record) => { + const signature = crypto + .createHmac('sha256', wogaaConfig.wogaaSecretKey) + .update(JSON.stringify(payload)) + .digest('hex') + return signature +} + +const isConfigValid = () => { + if (!wogaaConfig.wogaaSecretKey) { + return false + } + if (!wogaaConfig.wogaaStartEndpoint) { + return false + } + if (!wogaaConfig.wogaaSubmitEndpoint) { + return false + } + if (!wogaaConfig.wogaaFeedbackEndpoint) { + return false + } + + return true +} + +export const handleSubmit: ControllerHandler<{ formId: string }> = async ( + req, + _, + next, +) => { + const { formId } = req.params + + if (!req.sessionID || !formId || !isConfigValid()) { + return next() + } + + const logMeta = { + action: 'wogaaHandleSubmit', + formId, + } + + const payload = { + formSgId: formId, + transactionId: uuidGen(req.sessionID), + } + // fire and forget + void Axios.post(wogaaConfig.wogaaSubmitEndpoint, payload, { + headers: { + 'WOGAA-Signature': generateSignature(payload), + }, + }) + .then(() => { + logger.info({ + message: 'Successfully sent WOGAA submit endpoint', + meta: logMeta, + }) + }) + .catch((e) => { + logger.warn({ + message: 'Error sending to WOGAA submit endpoint', + meta: { ...logMeta, wogaaRespError: e }, + }) + }) + + return next() +} + +export const handleFormView: ControllerHandler<{ formId: string }> = async ( + req, + _, + next, +) => { + const { formId } = req.params + + if (!req.sessionID || !formId || !isConfigValid()) { + return next() + } + + const logMeta = { + action: 'wogaaHandleFormView', + formId, + } + const payload = { + formSgId: formId, + pageUrl: formId, + transactionId: uuidGen(req.sessionID), + } + void Axios.post(wogaaConfig.wogaaStartEndpoint, payload, { + headers: { + 'WOGAA-Signature': generateSignature(payload), + }, + }) + .then(() => { + logger.info({ + message: 'Successfully sent WOGAA load form endpoint', + meta: logMeta, + }) + }) + .catch((e) => { + logger.warn({ + message: 'Error sending to WOGAA load form endpoint', + meta: { ...logMeta, wogaaRespError: e }, + }) + }) + + return next() +} + +export const handleFormFeedback: ControllerHandler< + { formId: string }, + unknown, + { rating: number; comment: string } +> = async (req, _, next) => { + const { formId } = req.params + const { rating, comment } = req.body + + if (!req.sessionID || !formId || !isConfigValid()) { + return next() + } + + const logMeta = { + action: 'wogaaHandleFormFeedback', + formId, + } + const payload = { + formSgId: formId, + transactionId: uuidGen(req.sessionID), + rating, + comment, + } + + void Axios.post(wogaaConfig.wogaaFeedbackEndpoint, payload, { + headers: { + 'WOGAA-Signature': generateSignature(payload), + }, + }) + .then(() => { + logger.info({ + message: 'Successfully sent WOGAA form feedback endpoint', + meta: logMeta, + }) + }) + .catch((e) => { + logger.warn({ + message: 'Error sending to WOGAA form feedback endpoint', + meta: { ...logMeta, wogaaRespError: e }, + }) + }) + + return next() +} diff --git a/src/app/routes/api/v3/forms/public-forms.feedback.routes.ts b/src/app/routes/api/v3/forms/public-forms.feedback.routes.ts index 432cc64a76..0be6bbd705 100644 --- a/src/app/routes/api/v3/forms/public-forms.feedback.routes.ts +++ b/src/app/routes/api/v3/forms/public-forms.feedback.routes.ts @@ -1,6 +1,7 @@ import { Router } from 'express' import * as FeedbackController from '../../../../modules/feedback/feedback.controller' +import * as WogaaController from '../../../../modules/wogaa/wogaa.controller' export const PublicFormsFeedbackRouter = Router() @@ -22,4 +23,7 @@ export const PublicFormsFeedbackRouter = Router() */ PublicFormsFeedbackRouter.route( '/:formId([a-fA-F0-9]{24})/submissions/:submissionId([a-fA-F0-9]{24})/feedback', -).post(FeedbackController.handleSubmitFormFeedback) +).post( + WogaaController.handleFormFeedback, + FeedbackController.handleSubmitFormFeedback, +) diff --git a/src/app/routes/api/v3/forms/public-forms.form.routes.ts b/src/app/routes/api/v3/forms/public-forms.form.routes.ts index b14e3746e0..b72c967a0b 100644 --- a/src/app/routes/api/v3/forms/public-forms.form.routes.ts +++ b/src/app/routes/api/v3/forms/public-forms.form.routes.ts @@ -1,6 +1,7 @@ import { Router } from 'express' import * as PublicFormController from '../../../../modules/form/public-form/public-form.controller' +import * as WogaaController from '../../../../modules/wogaa/wogaa.controller' export const PublicFormsFormRouter = Router() @@ -19,6 +20,7 @@ export const PublicFormsFormRouter = Router() * @returns 500 when database error occurs */ PublicFormsFormRouter.route('/:formId([a-fA-F0-9]{24})').get( + WogaaController.handleFormView, PublicFormController.handleGetPublicForm, ) diff --git a/src/app/routes/api/v3/forms/public-forms.submissions.routes.ts b/src/app/routes/api/v3/forms/public-forms.submissions.routes.ts index 947e9ed643..e53836ac2d 100644 --- a/src/app/routes/api/v3/forms/public-forms.submissions.routes.ts +++ b/src/app/routes/api/v3/forms/public-forms.submissions.routes.ts @@ -5,6 +5,7 @@ import * as EmailSubmissionController from '../../../../modules/submission/email import * as EncryptSubmissionController from '../../../../modules/submission/encrypt-submission/encrypt-submission.controller' import * as MultirespondentSubmissionController from '../../../../modules/submission/multirespondent-submission/multirespondent-submission.controller' import * as SubmissionController from '../../../../modules/submission/submission.controller' +import * as WogaaController from '../../../../modules/wogaa/wogaa.controller' import { limitRate } from '../../../../utils/limit-rate' export const PublicFormsSubmissionsRouter = Router() @@ -29,6 +30,7 @@ PublicFormsSubmissionsRouter.route( '/:formId([a-fA-F0-9]{24})/submissions/email', ).post( limitRate({ max: rateLimitConfig.submissions }), + WogaaController.handleSubmit, EmailSubmissionController.handleEmailSubmission, ) @@ -45,6 +47,7 @@ PublicFormsSubmissionsRouter.route( '/:formId([a-fA-F0-9]{24})/submissions/storage', ).post( limitRate({ max: rateLimitConfig.submissions }), + WogaaController.handleSubmit, EncryptSubmissionController.handleStorageSubmission, ) @@ -61,6 +64,7 @@ PublicFormsSubmissionsRouter.route( '/:formId([a-fA-F0-9]{24})/submissions/multirespondent', ).post( limitRate({ max: rateLimitConfig.submissions }), + WogaaController.handleSubmit, MultirespondentSubmissionController.handleMultirespondentSubmission, ) From dd3f5862b3a018f5b2564e77cec26721b81ed8ad Mon Sep 17 00:00:00 2001 From: Ken Lee Shu Ming Date: Wed, 6 Mar 2024 14:55:29 +0800 Subject: [PATCH 11/15] fix: add wogaa config into .env (#7125) --- .ebextensions/env-file-creation.config | 1 + 1 file changed, 1 insertion(+) diff --git a/.ebextensions/env-file-creation.config b/.ebextensions/env-file-creation.config index 2d6b460e14..e31140a634 100644 --- a/.ebextensions/env-file-creation.config +++ b/.ebextensions/env-file-creation.config @@ -46,6 +46,7 @@ files: aws ssm get-parameter --name "${ENV_TYPE}-ndi" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env aws ssm get-parameter --name "${ENV_TYPE}-verified-fields" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env aws ssm get-parameter --name "${ENV_TYPE}-webhook-verified-content" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env + aws ssm get-parameter --name "${ENV_TYPE}-wogaa" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env aws ssm get-parameter --name "${ENV_SITE_NAME}-sgid" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env aws ssm get-parameter --name "${ENV_SITE_NAME}-payment" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env aws ssm get-parameter --name "${ENV_SITE_NAME}-cron-payment" --with-decryption --region $AWS_REGION | jq -r '.Parameter.Value' >> $TARGET_DIR/.env From a6438f99d72df0f760fb026a81feb45ef09dabe1 Mon Sep 17 00:00:00 2001 From: Ken Date: Wed, 6 Mar 2024 14:56:04 +0800 Subject: [PATCH 12/15] chore: bump version to v6.112.0 --- CHANGELOG.md | 17 +++++++++++++++++ frontend/package-lock.json | 4 ++-- frontend/package.json | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c7c0fc83b..cc0c5958d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,24 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v6.112.0](https://github.com/opengovsg/FormSG/compare/v6.112.0...v6.112.0) + +- fix: add wogaa config into .env [`#7125`](https://github.com/opengovsg/FormSG/pull/7125) + +#### [v6.112.0](https://github.com/opengovsg/FormSG/compare/v6.111.0...v6.112.0) + +> 6 March 2024 + +- feat(tracking): wogaa tracking [`#7123`](https://github.com/opengovsg/FormSG/pull/7123) +- chore(deps-dev): bump json5 from 1.0.1 to 1.0.2 [`#7119`](https://github.com/opengovsg/FormSG/pull/7119) +- build: merge v6.111.0 back into develop [`#7118`](https://github.com/opengovsg/FormSG/pull/7118) +- build: release v6.111.0 [`#7117`](https://github.com/opengovsg/FormSG/pull/7117) +- chore: bump version to v6.112.0 [`54946e9`](https://github.com/opengovsg/FormSG/commit/54946e99d101d48512fce17e0c4d6f9c09db315a) + #### [v6.111.0](https://github.com/opengovsg/FormSG/compare/v6.110.0...v6.111.0) +> 4 March 2024 + - feat(fe): update copy, copy btn [`#7116`](https://github.com/opengovsg/FormSG/pull/7116) - feat(virus-scanner): allow endpoint to be specified [`#7114`](https://github.com/opengovsg/FormSG/pull/7114) - chore: remove Anguilla from country listing [`#7108`](https://github.com/opengovsg/FormSG/pull/7108) @@ -15,6 +31,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - chore(OSS): add FerretDB migration instructions [`#7107`](https://github.com/opengovsg/FormSG/pull/7107) - build: merge v6.110.0 back into develop [`#7105`](https://github.com/opengovsg/FormSG/pull/7105) - build: release v6.110.0 [`#7104`](https://github.com/opengovsg/FormSG/pull/7104) +- chore: bump version to v6.111.0 [`d71e1bf`](https://github.com/opengovsg/FormSG/commit/d71e1bf707c8bd10d16266dc49d04979fb2cdfec) #### [v6.110.0](https://github.com/opengovsg/FormSG/compare/v6.109.0...v6.110.0) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fbb2e7dc88..e45e01aa06 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "form-frontend", - "version": "6.111.0", + "version": "6.112.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "form-frontend", - "version": "6.111.0", + "version": "6.112.0", "hasInstallScript": true, "dependencies": { "@chakra-ui/react": "^1.8.6", diff --git a/frontend/package.json b/frontend/package.json index b7ba9b626d..a580dd7881 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "form-frontend", - "version": "6.111.0", + "version": "6.112.0", "homepage": ".", "private": true, "dependencies": { diff --git a/package-lock.json b/package-lock.json index 8157f24117..3378e184a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "FormSG", - "version": "6.111.0", + "version": "6.112.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "FormSG", - "version": "6.111.0", + "version": "6.112.0", "hasInstallScript": true, "dependencies": { "@aws-sdk/client-cloudwatch-logs": "^3.347.1", diff --git a/package.json b/package.json index a2386c31b0..bc0e74f399 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "FormSG", "description": "Form Manager for Government", - "version": "6.111.0", + "version": "6.112.0", "homepage": "https://form.gov.sg", "authors": [ "FormSG " From aa97ae749ada2fab00bea0967b886ea739ef3e06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:34:10 +0000 Subject: [PATCH 13/15] fix(deps): bump jose from 4.14.4 to 4.15.5 (#7130) Bumps [jose](https://github.com/panva/jose) from 4.14.4 to 4.15.5. - [Release notes](https://github.com/panva/jose/releases) - [Changelog](https://github.com/panva/jose/blob/v4.15.5/CHANGELOG.md) - [Commits](https://github.com/panva/jose/compare/v4.14.4...v4.15.5) --- updated-dependencies: - dependency-name: jose dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3378e184a6..cb6191f105 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,7 @@ "http-status-codes": "^2.2.0", "intl-tel-input": "~12.4.0", "ip": "^1.1.9", - "jose": "^4.13.1", + "jose": "^4.15.5", "jsdom": "^21.1.1", "json-stringify-safe": "^5.0.1", "JSONStream": "^1.3.5", @@ -20303,9 +20303,9 @@ } }, "node_modules/jose": { - "version": "4.14.4", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz", - "integrity": "sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==", + "version": "4.15.5", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.5.tgz", + "integrity": "sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==", "funding": { "url": "https://github.com/sponsors/panva" } @@ -44466,9 +44466,9 @@ } }, "jose": { - "version": "4.14.4", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz", - "integrity": "sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==" + "version": "4.15.5", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.5.tgz", + "integrity": "sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==" }, "js-md5": { "version": "0.7.3", diff --git a/package.json b/package.json index bc0e74f399..fa9e9d5803 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "http-status-codes": "^2.2.0", "intl-tel-input": "~12.4.0", "ip": "^1.1.9", - "jose": "^4.13.1", + "jose": "^4.15.5", "jsdom": "^21.1.1", "json-stringify-safe": "^5.0.1", "JSONStream": "^1.3.5", From da0d5b75c32d38d908b15b9558f19c9bdbb9b8fd Mon Sep 17 00:00:00 2001 From: Kathleen Koh <89055608+kathleenkhy@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:12:16 +0800 Subject: [PATCH 14/15] chore: add logs for virus scanning duration (#7129) --- src/app/modules/submission/submission.service.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/app/modules/submission/submission.service.ts b/src/app/modules/submission/submission.service.ts index 15a7b898fe..3ee78d48f5 100644 --- a/src/app/modules/submission/submission.service.ts +++ b/src/app/modules/submission/submission.service.ts @@ -385,11 +385,20 @@ export const downloadCleanFile = (cleanFileKey: string, versionId: string) => { }) .createReadStream() + logger.info({ + message: 'File download from S3 has started', + meta: logMeta, + }) + readStream.pipe(writeStream) return ResultAsync.fromPromise( new Promise((resolve, reject) => { readStream.on('end', () => { + logger.info({ + message: 'Successfully downloaded file from S3', + meta: logMeta, + }) resolve(buffer) }) From d68b84a85456135ba0df6bf8ec0d0c415a226c12 Mon Sep 17 00:00:00 2001 From: LoneRifle Date: Fri, 8 Mar 2024 22:52:45 +0800 Subject: [PATCH 15/15] fix(virus-scanner): rework error logging in pino (#7128) Pino has in-built serialisers for logging Errors, but only on Errors or Errors in nested `err` properties. Change logging accordingly. --- .../virus-scanner/src/__tests/index.spec.ts | 6 +++--- .../src/__tests/s3.service.spec.ts | 4 ++-- serverless/virus-scanner/src/index.ts | 16 ++++++++-------- serverless/virus-scanner/src/s3.service.ts | 18 +++++++++--------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/serverless/virus-scanner/src/__tests/index.spec.ts b/serverless/virus-scanner/src/__tests/index.spec.ts index 6b1911a2bb..c1826e48e5 100644 --- a/serverless/virus-scanner/src/__tests/index.spec.ts +++ b/serverless/virus-scanner/src/__tests/index.spec.ts @@ -97,7 +97,7 @@ describe('handler', () => { expect(mockLoggerWarn).toHaveBeenCalledWith( expect.objectContaining({ message: 'File not found', - error: new Error('File not found'), + err: new Error('File not found'), quarantineFileKey: mockUUID, }), ) @@ -175,7 +175,7 @@ describe('handler', () => { expect(mockLoggerError).toHaveBeenCalledWith( expect.objectContaining({ message: 'Failed to scan file', - error: new Error('Failed to scan file'), + err: new Error('Failed to scan file'), quarantineFileKey: mockUUID, }), ) @@ -248,7 +248,7 @@ describe('handler', () => { expect(mockLoggerError).toHaveBeenCalledWith( expect.objectContaining({ message: 'Failed to move file to clean bucket', - error: new Error('Failed to move file'), + err: new Error('Failed to move file'), bucket: 'local-virus-scanner-quarantine-bucket', key: mockUUID, }), diff --git a/serverless/virus-scanner/src/__tests/s3.service.spec.ts b/serverless/virus-scanner/src/__tests/s3.service.spec.ts index c8bbee5c7f..958fdfbaf0 100644 --- a/serverless/virus-scanner/src/__tests/s3.service.spec.ts +++ b/serverless/virus-scanner/src/__tests/s3.service.spec.ts @@ -145,7 +145,7 @@ describe('S3Service', () => { expect.objectContaining({ bucketName: 'bucketName', objectKey: 'objectKey', - error: new Error('Body is empty'), + err: new Error('Body is empty'), }), 'Failed to get object from s3', ) @@ -172,7 +172,7 @@ describe('S3Service', () => { expect.objectContaining({ bucketName: 'bucketName', objectKey: 'objectKey', - error: new Error('VersionId is empty'), + err: new Error('VersionId is empty'), }), 'Failed to get object from s3', ) diff --git a/serverless/virus-scanner/src/index.ts b/serverless/virus-scanner/src/index.ts index a31fc37280..6f55c325e8 100644 --- a/serverless/virus-scanner/src/index.ts +++ b/serverless/virus-scanner/src/index.ts @@ -66,10 +66,10 @@ export const handler = async ( bucketName: quarantineBucket, objectKey: quarantineFileKey, }) - } catch (error) { + } catch (err) { logger.warn({ message: 'File not found', - error, + err, quarantineFileKey, }) return { @@ -88,10 +88,10 @@ export const handler = async ( let scanResult try { scanResult = await scanFileStream(body) - } catch (error) { + } catch (err) { logger.error({ message: 'Failed to scan file', - error, + err, quarantineFileKey, }) return { @@ -122,11 +122,11 @@ export const handler = async ( objectKey: quarantineFileKey, versionId, }) - } catch (error) { + } catch (err) { // Log but do not halt execution as we still want to return 400 for malicious file logger.error({ message: 'Failed to delete file from quarantine bucket', - error, + err, key: quarantineFileKey, }) } @@ -158,10 +158,10 @@ export const handler = async ( destinationBucketName: cleanBucket, destinationObjectKey: cleanFileKey, }) - } catch (error) { + } catch (err) { logger.error({ message: 'Failed to move file to clean bucket', - error, + err, bucket: quarantineBucket, key: quarantineFileKey, }) diff --git a/serverless/virus-scanner/src/s3.service.ts b/serverless/virus-scanner/src/s3.service.ts index 1dd6c78f0f..9ec6fb5ed8 100644 --- a/serverless/virus-scanner/src/s3.service.ts +++ b/serverless/virus-scanner/src/s3.service.ts @@ -72,17 +72,17 @@ export class S3Service { ) return { body, versionId } as GetS3FileStreamResult - } catch (error) { + } catch (err) { this.logger.error( { bucketName, objectKey, - error, + err, }, 'Failed to get object from s3', ) - throw error + throw err } } @@ -112,17 +112,17 @@ export class S3Service { }, 'Deleted document from s3', ) - } catch (error) { + } catch (err) { this.logger.error( { bucketName, objectKey, - error, + err, }, 'Failed to delete object from s3', ) - throw error + throw err } } @@ -189,7 +189,7 @@ export class S3Service { ) return VersionId - } catch (error) { + } catch (err) { this.logger.error( { sourceBucketName, @@ -197,12 +197,12 @@ export class S3Service { sourceObjectVersionId, destinationBucketName, destinationObjectKey, - error, + err, }, 'Failed to move object in s3', ) - throw error + throw err } } }