From abe7b6748ddfd5171db4b6527f01bde33ebff96e Mon Sep 17 00:00:00 2001 From: seaerchin Date: Tue, 26 Sep 2023 18:56:20 +0800 Subject: [PATCH 1/4] feat(formsg): clone repo on webhook trigger from forms --- src/config/config.ts | 7 +++ src/routes/formsgSiteCreation.ts | 86 +++++++++++++++++++++++++++++++- src/server.js | 6 ++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/config/config.ts b/src/config/config.ts index ec89cf26b..b36910f4a 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -283,6 +283,13 @@ const config = convict({ format: "required-string", default: "", }, + siteCloneFormKey: { + doc: "FormSG API key for site clone form", + env: "SITE_CLONE_FORM_KEY", + sensitive: true, + format: "required-string", + default: "", + }, }, postman: { apiKey: { diff --git a/src/routes/formsgSiteCreation.ts b/src/routes/formsgSiteCreation.ts index cd859ba11..8d80c9e6c 100644 --- a/src/routes/formsgSiteCreation.ts +++ b/src/routes/formsgSiteCreation.ts @@ -11,10 +11,12 @@ import { BadRequestError } from "@errors/BadRequestError" import { getField } from "@utils/formsg-utils" import { attachFormSGHandler } from "@root/middleware" +import GitFileSystemService from "@services/db/GitFileSystemService" import UsersService from "@services/identity/UsersService" import InfraService from "@services/infra/InfraService" import { mailer } from "@services/utilServices/MailClient" +const SITE_CLONE_FORM_KEY = config.get("formSg.siteCloneFormKey") const SITE_CREATE_FORM_KEY = config.get("formSg.siteCreateFormKey") const REQUESTER_EMAIL_FIELD = "Government E-mail" const SITE_NAME_FIELD = "Site Name" @@ -25,6 +27,7 @@ const LOGIN_TYPE_FIELD = "Login Type" export interface FormsgRouterProps { usersService: UsersService infraService: InfraService + gitFileSystemService: GitFileSystemService } export class FormsgRouter { @@ -32,9 +35,16 @@ export class FormsgRouter { private readonly infraService: FormsgRouterProps["infraService"] - constructor({ usersService, infraService }: FormsgRouterProps) { + private readonly gitFileSystemService: FormsgRouterProps["gitFileSystemService"] + + constructor({ + usersService, + infraService, + gitFileSystemService, + }: FormsgRouterProps) { this.usersService = usersService this.infraService = infraService + this.gitFileSystemService = gitFileSystemService // We need to bind all methods because we don't invoke them from the class directly autoBind(this) } @@ -157,9 +167,83 @@ export class FormsgRouter { await mailer.sendMail(email, subject, html) } + cloneSiteToEfs: RequestHandler< + never, + Record, + { data: { submissionId: string } }, + never, + { submission: DecryptedContent } + > = async (req, res) => { + // 1. Extract arguments + const { submissionId } = req.body.data + const { responses } = res.locals.submission + // NOTE: This is validated by formsg to be of domain `@open.gov.sg`; + // hence, not revalidating here + const requesterEmail = getField(responses, "Email") as string + // NOTE: The field is required by our form so this cannot be empty or undefined + const githubRepoName = getField(responses, "Github Repo Name") as string + + logger.info( + `${requesterEmail} requested for ${githubRepoName} to be cloned onto EFS` + ) + + this.gitFileSystemService + .clone(githubRepoName) + .map((path) => { + logger.info(`Cloned ${githubRepoName} to ${path}`) + this.sendCloneSuccess( + requesterEmail, + githubRepoName, + submissionId, + path + ) + }) + .mapErr((err) => { + logger.error( + `Cloning repo: ${githubRepoName} to EFS failed with error: ${JSON.stringify( + err + )}` + ) + this.sendCloneError( + requesterEmail, + githubRepoName, + submissionId, + err.message + ) + }) + } + + sendCloneSuccess = async ( + requesterEmail: string, + githubRepoName: string, + submissionId: string, + path: string + ) => { + const subject = `[Isomer] Clone site ${githubRepoName} SUCCESS` + const html = `

Isomer site ${githubRepoName} was cloned successfully to EFS path: ${path}. (Form submission id [${submissionId}])

` + await mailer.sendMail(requesterEmail, subject, html) + } + + async sendCloneError( + requesterEmail: string, + githubRepoName: string, + submissionId: string, + message: string + ) { + const subject = `[Isomer] Clone site ${githubRepoName} FAILURE` + const html = `

Isomer site ${githubRepoName} was not cloned successfully. Cloning failed with error: ${message} (Form submission id [${submissionId}])

` + await mailer.sendMail(requesterEmail, subject, html) + } + getRouter() { const router = express.Router({ mergeParams: true }) + router.post( + "/clone-site", + attachFormSGHandler(SITE_CLONE_FORM_KEY), + this.cloneSiteToEfs + ) + router.post( "/create-site", attachFormSGHandler(SITE_CREATE_FORM_KEY), diff --git a/src/server.js b/src/server.js index 707d1b0b2..044093a8f 100644 --- a/src/server.js +++ b/src/server.js @@ -351,7 +351,11 @@ const authV2Router = new AuthRouter({ statsMiddleware, sgidAuthRouter, }) -const formsgRouter = new FormsgRouter({ usersService, infraService }) +const formsgRouter = new FormsgRouter({ + usersService, + infraService, + gitFileSystemService, +}) const formsgSiteLaunchRouter = new FormsgSiteLaunchRouter({ usersService, infraService, From b890280ea4bdc9044de8c9c8d30bf22e161ee77a Mon Sep 17 00:00:00 2001 From: seaerchin Date: Wed, 27 Sep 2023 02:12:02 +0800 Subject: [PATCH 2/4] ci(.env.test): add env var --- .env.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.test b/.env.test index 67018d209..05c0523ec 100644 --- a/.env.test +++ b/.env.test @@ -22,6 +22,7 @@ export SYSTEM_GITHUB_TOKEN="github_token" # FormSG keys export SITE_CREATE_FORM_KEY="site_form_key" export SITE_LAUNCH_FORM_KEY="site_launch_form_key" +export SITE_CLONE_FORM_KEY="site_clone_form_key" # Required to connect to DynamoDB export AWS_ACCESS_KEY_ID="abc123" @@ -81,4 +82,4 @@ export SGID_PRIVATE_KEY="private" export SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect" # GrowthBook -export GROWTHBOOK_CLIENT_KEY="some random key" \ No newline at end of file +export GROWTHBOOK_CLIENT_KEY="some random key" From 4f0c00c92b56b471ee45e11b5ef1758df1448bae Mon Sep 17 00:00:00 2001 From: seaerchin Date: Tue, 3 Oct 2023 14:17:20 +0800 Subject: [PATCH 3/4] fix(env-var): add to ssm script --- .../predeploy/06_fetch_ssm_parameters.sh | 143 +++++++++--------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/.platform/hooks/predeploy/06_fetch_ssm_parameters.sh b/.platform/hooks/predeploy/06_fetch_ssm_parameters.sh index 31cebd0b0..ec059234f 100644 --- a/.platform/hooks/predeploy/06_fetch_ssm_parameters.sh +++ b/.platform/hooks/predeploy/06_fetch_ssm_parameters.sh @@ -20,100 +20,101 @@ fi ENV_TYPE=$(/opt/elasticbeanstalk/bin/get-config environment -k SSM_PREFIX) TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') -echo "Timestamp: $TIMESTAMP - ENV TYPE: $ENV_TYPE" > /tmp/ssm-type.txt +echo "Timestamp: $TIMESTAMP - ENV TYPE: $ENV_TYPE" >/tmp/ssm-type.txt # List of all env vars to fetch ENV_VARS=( - "AUTH_TOKEN_EXPIRY_DURATION_IN_MILLISECONDS" - "AWS_BACKEND_EB_ENV_NAME" - "AWS_REGION" - "CLIENT_ID" - "CLIENT_SECRET" - "CLOUDMERSIVE_API_KEY" - "COOKIE_DOMAIN" - "DB_ACQUIRE" - "DB_MAX_POOL" - "DB_MIN_POOL" - "DB_TIMEOUT" - "DB_URI" - "DD_AGENT_MAJOR_VERSION" - "DD_ENV" - "DD_LOGS_INJECTION" - "DD_SERVICE" - "DD_TAGS" - "DD_TRACE_STARTUP_LOGS" - "E2E_TEST_GH_TOKEN" - "E2E_TEST_REPO" - "E2E_TEST_SECRET" - "EFS_VOL_PATH" - "ENCRYPTION_SECRET" - "FF_DEPRECATE_SITE_QUEUES" - "FRONTEND_URL" - "GGS_EXPERIMENTAL_TRACKING_SITES" - "GITHUB_BUILD_ORG_NAME" - "GITHUB_BUILD_REPO_NAME" - "GITHUB_ORG_NAME" - "GROWTHBOOK_CLIENT_KEY" - "INCOMING_QUEUE_URL" - "ISOMERPAGES_REPO_PAGE_COUNT" - "JWT_SECRET" - "MAX_NUM_OTP_ATTEMPTS" - "MOCK_AMPLIFY_DOMAIN_ASSOCIATION_CALLS" - "MUTEX_TABLE_NAME" - "NETLIFY_ACCESS_TOKEN" - "NODE_ENV" - "OTP_EXPIRY" - "OTP_SECRET" - "OUTGOING_QUEUE_URL" - "POSTMAN_API_KEY" - "POSTMAN_SMS_CRED_NAME" - "REDIRECT_URI" - "SESSION_SECRET" - "SGID_CLIENT_ID" - "SGID_CLIENT_SECRET" - "SGID_PRIVATE_KEY" - "SGID_REDIRECT_URI" - "SITE_CREATE_FORM_KEY" - "SITE_LAUNCH_DYNAMO_DB_TABLE_NAME" - "SITE_LAUNCH_FORM_KEY" - "SITE_PASSWORD_SECRET_KEY" - "SSM_PREFIX" - "STEP_FUNCTIONS_ARN" - "SYSTEM_GITHUB_TOKEN" + "AUTH_TOKEN_EXPIRY_DURATION_IN_MILLISECONDS" + "AWS_BACKEND_EB_ENV_NAME" + "AWS_REGION" + "CLIENT_ID" + "CLIENT_SECRET" + "CLOUDMERSIVE_API_KEY" + "COOKIE_DOMAIN" + "DB_ACQUIRE" + "DB_MAX_POOL" + "DB_MIN_POOL" + "DB_TIMEOUT" + "DB_URI" + "DD_AGENT_MAJOR_VERSION" + "DD_ENV" + "DD_LOGS_INJECTION" + "DD_SERVICE" + "DD_TAGS" + "DD_TRACE_STARTUP_LOGS" + "E2E_TEST_GH_TOKEN" + "E2E_TEST_REPO" + "E2E_TEST_SECRET" + "EFS_VOL_PATH" + "ENCRYPTION_SECRET" + "FF_DEPRECATE_SITE_QUEUES" + "FRONTEND_URL" + "GGS_EXPERIMENTAL_TRACKING_SITES" + "GITHUB_BUILD_ORG_NAME" + "GITHUB_BUILD_REPO_NAME" + "GITHUB_ORG_NAME" + "GROWTHBOOK_CLIENT_KEY" + "INCOMING_QUEUE_URL" + "ISOMERPAGES_REPO_PAGE_COUNT" + "JWT_SECRET" + "MAX_NUM_OTP_ATTEMPTS" + "MOCK_AMPLIFY_DOMAIN_ASSOCIATION_CALLS" + "MUTEX_TABLE_NAME" + "NETLIFY_ACCESS_TOKEN" + "NODE_ENV" + "OTP_EXPIRY" + "OTP_SECRET" + "OUTGOING_QUEUE_URL" + "POSTMAN_API_KEY" + "POSTMAN_SMS_CRED_NAME" + "REDIRECT_URI" + "SESSION_SECRET" + "SGID_CLIENT_ID" + "SGID_CLIENT_SECRET" + "SGID_PRIVATE_KEY" + "SGID_REDIRECT_URI" + "SITE_CREATE_FORM_KEY" + "SITE_LAUNCH_DYNAMO_DB_TABLE_NAME" + "SITE_LAUNCH_FORM_KEY" + "SITE_CLONE_FORM_KEY" + "SITE_PASSWORD_SECRET_KEY" + "SSM_PREFIX" + "STEP_FUNCTIONS_ARN" + "SYSTEM_GITHUB_TOKEN" ) echo "Set AWS region" aws configure set default.region ap-southeast-1 -set +e # Do not exit if a command fails +set +e # Do not exit if a command fails for ENV_VAR in "${ENV_VARS[@]}"; do echo "Fetching ${ENV_VAR} from SSM" - + VALUE=$(aws ssm get-parameter --name "${ENV_TYPE}_${ENV_VAR}" --with-decryption --query "Parameter.Value" --output text 2>/dev/null) - STATUS=$? # Capture exit status of the aws ssm command - + STATUS=$? # Capture exit status of the aws ssm command + if [ $STATUS -ne 0 ]; then echo "Failed to fetch ${ENV_VAR}. Skipping." continue fi - - echo "${ENV_VAR}=${VALUE}" >> /tmp/isomer/.isomer.env + + echo "${ENV_VAR}=${VALUE}" >>/tmp/isomer/.isomer.env echo "Saved ${ENV_VAR}" done -set -e # Exit on command failure from this point onwards +set -e # Exit on command failure from this point onwards # Use flock to ensure that the EFS file is locked during the copy operation ( - flock -n 200 || exit 1 + flock -n 200 || exit 1 - # Copy the local file to EFS - echo "Copying local env file to EFS" - cp /tmp/isomer/.isomer.env /efs/isomer/.isomer.env + # Copy the local file to EFS + echo "Copying local env file to EFS" + cp /tmp/isomer/.isomer.env /efs/isomer/.isomer.env - # Ensure the file on EFS is owned by webapp so it has access - chown webapp:webapp /efs/isomer/.isomer.env + # Ensure the file on EFS is owned by webapp so it has access + chown webapp:webapp /efs/isomer/.isomer.env ) 200>/efs/isomer/.isomer.lock @@ -126,4 +127,4 @@ else echo "Couldn't acquire the lock. Another instance might be writing to the file." fi -echo "Operation completed." \ No newline at end of file +echo "Operation completed." From 62500ea32385be7ffc30bf2c897d620f13338fef Mon Sep 17 00:00:00 2001 From: seaerchin Date: Tue, 3 Oct 2023 14:17:47 +0800 Subject: [PATCH 4/4] refactor(formsg): shift cloning to separate router --- src/routes/formsgSiteClone.ts | 125 ++++++++++++++++++++++++++++++++++ src/server.js | 5 ++ 2 files changed, 130 insertions(+) create mode 100644 src/routes/formsgSiteClone.ts diff --git a/src/routes/formsgSiteClone.ts b/src/routes/formsgSiteClone.ts new file mode 100644 index 000000000..67ec0225c --- /dev/null +++ b/src/routes/formsgSiteClone.ts @@ -0,0 +1,125 @@ +import { DecryptedContent } from "@opengovsg/formsg-sdk/dist/types" +import autoBind from "auto-bind" +import express, { RequestHandler } from "express" + +import { config } from "@config/config" + +import logger from "@logger/logger" + +import { getField } from "@utils/formsg-utils" + +import { ISOMER_ADMIN_EMAIL } from "@root/constants" +import { attachFormSGHandler } from "@root/middleware" +import GitFileSystemService from "@services/db/GitFileSystemService" +import UsersService from "@services/identity/UsersService" +import InfraService from "@services/infra/InfraService" +import { mailer } from "@services/utilServices/MailClient" + +const SITE_CLONE_FORM_KEY = config.get("formSg.siteCloneFormKey") + +export interface FormsgSiteCloneRouterProps { + usersService: UsersService + infraService: InfraService + gitFileSystemService: GitFileSystemService +} + +export class FormsgSiteCloneRouter { + private readonly gitFileSystemService: FormsgSiteCloneRouterProps["gitFileSystemService"] + + constructor({ gitFileSystemService }: FormsgSiteCloneRouterProps) { + this.gitFileSystemService = gitFileSystemService + // We need to bind all methods because we don't invoke them from the class directly + autoBind(this) + } + + cloneSiteToEfs: RequestHandler< + never, + Record, + { data: { submissionId: string } }, + never, + { submission: DecryptedContent } + > = async (req, res) => { + // 1. Extract arguments + const { submissionId } = req.body.data + const { responses } = res.locals.submission + const requesterEmail = getField(responses, "Email") + // NOTE: The field is required by our form so this cannot be empty or undefined + const githubRepoName = getField(responses, "Github Repo Name") as string + + if ( + !requesterEmail || + !githubRepoName || + !requesterEmail.endsWith("@open.gov.sg") + ) { + return this.sendCloneError( + ISOMER_ADMIN_EMAIL, + githubRepoName, + submissionId, + "Invalid email or missing github repo name detected for submission" + ) + } + + logger.info( + `${requesterEmail} requested for ${githubRepoName} to be cloned onto EFS` + ) + + this.gitFileSystemService + .clone(githubRepoName) + .map((path) => { + logger.info(`Cloned ${githubRepoName} to ${path}`) + this.sendCloneSuccess( + requesterEmail, + githubRepoName, + submissionId, + path + ) + }) + .mapErr((err) => { + logger.error( + `Cloning repo: ${githubRepoName} to EFS failed with error: ${JSON.stringify( + err + )}` + ) + this.sendCloneError( + requesterEmail, + githubRepoName, + submissionId, + err.message + ) + }) + } + + sendCloneSuccess = async ( + requesterEmail: string, + githubRepoName: string, + submissionId: string, + path: string + ) => { + const subject = `[Isomer] Clone site ${githubRepoName} SUCCESS` + const html = `

Isomer site ${githubRepoName} was cloned successfully to EFS path: ${path}. (Form submission id [${submissionId}])

` + await mailer.sendMail(requesterEmail, subject, html) + } + + async sendCloneError( + requesterEmail: string, + githubRepoName: string, + submissionId: string, + message: string + ) { + const subject = `[Isomer] Clone site ${githubRepoName} FAILURE` + const html = `

Isomer site ${githubRepoName} was not cloned successfully. Cloning failed with error: ${message} (Form submission id [${submissionId}])

` + await mailer.sendMail(requesterEmail, subject, html) + } + + getRouter() { + const router = express.Router({ mergeParams: true }) + + router.post( + "/clone-site", + attachFormSGHandler(SITE_CLONE_FORM_KEY), + this.cloneSiteToEfs + ) + + return router + } +} diff --git a/src/server.js b/src/server.js index 044093a8f..1736cf847 100644 --- a/src/server.js +++ b/src/server.js @@ -147,6 +147,7 @@ const FRONTEND_URL = config.get("app.frontendUrl") // Import routes const { errorHandler } = require("@middleware/errorHandler") +const { FormsgSiteCloneRouter } = require("@routes/formsgSiteClone") const { FormsgRouter } = require("@routes/formsgSiteCreation") const { FormsgSiteLaunchRouter } = require("@routes/formsgSiteLaunch") const { AuthRouter } = require("@routes/v2/auth") @@ -360,6 +361,9 @@ const formsgSiteLaunchRouter = new FormsgSiteLaunchRouter({ usersService, infraService, }) +const formsgSiteCloneRouter = new FormsgSiteCloneRouter({ + gitFileSystemService, +}) const app = express() @@ -407,6 +411,7 @@ app.use("/v2/sites/:siteName", authenticatedSitesSubrouterV2) // FormSG Backend handler routes app.use("/formsg", formsgRouter.getRouter()) app.use("/formsg", formsgSiteLaunchRouter.getRouter()) +app.use("/formsg", formsgSiteCloneRouter.getRouter()) // catch unknown routes app.use((req, res, next) => {