feat(formsg): clone repo on webhook trigger from forms

.env.test

@@ -22,6 +22,7 @@ export SYSTEM_GITHUB_TOKEN="github_token"
 # FormSG keys
 export SITE_CREATE_FORM_KEY="site_form_key"
 export SITE_LAUNCH_FORM_KEY="site_launch_form_key"
+export SITE_CLONE_FORM_KEY="site_clone_form_key"
 
 # Required to connect to DynamoDB
 export AWS_ACCESS_KEY_ID="abc123"
@@ -81,4 +82,4 @@ export SGID_PRIVATE_KEY="private"
 export SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect"
 
 # GrowthBook
-export GROWTHBOOK_CLIENT_KEY="some random key"
+export GROWTHBOOK_CLIENT_KEY="some random key"

.platform/hooks/predeploy/06_fetch_ssm_parameters.sh

@@ -20,100 +20,101 @@ fi
 ENV_TYPE=$(/opt/elasticbeanstalk/bin/get-config environment -k SSM_PREFIX)
 TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
 
-echo "Timestamp: $TIMESTAMP - ENV TYPE: $ENV_TYPE" > /tmp/ssm-type.txt
+echo "Timestamp: $TIMESTAMP - ENV TYPE: $ENV_TYPE" >/tmp/ssm-type.txt
 
 # List of all env vars to fetch
 ENV_VARS=(
-  "AUTH_TOKEN_EXPIRY_DURATION_IN_MILLISECONDS"
-  "AWS_BACKEND_EB_ENV_NAME"
-  "AWS_REGION"
-  "CLIENT_ID"
-  "CLIENT_SECRET"
-  "CLOUDMERSIVE_API_KEY"
-  "COOKIE_DOMAIN"
-  "DB_ACQUIRE"
-  "DB_MAX_POOL"
-  "DB_MIN_POOL"
-  "DB_TIMEOUT"
-  "DB_URI"
-  "DD_AGENT_MAJOR_VERSION"
-  "DD_ENV"
-  "DD_LOGS_INJECTION"
-  "DD_SERVICE"
-  "DD_TAGS"
-  "DD_TRACE_STARTUP_LOGS"
-  "E2E_TEST_GH_TOKEN"
-  "E2E_TEST_REPO"
-  "E2E_TEST_SECRET"
-  "EFS_VOL_PATH"
-  "ENCRYPTION_SECRET"
-  "FF_DEPRECATE_SITE_QUEUES"
-  "FRONTEND_URL"
-  "GGS_EXPERIMENTAL_TRACKING_SITES"
-  "GITHUB_BUILD_ORG_NAME"
-  "GITHUB_BUILD_REPO_NAME"
-  "GITHUB_ORG_NAME"
-  "GROWTHBOOK_CLIENT_KEY"
-  "INCOMING_QUEUE_URL"
-  "ISOMERPAGES_REPO_PAGE_COUNT"
-  "JWT_SECRET"
-  "MAX_NUM_OTP_ATTEMPTS"
-  "MOCK_AMPLIFY_DOMAIN_ASSOCIATION_CALLS"
-  "MUTEX_TABLE_NAME"
-  "NETLIFY_ACCESS_TOKEN"
-  "NODE_ENV"
-  "OTP_EXPIRY"
-  "OTP_SECRET"
-  "OUTGOING_QUEUE_URL"
-  "POSTMAN_API_KEY"
-  "POSTMAN_SMS_CRED_NAME"
-  "REDIRECT_URI"
-  "SESSION_SECRET"
-  "SGID_CLIENT_ID" 
-  "SGID_CLIENT_SECRET" 
-  "SGID_PRIVATE_KEY" 
-  "SGID_REDIRECT_URI"
-  "SITE_CREATE_FORM_KEY"
-  "SITE_LAUNCH_DYNAMO_DB_TABLE_NAME"
-  "SITE_LAUNCH_FORM_KEY"
-  "SITE_PASSWORD_SECRET_KEY"
-  "SSM_PREFIX"
-  "STEP_FUNCTIONS_ARN"
-  "SYSTEM_GITHUB_TOKEN"
+    "AUTH_TOKEN_EXPIRY_DURATION_IN_MILLISECONDS"
+    "AWS_BACKEND_EB_ENV_NAME"
+    "AWS_REGION"
+    "CLIENT_ID"
+    "CLIENT_SECRET"
+    "CLOUDMERSIVE_API_KEY"
+    "COOKIE_DOMAIN"
+    "DB_ACQUIRE"
+    "DB_MAX_POOL"
+    "DB_MIN_POOL"
+    "DB_TIMEOUT"
+    "DB_URI"
+    "DD_AGENT_MAJOR_VERSION"
+    "DD_ENV"
+    "DD_LOGS_INJECTION"
+    "DD_SERVICE"
+    "DD_TAGS"
+    "DD_TRACE_STARTUP_LOGS"
+    "E2E_TEST_GH_TOKEN"
+    "E2E_TEST_REPO"
+    "E2E_TEST_SECRET"
+    "EFS_VOL_PATH"
+    "ENCRYPTION_SECRET"
+    "FF_DEPRECATE_SITE_QUEUES"
+    "FRONTEND_URL"
+    "GGS_EXPERIMENTAL_TRACKING_SITES"
+    "GITHUB_BUILD_ORG_NAME"
+    "GITHUB_BUILD_REPO_NAME"
+    "GITHUB_ORG_NAME"
+    "GROWTHBOOK_CLIENT_KEY"
+    "INCOMING_QUEUE_URL"
+    "ISOMERPAGES_REPO_PAGE_COUNT"
+    "JWT_SECRET"
+    "MAX_NUM_OTP_ATTEMPTS"
+    "MOCK_AMPLIFY_DOMAIN_ASSOCIATION_CALLS"
+    "MUTEX_TABLE_NAME"
+    "NETLIFY_ACCESS_TOKEN"
+    "NODE_ENV"
+    "OTP_EXPIRY"
+    "OTP_SECRET"
+    "OUTGOING_QUEUE_URL"
+    "POSTMAN_API_KEY"
+    "POSTMAN_SMS_CRED_NAME"
+    "REDIRECT_URI"
+    "SESSION_SECRET"
+    "SGID_CLIENT_ID"
+    "SGID_CLIENT_SECRET"
+    "SGID_PRIVATE_KEY"
+    "SGID_REDIRECT_URI"
+    "SITE_CREATE_FORM_KEY"
+    "SITE_LAUNCH_DYNAMO_DB_TABLE_NAME"
+    "SITE_LAUNCH_FORM_KEY"
+    "SITE_CLONE_FORM_KEY"
+    "SITE_PASSWORD_SECRET_KEY"
+    "SSM_PREFIX"
+    "STEP_FUNCTIONS_ARN"
+    "SYSTEM_GITHUB_TOKEN"
 )
 
 echo "Set AWS region"
 aws configure set default.region ap-southeast-1
 
-set +e  # Do not exit if a command fails
+set +e # Do not exit if a command fails
 
 for ENV_VAR in "${ENV_VARS[@]}"; do
     echo "Fetching ${ENV_VAR} from SSM"
-    
+
     VALUE=$(aws ssm get-parameter --name "${ENV_TYPE}_${ENV_VAR}" --with-decryption --query "Parameter.Value" --output text 2>/dev/null)
-    STATUS=$?  # Capture exit status of the aws ssm command
-    
+    STATUS=$? # Capture exit status of the aws ssm command
+
     if [ $STATUS -ne 0 ]; then
         echo "Failed to fetch ${ENV_VAR}. Skipping."
         continue
     fi
-    
-    echo "${ENV_VAR}=${VALUE}" >> /tmp/isomer/.isomer.env
+
+    echo "${ENV_VAR}=${VALUE}" >>/tmp/isomer/.isomer.env
     echo "Saved ${ENV_VAR}"
 done
 
-set -e  # Exit on command failure from this point onwards
+set -e # Exit on command failure from this point onwards
 
 # Use flock to ensure that the EFS file is locked during the copy operation
 (
-  flock -n 200 || exit 1
+    flock -n 200 || exit 1
 
-  # Copy the local file to EFS
-  echo "Copying local env file to EFS"
-  cp /tmp/isomer/.isomer.env /efs/isomer/.isomer.env
+    # Copy the local file to EFS
+    echo "Copying local env file to EFS"
+    cp /tmp/isomer/.isomer.env /efs/isomer/.isomer.env
 
-  # Ensure the file on EFS is owned by webapp so it has access
-  chown webapp:webapp /efs/isomer/.isomer.env
+    # Ensure the file on EFS is owned by webapp so it has access
+    chown webapp:webapp /efs/isomer/.isomer.env
 
 ) 200>/efs/isomer/.isomer.lock
 
@@ -126,4 +127,4 @@ else
     echo "Couldn't acquire the lock. Another instance might be writing to the file."
 fi
 
-echo "Operation completed."
+echo "Operation completed."

src/config/config.ts

@@ -2,7 +2,7 @@
 
 convict.addFormat({
   name: "required-string",
   validate: (val: any) => {
     if (!val) throw new Error("value cannot be empty, null or undefined")
     if (typeof val !== "string") throw new Error("value must be a string")
   },
@@ -10,14 +10,14 @@
 
 convict.addFormat({
   name: "required-positive-number",
   validate: (val: any) => {
     if (val === null || val === undefined || val === "")
       throw new Error("value cannot be empty, null or undefined")
     if (typeof val !== "number") throw new Error("value must be a number")
   },
   coerce: (val: string) => {
     const coercedVal = Number(val)
     if (isNaN(coercedVal)) {
       throw new Error(
         "value provided is not a positive number. please provide a valid positive number"
       )
@@ -31,7 +31,7 @@
 
 convict.addFormat({
   name: "required-boolean",
   validate: (val: any) => {
     if (val === null || val === undefined)
       throw new Error("value cannot be empty, null or undefined")
     if (typeof val !== "boolean") throw new Error("value must be a boolean")
@@ -283,6 +283,13 @@
       format: "required-string",
       default: "",
     },
+    siteCloneFormKey: {
+      doc: "FormSG API key for site clone form",
+      env: "SITE_CLONE_FORM_KEY",
+      sensitive: true,
+      format: "required-string",
+      default: "",
+    },
   },
   postman: {
     apiKey: {

src/routes/formsgSiteClone.ts

@@ -0,0 +1,125 @@
+import { DecryptedContent } from "@opengovsg/formsg-sdk/dist/types"
+import autoBind from "auto-bind"
+import express, { RequestHandler } from "express"
+
+import { config } from "@config/config"
+
+import logger from "@logger/logger"
+
+import { getField } from "@utils/formsg-utils"
+
+import { ISOMER_ADMIN_EMAIL } from "@root/constants"
+import { attachFormSGHandler } from "@root/middleware"
+import GitFileSystemService from "@services/db/GitFileSystemService"
+import UsersService from "@services/identity/UsersService"
+import InfraService from "@services/infra/InfraService"
+import { mailer } from "@services/utilServices/MailClient"
+
+const SITE_CLONE_FORM_KEY = config.get("formSg.siteCloneFormKey")
+
+export interface FormsgSiteCloneRouterProps {
+  usersService: UsersService
+  infraService: InfraService
+  gitFileSystemService: GitFileSystemService
+}
+
+export class FormsgSiteCloneRouter {
+  private readonly gitFileSystemService: FormsgSiteCloneRouterProps["gitFileSystemService"]
+
+  constructor({ gitFileSystemService }: FormsgSiteCloneRouterProps) {
+    this.gitFileSystemService = gitFileSystemService
+    // We need to bind all methods because we don't invoke them from the class directly
+    autoBind(this)
+  }
+
+  cloneSiteToEfs: RequestHandler<
+    never,
+    Record<string, never>,
+    { data: { submissionId: string } },
+    never,
+    { submission: DecryptedContent }
+  > = async (req, res) => {
+    // 1. Extract arguments
+    const { submissionId } = req.body.data
+    const { responses } = res.locals.submission
+    const requesterEmail = getField(responses, "Email")
+    // NOTE: The field is required by our form so this cannot be empty or undefined
+    const githubRepoName = getField(responses, "Github Repo Name") as string
+
+    if (
+      !requesterEmail ||
+      !githubRepoName ||
+      !requesterEmail.endsWith("@open.gov.sg")
+    ) {
+      return this.sendCloneError(
+        ISOMER_ADMIN_EMAIL,
+        githubRepoName,
+        submissionId,
+        "Invalid email or missing github repo name detected for submission"
+      )
+    }
+
+    logger.info(
+      `${requesterEmail} requested for ${githubRepoName} to be cloned onto EFS`
+    )
+
+    this.gitFileSystemService
+      .clone(githubRepoName)
+      .map((path) => {
+        logger.info(`Cloned ${githubRepoName} to ${path}`)
+        this.sendCloneSuccess(
+          requesterEmail,
+          githubRepoName,
+          submissionId,
+          path
+        )
+      })
+      .mapErr((err) => {
+        logger.error(
+          `Cloning repo: ${githubRepoName} to EFS failed with error: ${JSON.stringify(
+            err
+          )}`
+        )
+        this.sendCloneError(
+          requesterEmail,
+          githubRepoName,
+          submissionId,
+          err.message
+        )
+      })
+  }
+
+  sendCloneSuccess = async (
+    requesterEmail: string,
+    githubRepoName: string,
+    submissionId: string,
+    path: string
+  ) => {
+    const subject = `[Isomer] Clone site ${githubRepoName} SUCCESS`
+    const html = `<p>Isomer site ${githubRepoName} was cloned successfully to EFS path: ${path}. (Form submission id [${submissionId}])</p>`
+    await mailer.sendMail(requesterEmail, subject, html)
+  }
+
+  async sendCloneError(
+    requesterEmail: string,
+    githubRepoName: string,
+    submissionId: string,
+    message: string
+  ) {
+    const subject = `[Isomer] Clone site ${githubRepoName} FAILURE`
+    const html = `<p>Isomer site ${githubRepoName} was <b>not</b> cloned successfully. Cloning failed with error: ${message} (Form submission id [${submissionId}])</p>`
+    await mailer.sendMail(requesterEmail, subject, html)
+  }
+
+  getRouter() {
+    const router = express.Router({ mergeParams: true })
+
+    router.post(
+      "/clone-site",
+      attachFormSGHandler(SITE_CLONE_FORM_KEY),
+      this.cloneSiteToEfs
+    )
+
+    return router
+  }
+}