trial/dan-1292-create-infra-scripts-to-automate-the-deployment-onto-ecs

deployment/aws_ecs_fargate/cloudformation/deploy.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Variables
+ECS_FARGATE_VPC_CIDR=$1
+if [ -z "$ECS_FARGATE_VPC_CIDR" ]; then
+    echo "Missing the  required vpc cidr argument.  Example: ./deploy.sh 172.30.0.0/16"
+    exit 1
+fi
+
+AWS_REGION="${AWS_REGION:-us-west-1}"
+TEMPLATE_DIR="$(pwd)"
+SERVICE_DIR="$TEMPLATE_DIR/services"
+ENVIRONMENT="${ENVIRONMENT:-production}"
+S3_BUCKET="${S3_BUCKET:-onyx-ecs-fargate-configs}"
+
+INFRA_ORDER=(
+  "onyx_efs_template.yaml"
+  "onyx_cluster_template.yaml"
+  "onyx_lambda_cron_restart_services_template.yaml"
+  "onyx_acm_template.yaml"
+)
+
+# Deployment order for services
+SERVICE_ORDER=(
+  "onyx_postgres_service_template.yaml"
+  "onyx_redis_service_template.yaml"
+  "onyx_vespaengine_service_template.yaml"
+  "onyx_model_server_indexing_service_template.yaml"
+  "onyx_model_server_inference_service_template.yaml"
+  "onyx_backend_api_server_service_template.yaml"
+  "onyx_backend_background_server_service_template.yaml"
+  "onyx_web_server_service_template.yaml"
+  "onyx_nginx_service_template.yaml"
+)
+
+# JSON file mapping for services
+COMMON_PARAMETERS_FILE="$SERVICE_DIR/onyx_services_parameters.json"
+NGINX_PARAMETERS_FILE="$SERVICE_DIR/onyx_nginx_parameters.json"
+EFS_PARAMETERS_FILE="onyx_efs_parameters.json"
+ACM_PARAMETERS_FILE="onyx_acm_parameters.json"
+CLUSTER_PARAMETERS_FILE="onyx_cluster_parameters.json"
+LAMBDA_PARAMETERS_FILE="onyx_lambda_cron_restart_services_parameters.json"
+
+# Function to validate a CloudFormation template
+validate_template() {
+  local template_file=$1
+  echo "Validating template: $template_file..."
+  aws cloudformation validate-template --template-body file://"$template_file" --region "$AWS_REGION" > /dev/null
+  if [ $? -ne 0 ]; then
+    echo "Error: Validation failed for $template_file. Exiting."
+    exit 1
+  fi
+  echo "Validation succeeded for $template_file."
+}
+
+# Function to deploy a CloudFormation stack
+deploy_stack() {
+  local stack_name=$1
+  local template_file=$2
+  local config_file=$3
+
+  echo "Checking if stack $stack_name exists..."
+  if aws cloudformation describe-stacks --stack-name "$stack_name" --region "$AWS_REGION" > /dev/null 2>&1; then
+    echo "Stack $stack_name already exists. Skipping deployment."
+    return 0
+  fi
+
+  echo "Deploying stack: $stack_name with template: $template_file and config: $config_file..."
+  if [ -f "$config_file" ]; then
+    aws cloudformation deploy \
+      --stack-name "$stack_name" \
+      --template-file "$template_file" \
+      --parameter-overrides file://"$config_file" \
+      --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \
+      --region "$AWS_REGION" \
+      --no-cli-auto-prompt > /dev/null
+  else
+    echo "Missing required parameter json file"
+	exit 1
+  fi
+
+  if [ $? -ne 0 ]; then
+    echo "Error: Deployment failed for $stack_name. Exiting."
+    exit 1
+  fi
+  echo "Stack deployed successfully: $stack_name."
+}
+
+convert_underscores_to_hyphens() {
+  local input_string="$1"
+  local converted_string="${input_string//_/-}"
+  echo "$converted_string"
+}
+
+deploy_infra_stacks() {
+    for template_name in "${INFRA_ORDER[@]}"; do
+      template_file="$template_name"
+      stack_name="$ENVIRONMENT-$(basename "$template_name" _template.yaml)"
+      stack_name=$(convert_underscores_to_hyphens "$stack_name")
+
+      # Use the common parameters file for specific services
+      if [[ "$template_name" =~ ^(onyx_cluster_template.yaml)$ ]]; then
+        config_file="$CLUSTER_PARAMETERS_FILE"
+      elif [[ "$template_name" =~ ^(onyx_efs_template.yaml)$ ]]; then
+        config_file="$EFS_PARAMETERS_FILE"
+      elif [[ "$template_name" =~ ^(onyx_acm_template.yaml)$ ]]; then
+          config_file="$ACM_PARAMETERS_FILE"
+      elif [[ "$template_name" =~ ^(onyx_lambda_cron_restart_services_template.yaml)$ ]]; then
+          config_file="$LAMBDA_PARAMETERS_FILE"
+      else
+          config_file=""
+      fi
+
+      if [ -f "$template_file" ]; then
+        validate_template "$template_file"
+        deploy_stack "$stack_name" "$template_file" "$config_file"
+        if [[ "$template_name" =~ ^(onyx_cluster_template.yaml)$ ]]; then
+            echo "s3 bucket now exists, copying nginx and postgres configs to s3 bucket"
+            ECS_FARGATE_VPC_CIDR=${ECS_FARGATE_VPC_CIDR} ../../data/postgres/update_pg_hba.sh
+            aws s3 cp ../../data/postgres/pg_hba.conf "s3://${ENVIRONMENT}-${S3_BUCKET}/postgres/"
+            aws s3 cp ../../data/nginx/ "s3://${ENVIRONMENT}-${S3_BUCKET}/nginx/" --recursive
+        fi
+      else
+        echo "Warning: Template file $template_file not found. Skipping."
+      fi
+    done
+}
+
+deploy_services_stacks() { 
+    for template_name in "${SERVICE_ORDER[@]}"; do
+      template_file="$SERVICE_DIR/$template_name"
+      stack_name="$ENVIRONMENT-$(basename "$template_name" _template.yaml)"
+      stack_name=$(convert_underscores_to_hyphens "$stack_name")
+
+      # Use the common parameters file for specific services
+      if [[ "$template_name" =~ ^(onyx_backend_api_server_service_template.yaml|onyx_postgres_service_template.yaml|onyx_backend_background_server_service_template.yaml|onyx_redis_service_template.yaml|onyx_model_server_indexing_service_template.yaml|onyx_model_server_inference_service_template.yaml|onyx_vespaengine_service_template.yaml|onyx_web_server_service_template.yaml)$ ]]; then
+        config_file="$COMMON_PARAMETERS_FILE"
+      elif [[ "$template_name" =~ ^(onyx_nginx_service_template.yaml)$ ]]; then
+        config_file="$NGINX_PARAMETERS_FILE"
+      else
+          config_file=""
+      fi
+
+      if [ -f "$template_file" ]; then
+        validate_template "$template_file"
+        deploy_stack "$stack_name" "$template_file" "$config_file"
+      else
+        echo "Warning: Template file $template_file not found. Skipping."
+      fi
+    done
+}
+
+echo "Starting deployment of Onyx to ECS Fargate Cluster..."
+deploy_infra_stacks
+deploy_services_stacks
+
+echo "All templates validated and deployed successfully."

deployment/aws_ecs_fargate/cloudformation/onyx_acm_parameters.json

@@ -0,0 +1,5 @@
+[
+    {"ParameterKey": "DomainName", "ParameterValue": "danswer.ai"},
+    {"ParameterKey": "Environment", "ParameterValue": "production"},
+    {"ParameterKey": "ValidationMethod", "ParameterValue": "DNS"}
+]

deployment/aws_ecs_fargate/cloudformation/onyx_acm_template.yaml

@@ -0,0 +1,31 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: CloudFormation template to create an ACM Certificate.
+
+Parameters:
+  DomainName:
+    Type: String
+    Description: The primary domain name for the certificate (e.g., example.com).
+    Default: example.com
+  Environment:
+    Type: String
+    Default: production
+  ValidationMethod:
+    Type: String
+    Default: DNS
+
+Resources:
+  Certificate:
+    Type: AWS::CertificateManager::Certificate
+    Properties:
+      DomainName: !Ref DomainName
+      ValidationMethod: !Ref ValidationMethod
+      Tags:
+        - Key: env
+          Value: !Ref Environment
+
+Outputs:
+  OutputAcm:
+    Description: ACM Cert Id
+    Value: !Ref Certificate
+    Export:
+      Name: !Sub ${AWS::StackName}-OnyxCertificate

deployment/aws_ecs_fargate/cloudformation/onyx_cluster_parameters.json

@@ -0,0 +1,5 @@
+[
+    {"ParameterKey": "Environment", "ParameterValue": "production"},
+    {"ParameterKey": "OnyxNamespace", "ParameterValue": "onyx2"},
+    {"ParameterKey": "VpcID", "ParameterValue": "vpc-098cfa79d637dabff"}
+]

deployment/aws_ecs_fargate/cloudformation/onyx_cluster_template.yaml

@@ -0,0 +1,156 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: The template used to create an ECS Cluster from the ECS Console.
+
+Parameters:
+  Environment:
+    Type: String
+    Description: The environment that is used in the name of the cluster as well.
+  OnyxNamespace:
+    Type: String
+    Default: onyx
+  VpcID:
+    Type: String
+    Default: vpc-098cfa79d637dabff
+
+Resources:
+  ECSCluster:
+    Type: AWS::ECS::Cluster
+    Properties:
+      ClusterName: !Sub ${Environment}-onyx-cluster
+      CapacityProviders:
+        - FARGATE
+        - FARGATE_SPOT
+      ClusterSettings:
+        - Name: containerInsights
+          Value: enhanced
+      ServiceConnectDefaults:
+        Namespace: !Sub ${Environment}-onyx-cluster
+      Tags:
+        - Key: env
+          Value: !Ref Environment
+        - Key: app
+          Value: onyx
+
+  S3Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: !Sub ${Environment}-onyx-ecs-fargate-configs
+      AccessControl: Private
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
+          - ServerSideEncryptionByDefault:
+              SSEAlgorithm: AES256
+      PublicAccessBlockConfiguration:
+        BlockPublicAcls: true
+        BlockPublicPolicy: true
+        IgnorePublicAcls: true
+        RestrictPublicBuckets: true
+
+  PrivateDnsNamespace:
+    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
+    Properties:
+      Description: AWS Cloud Map private DNS namespace for resources for onyx website.
+      Vpc: !Ref VpcID
+      Name: !Ref OnyxNamespace
+      Properties:
+        DnsProperties:
+          SOA:
+            TTL: 50
+
+  ECSTaskRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub ${Environment}-OnyxEcsTaskRole
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: ecs-tasks.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: "EFSPolicy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Sid: "VisualEditor0"
+                Effect: Allow
+                Action:
+                  - "elasticfilesystem:*"
+                Resource:
+                  - !Sub "arn:aws:elasticfilesystem:*:${AWS::AccountId}:access-point/*"
+                  - !Sub "arn:aws:elasticfilesystem:*:${AWS::AccountId}:file-system/*"
+              - Sid: "VisualEditor1"
+                Effect: Allow
+                Action: "elasticfilesystem:*"
+                Resource: "*"
+        - PolicyName: "S3Policy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Sid: "VisualEditor0"
+                Effect: Allow
+                Action:
+                  - "s3:GetObject"
+                  - "s3:ListBucket"
+                Resource:
+                  - !Sub "arn:aws:s3:::${Environment}-onyx-ecs-fargate-configs/*"
+                  - !Sub "arn:aws:s3:::${Environment}-onyx-ecs-fargate-configs"
+
+  ECSTaskExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub ${Environment}-OnyxECSTaskExecutionRole
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: ecs-tasks.amazonaws.com
+            Action: sts:AssumeRole
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
+      Policies:
+        - PolicyName: "CloudWatchLogsPolicy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Sid: "VisualEditor0"
+                Effect: Allow
+                Action: "logs:CreateLogGroup"
+                Resource: !Sub "arn:aws:logs:*:${AWS::AccountId}:log-group:*"
+        - PolicyName: "SecretsManagerPolicy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - secretsmanager:GetSecretValue
+                Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${Environment}/postgres/user/password-*
+
+Outputs:
+  OutputEcsCluster:
+    Description: Onyx ECS Cluster
+    Value: !Ref ECSCluster
+    Export:
+      Name: !Sub ${AWS::StackName}-ECSClusterName
+  OutputECSTaskRole:
+    Description: Onyx ECS Task Role
+    Value: !Ref ECSTaskRole
+    Export:
+      Name: !Sub ${AWS::StackName}-ECSTaskRole
+  OutputECSTaskExecutionRole:
+    Description: Onyx ECS TaskExecutionRole
+    Value: !Ref ECSTaskExecutionRole
+    Export:
+      Name: !Sub ${AWS::StackName}-ECSTaskExecutionRole
+  OutputOnyxNamespace:
+    Description: Onyx CloudMap namespace ID for ECS service discvoery.
+    Value: !Ref PrivateDnsNamespace
+    Export:
+      Name: !Sub ${AWS::StackName}-OnyxNamespace
+  OutputOnyxNamespaceName:
+    Description: Onyx CloudMap namespace domain name for ECS service discvoery.
+    Value: !Ref OnyxNamespace
+    Export:
+      Name: !Sub ${AWS::StackName}-OnyxNamespaceName

deployment/aws_ecs_fargate/cloudformation/onyx_efs_parameters.json

@@ -0,0 +1,7 @@
+[
+    {"ParameterKey": "EFSName", "ParameterValue": "onyx-efs"},
+    {"ParameterKey": "Environment", "ParameterValue": "production"},
+    {"ParameterKey": "VpcID", "ParameterValue": "vpc-098cfa79d637dabff"},
+    {"ParameterKey": "SubnetId1", "ParameterValue": "subnet-0af372d676f65b287"},
+    {"ParameterKey": "SubnetId2", "ParameterValue": "subnet-06721b79e2e51d22c"}
+]