[joysl] Neptune Autoscaling (#180)

* [joysl] Neptune Autocaling

With this commit, user can choose to enable Neptune Autoscaling along with following configurable paramters [TargetUtilization, MinCapacity,MaxCapacity]

* [joysl] Fix the description for Autoscaling param

* [joysl] Remove extra whitespace

* [joysl] Revert doublespace update on CF templates

source/common/config/rush/repo-state.json

@@ -1,5 +1,5 @@
 // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush.
 {
-  "pnpmShrinkwrapHash": "6ff045f325a4557e2c96b1cf8e6a73cea9638863",
+  "pnpmShrinkwrapHash": "a91e3cdfb0f70338c98919f90c62b3eb5bfe08c0",
   "preferredVersionsHash": "14c05a7722342014cec64c4bef7d9bed0d0b7b7f"
 }

source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml

@@ -2,7 +2,7 @@
 #-----------------------------------------------------------------------------------------------------------------------
 #   Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 #
-#  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+#  Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance
 #  with the License. A copy of the License is located at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
@@ -107,6 +107,30 @@ Parameters:
       - false
     MinLength: 1
 
+  ApplyNeptuneAutoscaling:
+    Description: If true, will apply auto-scaling as defined in `./cfn-autoscaling-neptune.yml'
+    Type: String
+    Default: false
+    AllowedValues:
+      - true
+      - false
+    MinLength: 1
+
+  NeptuneAutoScalingMinCapacity:
+    Description: The no. of desired minimum read replicas to provision
+    Type: Number
+    Default: 1
+
+  NeptuneAutoScalingMaxCapacity:
+    Description: The no. of desired maximum read replicas to provision
+    Type: Number
+    Default: 1
+
+  NeptuneTargetUtilization:
+    Description: Target utilization (0 to 100).
+    Type: Number
+    Default: 75
+
   CdfService:
     Description: Service name to tag resources.
     Type: String
@@ -136,11 +160,16 @@ Conditions:
 
   DeployInVPC: !Not [!Equals [!Ref VpcId, 'N/A']]
 
-  DeployWithProvisionedCapacity: !Not [!Equals [!Ref ProvisionedConcurrentExecutions, '0']]
+  DeployWithProvisionedCapacity:
+    !Not [!Equals [!Ref ProvisionedConcurrentExecutions, '0']]
   DeployWithScaling: !Equals [!Ref ApplyAutoscaling, 'true']
+  DeployWithNeptuneScaling: !Equals [!Ref ApplyNeptuneAutoscaling, 'true']
 
   DeployWithLambdaAuth:
-    !Or [!Equals [!Ref AuthType, 'LambdaRequest'], !Equals [!Ref AuthType, 'LambdaToken']]
+    !Or [
+      !Equals [!Ref AuthType, 'LambdaRequest'],
+      !Equals [!Ref AuthType, 'LambdaToken'],
+    ]
   EnableApiGatewayAccessLogs: !Equals [!Ref EnableApiGatewayAccessLogs, 'true']
 
 Resources:
@@ -321,6 +350,25 @@ Resources:
           Value: !Ref CdfService
     DependsOn: LambdaFunction
 
+  # Optional Neptune auto-scaling configuration
+  AutoScalingNeptuneNestedStack:
+    Type: AWS::CloudFormation::Stack
+    Condition: DeployWithNeptuneScaling
+    Properties:
+      TemplateURL: ./cfn-autoscaling-neptune.yml
+      Parameters:
+        NeptuneClusterID: !Sub '{{resolve:ssm:/cdf/asset-library/${Environment}/neptuneClusterID}}'
+        MinCapacity: !Ref NeptuneAutoScalingMinCapacity
+        MaxCapacity: !Ref NeptuneAutoScalingMaxCapacity
+        TargetUtilization: !Ref NeptuneTargetUtilization
+      TimeoutInMinutes: 5
+      Tags:
+        - Key: cdf_environment
+          Value: !Ref Environment
+        - Key: cdf_service
+          Value: !Ref CdfService
+    DependsOn: LambdaFunction
+
   IotFleetIndex:
     Type: Custom::IotFleetIndex
     Condition: DeployLiteMode

source/packages/services/assetlibrary/infrastructure/cfn-autoscaling-neptune.yml

@@ -0,0 +1,60 @@
+---
+#-----------------------------------------------------------------------------------------------------------------------
+#   Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance
+#  with the License. A copy of the License is located at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
+#  and limitations under the License.
+#-----------------------------------------------------------------------------------------------------------------------
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+
+Description: |
+  CDF Asset Library Service Scaling
+  =================================
+  This template provides an example of how to apply auto-scaling to the Asset Library Neptune Cluster.
+
+Parameters:
+  MinCapacity:
+    Description: The minimum value to scale to in response to a scale-in event.
+    Type: Number
+    Default: 1
+  MaxCapacity:
+    Description: The maximum value to scale to in response to a scale-out event.
+    Type: Number
+    Default: 1
+  TargetUtilization:
+    Description: Target utilization (0 to 90).
+    Type: Number
+    Default: 75
+  NeptuneClusterID:
+    Description: This parameter will contain the ClusterID used for Neptune Auto Scaling
+    Type: String
+    Default: cluster:test
+
+Resources:
+  ScalableTarget:
+    Type: AWS::ApplicationAutoScaling::ScalableTarget
+    Properties:
+      MaxCapacity: !Ref MaxCapacity
+      MinCapacity: !Ref MinCapacity
+      ResourceId: !Ref NeptuneClusterID
+      RoleARN: !Sub arn:aws:iam::${AWS::AccountId}:role/aws-service-role/neptune.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_NeptuneCluster
+      ScalableDimension: neptune:cluster:ReadReplicaCount
+      ServiceNamespace: neptune
+
+  TargetTrackingScalingPolicy:
+    Type: AWS::ApplicationAutoScaling::ScalingPolicy
+    Properties:
+      PolicyName: utilizationNeptune
+      PolicyType: TargetTrackingScaling
+      ScalingTargetId: !Ref ScalableTarget
+      TargetTrackingScalingPolicyConfiguration:
+        TargetValue: !Ref TargetUtilization
+        PredefinedMetricSpecification:
+          PredefinedMetricType: NeptuneReaderAverageCPUUtilization

source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml

@@ -2,7 +2,7 @@
 #-----------------------------------------------------------------------------------------------------------------------
 #   Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 #
-#  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+#  Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance
 #  with the License. A copy of the License is located at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
@@ -33,7 +33,7 @@ Parameters:
       Neptune DB instance type. The list of available instance types for your region can be found here: 
       https://aws.amazon.com/neptune/pricing/
     Type: String
-    AllowedPattern: "^db\\.[tr]\\d+[a-z0-9]*\\.[a-z0-9]*$"
+    AllowedPattern: '^db\\.[tr]\\d+[a-z0-9]*\\.[a-z0-9]*$'
     ConstraintDescription: Must be a valid Neptune instance type.
   CreateDBReplicaInstance:
     Description: Create a read replica for MultiAZ?
@@ -308,6 +308,14 @@ Resources:
     DependsOn:
       - NeptuneDBInstance
 
+  DBClusterIdSsmParameter:
+    Type: 'AWS::SSM::Parameter'
+    Properties:
+      Description: VPC ID for the account
+      Name: !Sub '/cdf/asset-library/${Environment}/neptuneClusterID'
+      Type: String
+      Value: !Sub 'cluster:${NeptuneDBCluster}'
+
 Outputs:
   DBClusterId:
     Description: Neptune Cluster Identifier

source/packages/services/installer/src/commands/modules/service/assetLibrary.ts

@@ -19,7 +19,11 @@ import { ModuleName, PostmanEnvironment, RestModule } from '../../../models/modu
 import { applicationConfigurationPrompt } from '../../../prompts/applicationConfiguration.prompt';
 import {
     enableAutoScaling,
-    provisionedConcurrentExecutions,
+    enableNeptuneAutoScaling,
+    maxNeptuneReadReplicas,
+    minNeptuneReadReplicas,
+    neptuneTargetUtilization,
+    provisionedConcurrentExecutions
 } from '../../../prompts/autoscaling.prompt';
 import { customDomainPrompt } from '../../../prompts/domain.prompt';
 import { redeployIfAlreadyExistsPrompt } from '../../../prompts/modules.prompt';
@@ -175,6 +179,10 @@ export class AssetLibraryInstaller implements RestModule {
                     },
                     enableAutoScaling(this.name, answers),
                     provisionedConcurrentExecutions(this.name, answers),
+                    enableNeptuneAutoScaling(this.name, answers),
+                    minNeptuneReadReplicas(this.name, answers),
+                    maxNeptuneReadReplicas(this.name, answers),
+                    neptuneTargetUtilization(this.name, answers),
                     ...applicationConfigurationPrompt(this.name, answers, [
                         {
                             question: 'Enable authorization?',
@@ -288,6 +296,11 @@ export class AssetLibraryInstaller implements RestModule {
             'ProvisionedConcurrentExecutions',
             answers.assetLibrary.provisionedConcurrentExecutions
         );
+        addIfSpecified('ApplyNeptuneAutoscaling', answers.assetLibrary.enableNeptuneAutoScaling);
+        addIfSpecified('NeptuneAutoScalingMinCapacity', answers.assetLibrary.minNeptuneReadReplicaCapacity);
+        addIfSpecified('NeptuneAutoScalingMaxCapacity', answers.assetLibrary.maxNeptuneReadReplicaCapacity);
+        addIfSpecified('NeptuneTargetUtilization', answers.assetLibrary.neptuneTargetUtilization);
+
         addIfSpecified(
             'CustomResourceVPCLambdaArn',
             answers.assetLibrary.mode === 'full'

source/packages/services/installer/src/models/answers.ts

@@ -103,6 +103,13 @@ export interface ProvisionedConcurrencyModuleAttribues extends ServiceModuleAttr
     provisionedConcurrentExecutions?: number;
     enableAutoScaling?: boolean;
 }
+
+export interface NeptuneScalingAttributes {
+  minNeptuneReadReplicaCapacity?: number;
+  maxNeptuneReadReplicaCapacity?: number;
+  enableNeptuneAutoScaling?: boolean;
+  neptuneTargetUtilization?: number;
+}
 export interface RestServiceModuleAttribues extends ServiceModuleAttributes {
     enableCustomDomain?: boolean;
     customDomainBasePath?: string;
@@ -123,7 +130,8 @@ export interface OrganizationManager extends RestServiceModuleAttribues {
 
 export interface AssetLibrary
     extends RestServiceModuleAttribues,
-        ProvisionedConcurrencyModuleAttribues {
+        ProvisionedConcurrencyModuleAttribues,
+        NeptuneScalingAttributes {
     mode?: 'full' | 'lite';
     neptuneDbInstanceType?: string;
     createDbReplicaInstance?: boolean;

source/packages/services/installer/src/prompts/autoscaling.prompt.ts

@@ -13,7 +13,8 @@
 import { Question } from 'inquirer';
 import {
     Answers,
-    ProvisionedConcurrencyModuleAttribues as ProvisionedConcurrencyModuleAttributes,
+    NeptuneScalingAttributes,
+    ProvisionedConcurrencyModuleAttribues as ProvisionedConcurrencyModuleAttributes
 } from '../models/answers';
 import { ModuleName } from '../models/modules';
 
@@ -49,3 +50,71 @@ export function provisionedConcurrentExecutions(
         },
     };
 }
+
+export function enableNeptuneAutoScaling(moduleName: ModuleName, answers: Answers): Question {
+  return {
+    message: 'Deployed with a autoscaling for Neptune?',
+    type: 'confirm',
+    name: `${moduleName}.enableNeptuneAutoScaling`,
+    default: (answers[moduleName] as NeptuneScalingAttributes)?.enableNeptuneAutoScaling ?? false,
+    askAnswered: true,
+  };
+}
+
+
+export function maxNeptuneReadReplicas(moduleName: ModuleName, answers: Answers): Question {
+  return {
+    message: 'The max number of read replicas when neptune is scaled. Default is 1',
+    type: 'input',
+    name: `${moduleName}.maxNeptuneReadReplicas`,
+    default: (answers[moduleName] as NeptuneScalingAttributes)?.maxNeptuneReadReplicaCapacity ?? 1,
+    askAnswered: true,
+    when(answers: Answers) {
+      return answers.assetLibrary?.enableNeptuneAutoScaling
+    },
+    validate(answer: number) {
+      if (answer < 0) {
+        return `You must enter number 0 or larger than 0.`;
+      }
+      return true;
+    },
+  };
+}
+
+export function minNeptuneReadReplicas(moduleName: ModuleName, answers: Answers): Question {
+  return {
+    message:  'The minimum number of read replicas when neptune is scaled. Default is 1',
+    type: 'input',
+    name: `${moduleName}.minNeptuneReadReplicas`,
+    default: (answers[moduleName] as NeptuneScalingAttributes)?.minNeptuneReadReplicaCapacity ?? 1,
+    askAnswered: true,
+    when(answers: Answers) {
+      return answers.assetLibrary?.enableNeptuneAutoScaling
+    },
+    validate(answer: number) {
+      if (answer < 0) {
+        return `You must enter number 0 or larger than 0.`;
+      }
+      return true;
+    },
+  };
+}
+
+export function neptuneTargetUtilization(moduleName: ModuleName, answers: Answers): Question {
+  return {
+    message:  'The target CPU utilization for Neptune to auto-scale read replica?',
+    type: 'input',
+    name: `${moduleName}.neptuneTargetUtilization`,
+    default: (answers[moduleName] as NeptuneScalingAttributes)?.neptuneTargetUtilization ?? 75,
+    askAnswered: true,
+    when(answers: Answers) {
+      return answers.assetLibrary?.enableNeptuneAutoScaling
+    },
+    validate(answer: number) {
+      if (answer <= 0) {
+        return `You must enter number 1 or larger .`;
+      }
+      return true;
+    },
+  };
+}