serverless.yml

# serverless.yml

service:
  name: cloudinary-to-s3

frameworkVersion: '>=1.0.0 <2.0.0'

plugins:
  - serverless-dotenv-plugin
  - serverless-pseudo-parameters
  - serverless-webpack
  - serverless-step-functions

package:
  # https://github.com/serverless-heaven/serverless-webpack/issues/299
  individually: false

custom:
  webpack:
    includeModules:
      forceExclude:
        - aws-sdk

provider:
  name: aws
  runtime: nodejs12.x
  stage: ${opt:stage, 'dev'} # Set the default stage used. Default is dev
  region: ap-southeast-2 # Overwrite the default region used. Default is us-east-1
  memorySize: 3008 # Overwrite the default memory size. Default is 1024
  timeout: 6 # The default is 6
  versionFunctions: false
  iamRoleStatements:
    - Effect: Allow
      Action:
        - s3:putObject
      Resource:
        - arn:aws:s3:::${env:S3_BUCKET_NAME}/*
    - Effect: 'Allow'
      Action:
        - 'sqs:sendMessage'
        - 'sqs:ReceiveMessage'
        - 'sqs:DeleteMessage'
        - 'sqs:GetQueueAttributes'
      Resource:
        - Fn::GetAtt:
            - 'ResourceUploadQueue'
            - 'Arn'
    - Effect: Allow
      Action:
        - states:StartExecution
      Resource: ${self:resources.Outputs.MyStateMachine.Value}
    - Effect: Allow #allow lambda to read from the event stream
      Action:
        - dynamodb:PutItem
      Resource:
        - ${self:resources.Outputs.MetaDataTableArn.Value}
        - ${self:resources.Outputs.FailedRecordsTableArn.Value}

functions:
  getResources:
    handler: src/get.handler
    environment:
      CLOUDINARY_API_KEY: ${env:CLOUDINARY_API_KEY}
      CLOUDINARY_API_SECRET: ${env:CLOUDINARY_API_SECRET}
      CLOUDINARY_CLOUD_ID: ${env:CLOUDINARY_CLOUD_ID}
      QUEUE:
        Ref: ResourceUploadQueue
  upload:
    handler: src/upload.handler
    timeout: 180 # 3 mins to accommodate file sizes
    environment:
      S3_BUCKET_NAME: ${env:S3_BUCKET_NAME}
      META_DATA_TABLE_NAME: ${env:META_DATA_TABLE_NAME}
      FAILED_RECORDS_TABLE_NAME: ${env:FAILED_RECORDS_TABLE_NAME}
    events:
      - sqs:
          arn:
            Fn::GetAtt:
              - 'ResourceUploadQueue'
              - 'Arn'
          batchSize: 1 # Set to 1 to limit memory
  start:
    handler: src/start.handler
    environment:
      STATE_MACHINE_ARN: ${self:resources.Outputs.MyStateMachine.Value}

stepFunctions:
  stateMachines:
    CloudinaryStorageMigration:
      name: CloudinaryStorageMigration
      definition:
        StartAt: GetResources
        States:
          GetResources:
            Type: Task
            Resource: arn:aws:lambda:${opt:region}:#{AWS::AccountId}:function:${self:service}-${opt:stage, self:provider.stage}-getResources
            Next: HasNext
            Retry:
              - ErrorEquals:
                  - Lambda.ServiceException
                  - Lambda.SdkClientException
                  - Lambda.Unknown
                IntervalSeconds: 2
                MaxAttempts: 6
                BackoffRate: 2
          HasNext:
            Type: Choice
            Choices:
              - Variable: '$.hasNext'
                BooleanEquals: true
                Next: Wait1Second
            Default: Done
          Wait1Second:
            Type: Wait
            Seconds: 1
            Next: LimitReached
          LimitReached:
            Type: Choice
            Choices:
              - Variable: '$.limitReached'
                BooleanEquals: true
                Next: StartNewExecution
            Default: GetResources
          StartNewExecution:
            Type: Task
            Resource: arn:aws:lambda:${opt:region}:#{AWS::AccountId}:function:${self:service}-${opt:stage, self:provider.stage}-start
            Next: Done
            Retry:
              - ErrorEquals:
                  - Lambda.ServiceException
                  - Lambda.SdkClientException
                  - Lambda.Unknown
                IntervalSeconds: 2
                MaxAttempts: 6
                BackoffRate: 2
          Done:
            Type: Succeed

resources:
  Resources:
    ResourceUploadQueue:
      DependsOn:
        - ResourceUploadDeadLetterQueue
      Type: AWS::SQS::Queue
      Properties:
        VisibilityTimeout: 300 # The extra time allows for Lambda to retry if the function execution is throttled
        MessageRetentionPeriod: 604800
        RedrivePolicy:
          deadLetterTargetArn:
            Fn::GetAtt:
              - 'ResourceUploadDeadLetterQueue'
              - 'Arn'
          maxReceiveCount: 5 # This will help avoid sending messages to the dead-letter queue due to throttling.
    ResourceUploadDeadLetterQueue:
      Type: AWS::SQS::Queue
      Properties:
        MessageRetentionPeriod: 1209600
    MetaDataTable:
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${env:META_DATA_TABLE_NAME}
        AttributeDefinitions:
          - AttributeName: cloudinary_url
            AttributeType: S
        KeySchema:
          - AttributeName: cloudinary_url
            KeyType: HASH
        SSESpecification:
          SSEEnabled: false
        BillingMode: PAY_PER_REQUEST
    FailedRecordsTable:
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${env:FAILED_RECORDS_TABLE_NAME}
        AttributeDefinitions:
          - AttributeName: cloudinary_url
            AttributeType: S
        KeySchema:
          - AttributeName: cloudinary_url
            KeyType: HASH
        SSESpecification:
          SSEEnabled: false
        BillingMode: PAY_PER_REQUEST
  Outputs:
    MyStateMachine:
      Description: The ARN of the example state machine
      Value:
        Ref: CloudinaryStorageMigration
    MetaDataTableArn:
      Value:
        'Fn::GetAtt': [MetaDataTable, Arn]
    FailedRecordsTableArn:
      Value:
        'Fn::GetAtt': [FailedRecordsTable, Arn]