diff --git a/packages/cli/templates/.env.local b/packages/cli/templates/.env.local index 6ad904a..53725fd 100644 --- a/packages/cli/templates/.env.local +++ b/packages/cli/templates/.env.local @@ -30,6 +30,7 @@ SFN_COMMAND_ARN=arn:aws:states:ap-northeast-1:101010101010:stateMachine:command SNS_ENDPOINT=http://localhost:4002 SNS_REGION=ap-northeast-1 SNS_TOPIC_ARN=arn:aws:sns:ap-northeast-1:101010101010:CqrsSnsTopic +SNS_ALARM_TOPIC_ARN=arn:aws:sns:ap-northeast-1:101010101010:AlarmSnsTopic # Cognito endpoint, useful for local development COGNITO_URL=http://localhost:9229 COGNITO_USER_POOL_ID=local_2G7noHgW diff --git a/packages/cli/templates/infra-local/cognito-local/package-lock.json b/packages/cli/templates/infra-local/cognito-local/package-lock.json index f987335..4987b73 100644 --- a/packages/cli/templates/infra-local/cognito-local/package-lock.json +++ b/packages/cli/templates/infra-local/cognito-local/package-lock.json @@ -919,10 +919,9 @@ } }, "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", - "license": "MIT", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -943,7 +942,7 @@ "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "0.1.12", "proxy-addr": "~2.0.7", "qs": "6.13.0", "range-parser": "~1.2.1", @@ -958,6 +957,10 @@ }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/fast-redact": { @@ -1854,10 +1857,9 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", - "license": "MIT" + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==" }, "node_modules/picomatch": { "version": "2.3.1", diff --git a/packages/cli/templates/infra-local/elasticmq.conf b/packages/cli/templates/infra-local/elasticmq.conf index 2095451..c74f59e 100644 --- a/packages/cli/templates/infra-local/elasticmq.conf +++ b/packages/cli/templates/infra-local/elasticmq.conf @@ -15,6 +15,11 @@ queues { delay = 5 seconds receiveMessageWait = 0 seconds } + alarm-queue { + defaultVisibilityTimeout = 60 seconds + delay = 5 seconds + receiveMessageWait = 0 seconds + } } aws { diff --git a/packages/cli/templates/infra-local/serverless.yml b/packages/cli/templates/infra-local/serverless.yml index 94155d1..6b3b175 100644 --- a/packages/cli/templates/infra-local/serverless.yml +++ b/packages/cli/templates/infra-local/serverless.yml @@ -40,6 +40,10 @@ custom: ['notification-action', 'command-status', 'task-status'], } queue: http://localhost:9324/101010101010/notification-queue + - topic: + topicName: AlarmSnsTopic + rawMessageDelivery: 'true' + queue: http://localhost:9324/101010101010/alarm-queue # host: 0.0.0.0 # Optional, defaults to 127.0.0.1 if not provided to serverless-offline # sns-endpoint: http://127.0.0.1:4567 # Optional. Only if you want to use a custom SNS provider endpoint # sns-subscribe-endpoint: http://127.0.0.1:3000 # Optional. Only if you want to use a custom subscribe endpoint from SNS to send messages back to diff --git a/packages/cli/templates/infra/libs/infra-stack.ts b/packages/cli/templates/infra/libs/infra-stack.ts index adcd583..3817872 100644 --- a/packages/cli/templates/infra/libs/infra-stack.ts +++ b/packages/cli/templates/infra/libs/infra-stack.ts @@ -334,6 +334,7 @@ export class InfraStack extends cdk.Stack { S3_BUCKET_NAME: ddbBucket.bucketName, SFN_COMMAND_ARN: commandSfnArn, SNS_TOPIC_ARN: mainSns.topicArn, + SNS_ALARM_TOPIC_ARN: alarmSns.topicArn, COGNITO_USER_POOL_ID: userPool.userPoolId, APPSYNC_ENDPOINT: appSyncApi.graphqlUrl, SES_FROM_EMAIL: props.config.fromEmailAddress, @@ -838,6 +839,7 @@ export class InfraStack extends cdk.Stack { ddbBucket.grantReadWrite(lambdaApi) publicBucket.grantReadWrite(lambdaApi) mainSns.grantPublish(lambdaApi) + alarmSns.grantPublish(lambdaApi) taskSqs.grantSendMessages(lambdaApi) notifySqs.grantSendMessages(lambdaApi) appSyncApi.grantMutation(lambdaApi) @@ -912,6 +914,7 @@ export class InfraStack extends cdk.Stack { ddbBucket.grantReadWrite(taskRole) publicBucket.grantReadWrite(taskRole) mainSns.grantPublish(taskRole) + alarmSns.grantPublish(taskRole) taskSqs.grantSendMessages(taskRole) notifySqs.grantSendMessages(taskRole) appSyncApi.grantMutation(taskRole) diff --git a/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap b/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap index f076409..6d53ae4 100644 --- a/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap +++ b/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap @@ -642,6 +642,9 @@ exports[`snapshot test for InfraStack 1`] = ` }, "SES_FROM_EMAIL": "noreply@mbc-cqrs-serverless.mbc-net.com", "SFN_COMMAND_ARN": "arn:aws:states:ap-northeast-1:101010101010:stateMachine:dev-cdk-test-deploy-command-handler", + "SNS_ALARM_TOPIC_ARN": { + "Ref": "alarmsnsFB7BBC3B", + }, "SNS_TOPIC_ARN": { "Ref": "mainsnsC0381B34", }, @@ -994,6 +997,13 @@ exports[`snapshot test for InfraStack 1`] = ` "Ref": "mainsnsC0381B34", }, }, + { + "Action": "sns:Publish", + "Effect": "Allow", + "Resource": { + "Ref": "alarmsnsFB7BBC3B", + }, + }, { "Action": [ "sqs:SendMessage", diff --git a/packages/core/.env.example b/packages/core/.env.example index 72c5f03..c105ed5 100644 --- a/packages/core/.env.example +++ b/packages/core/.env.example @@ -11,7 +11,7 @@ LOG_LEVEL=verbose # debug, verbose, info, warn, error, fatal # disable event route for API GW integration EVENT_SOURCE_DISABLED=false # DynamoDB endpoint, useful for local development -DYNAMODB_ENDPOINT=http://0.0.0.0:8000 +DYNAMODB_ENDPOINT=http://0.0.0.0:8000 DYNAMODB_REGION=ap-northeast-1 # set the limit size for `attributes` of object in DDB ATTRIBUTE_LIMIT_SIZE=389120 # bytes, refer to https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html#limits-attributes @@ -28,6 +28,7 @@ SFN_COMMAND_ARN=arn:aws:states:ap-northeast-1:101010101010:stateMachine:command SNS_ENDPOINT=http://0.0.0.0:4002 SNS_REGION=ap-northeast-1 SNS_TOPIC_ARN=arn:aws:sns:ap-northeast-1:101010101010:CqrsSnsTopic +SNS_ALARM_TOPIC_ARN=arn:aws:sns:ap-northeast-1:101010101010:AlarmSnsTopic # Cognito endpoint, useful for local development COGNITO_URL=http://0.0.0.0:9229 COGNITO_USER_POOL_ID=local_2G7noHgW diff --git a/packages/core/src/commands/command.event.handler.spec.ts b/packages/core/src/commands/command.event.handler.spec.ts index 56e05d4..db54174 100644 --- a/packages/core/src/commands/command.event.handler.spec.ts +++ b/packages/core/src/commands/command.event.handler.spec.ts @@ -27,6 +27,7 @@ import { DataSyncDdsHandler } from './handlers/data-sync-dds.handler' import { HistoryService } from './history.service' import { DataSyncCommandSfnName } from '../command-events/sfn-name.enum' import { TtlService } from './ttl.service' +import { SnsClientFactory } from '../queue/sns-client-factory' export class MockedHandler implements IDataSyncHandler { async up(cmd: CommandModel): Promise { @@ -127,6 +128,8 @@ const sfnFinishDataEvent = createEvent(DataSyncCommandSfnName.FINISH) const keys = { NODE_ENV: 'env', APP_NAME: 'app_name', + SNS_TOPIC_ARN: 'main_topic_arn', + SNS_ALARM_TOPIC_ARN: 'alarm_topic_arn', } describe('DataSyncCommandSfnEventHandler', () => { @@ -147,6 +150,7 @@ describe('DataSyncCommandSfnEventHandler', () => { SnsService, MockedHandler, TtlService, + SnsClientFactory, { provide: MODULE_OPTIONS_TOKEN, useValue: { @@ -240,6 +244,30 @@ describe('DataSyncCommandSfnEventHandler', () => { expect(result).toEqual(expect.objectContaining({ result: -1 })) }) + it('should publish sns alarm when executing the stale check version event ', async () => { + // Arrange + dynamoDBMock.on(UpdateItemCommand).resolves({} as any) + snsMock.on(PublishCommand).resolves({} as any) + dynamoDBMock.on(GetItemCommand).resolves({ + Item: { + version: { + N: '1', + }, + }, + }) + + // Action + const result = await commandEventHandler.execute(sfnCheckVersionEvent) + + // Assert + expect(snsMock).toHaveReceivedCommandTimes(PublishCommand, 3) + expect(snsMock).toHaveReceivedNthCommandWith(2, PublishCommand, { + Message: expect.stringMatching( + /(?=.*"action":"sfn-alarm")(?=.*next version must be 2 but got 1)/g, + ), + }) + }) + it('should call the AWS service with the correct parameters when executing the correct check version event', async () => { // Arrange dynamoDBMock.on(UpdateItemCommand).resolves({} as any) @@ -447,7 +475,7 @@ describe('DataSyncCommandSfnEventHandler', () => { }) }) - it('should throw not found handler error when executing the sync data event', async () => { + it('should throw not found handler error and publish sns alarm when executing the sync data event', async () => { // Arrange dynamoDBMock.on(UpdateItemCommand).resolves({} as any) snsMock.on(PublishCommand).resolves({} as any) @@ -461,9 +489,15 @@ describe('DataSyncCommandSfnEventHandler', () => { }), ), ).rejects.toThrow('SyncDataHandler not found!') + + expect(snsMock).toHaveReceivedNthCommandWith(3, PublishCommand, { + Message: expect.stringMatching( + /(?=.*"action":"sfn-alarm")(?=.*Error: SyncDataHandler not found!)/g, + ), + }) }) - it('should throw empty handler error when executing the sync data event', async () => { + it('should throw empty handler error and publish sns alarm when executing the sync data event', async () => { // Arrange dynamoDBMock.on(UpdateItemCommand).resolves({} as any) snsMock.on(PublishCommand).resolves({} as any) @@ -478,6 +512,12 @@ describe('DataSyncCommandSfnEventHandler', () => { }), ), ).rejects.toThrow('SyncDataHandler empty!') + + expect(snsMock).toHaveReceivedNthCommandWith(3, PublishCommand, { + Message: expect.stringMatching( + /(?=.*"action":"sfn-alarm")(?=.*Error: SyncDataHandler empty!)/g, + ), + }) }) it('should call handler up when executing the correct sync data event', async () => { diff --git a/packages/core/src/commands/command.event.handler.ts b/packages/core/src/commands/command.event.handler.ts index 3f257e9..60d4cc0 100644 --- a/packages/core/src/commands/command.event.handler.ts +++ b/packages/core/src/commands/command.event.handler.ts @@ -1,4 +1,5 @@ import { Inject, Injectable, Logger } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' import { DataSyncCommandSfnEvent, @@ -7,7 +8,8 @@ import { import { DataSyncCommandSfnName } from '../command-events/sfn-name.enum' import { S3Service } from '../data-store' import { removeSortKeyVersion } from '../helpers/key' -import { CommandModuleOptions } from '../interfaces' +import { CommandModuleOptions, INotification } from '../interfaces' +import { SnsService } from '../queue' import { MODULE_OPTIONS_TOKEN } from './command.module-definition' import { CommandService } from './command.service' import { DataService } from './data.service' @@ -17,6 +19,7 @@ import { HistoryService } from './history.service' @Injectable() export class CommandEventHandler { private readonly logger: Logger + private readonly alarmTopicArn: string constructor( @Inject(MODULE_OPTIONS_TOKEN) @@ -25,10 +28,13 @@ export class CommandEventHandler { private readonly dataService: DataService, private readonly historyService: HistoryService, private readonly s3Service: S3Service, + private readonly snsService: SnsService, + private readonly config: ConfigService, ) { this.logger = new Logger( `${CommandEventHandler.name}:${this.options.tableName}`, ) + this.alarmTopicArn = this.config.get('SNS_ALARM_TOPIC_ARN') } async execute( @@ -55,6 +61,7 @@ export class CommandEventHandler { getCommandStatus(event.stepStateName, CommandStatus.STATUS_FAILED), event.commandRecord.requestId, ) + await this.publishAlarm(event, (error as Error).stack) throw error } } @@ -125,12 +132,15 @@ export class CommandEventHandler { } } - return { + const errorDetails = { result: -1, error: 'version is not match', cause: 'next version must be ' + nextVersion + ' but got ' + new String(1), } + + await this.publishAlarm(event, errorDetails) + return errorDetails } protected async setTtlCommand( @@ -198,4 +208,27 @@ export class CommandEventHandler { return null } + + protected async publishAlarm( + event: DataSyncCommandSfnEvent, + errorDetails: any, + ): Promise { + this.logger.debug('event', event) + const alarm: INotification = { + action: 'sfn-alarm', + id: `${event.commandKey.pk}#${event.commandKey.sk}`, + table: this.options.tableName, + pk: event.commandKey.pk, + sk: event.commandKey.sk, + tenantCode: event.commandKey.pk.substring( + event.commandKey.pk.indexOf('#') + 1, + ), + content: { + errorMessage: errorDetails, + sfnId: event.source, + }, + } + this.logger.error('alarm:::', alarm) + await this.snsService.publish(alarm, this.alarmTopicArn) + } } diff --git a/packages/core/src/commands/ttl.service.ts b/packages/core/src/commands/ttl.service.ts index 09a9feb..104fe22 100644 --- a/packages/core/src/commands/ttl.service.ts +++ b/packages/core/src/commands/ttl.service.ts @@ -24,7 +24,7 @@ export class TtlService implements ITtlService { startDate?: Date, ): Promise { const numberOfDay = await this.getTtlConfiguration(type, tenantCode) - this.logger.log('numberOfDay', numberOfDay) + this.logger.debug('numberOfDay', numberOfDay) return numberOfDay ? this.calculateUnixTime(numberOfDay, startDate) : null } diff --git a/packages/core/src/queue/queue.module.ts b/packages/core/src/queue/queue.module.ts index c62f5c0..bfb75e4 100644 --- a/packages/core/src/queue/queue.module.ts +++ b/packages/core/src/queue/queue.module.ts @@ -1,10 +1,11 @@ import { Global, Module } from '@nestjs/common' import { SnsService } from './sns.service' +import { SnsClientFactory } from './sns-client-factory' @Global() @Module({ - providers: [SnsService], + providers: [SnsService, SnsClientFactory], exports: [SnsService], }) export class QueueModule {} diff --git a/packages/core/src/queue/sns-client-factory.ts b/packages/core/src/queue/sns-client-factory.ts new file mode 100644 index 0000000..0fddf55 --- /dev/null +++ b/packages/core/src/queue/sns-client-factory.ts @@ -0,0 +1,21 @@ +import { SNSClient } from '@aws-sdk/client-sns' +import { Injectable } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' + +@Injectable() +export class SnsClientFactory { + private clients = new Map() + + constructor(private readonly config: ConfigService) {} + + getClient(topicArn: string): SNSClient { + if (!this.clients.has(topicArn)) { + const snsClient = new SNSClient({ + endpoint: this.config.get('SNS_ENDPOINT'), + region: this.config.get('SNS_REGION'), + }) + this.clients.set(topicArn, snsClient) + } + return this.clients.get(topicArn) + } +} diff --git a/packages/core/src/queue/sns.service.ts b/packages/core/src/queue/sns.service.ts index cfdab27..7ab251c 100644 --- a/packages/core/src/queue/sns.service.ts +++ b/packages/core/src/queue/sns.service.ts @@ -1,33 +1,33 @@ -import { PublishCommand, SNSClient } from '@aws-sdk/client-sns' +import { PublishCommand } from '@aws-sdk/client-sns' import { Injectable } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { SnsEvent } from './sns.event' - -const CLIENT_INSTANCE = Symbol('sns-client') +import { SnsClientFactory } from './sns-client-factory' @Injectable() export class SnsService { - private [CLIENT_INSTANCE]: SNSClient - private topicArn: string - - constructor(private readonly config: ConfigService) { - this[CLIENT_INSTANCE] = new SNSClient({ - endpoint: config.get('SNS_ENDPOINT'), - region: config.get('SNS_REGION'), - }) + private readonly defaultTopicArn: string - this.topicArn = config.get('SNS_TOPIC_ARN') + constructor( + private readonly snsClientFactory: SnsClientFactory, + private readonly config: ConfigService, + ) { + this.defaultTopicArn = this.config.get('SNS_TOPIC_ARN') } - get client(): SNSClient { - return this[CLIENT_INSTANCE] - } + publish(msg: T, topicArn?: string) { + const resolvedTopicArn = topicArn || this.defaultTopicArn + + if (!resolvedTopicArn) { + throw new Error('No topic ARN provided or configured as default.') + } + + const client = this.snsClientFactory.getClient(resolvedTopicArn) - publish(msg: T) { - return this.client.send( + return client.send( new PublishCommand({ - TopicArn: this.topicArn, + TopicArn: resolvedTopicArn, MessageAttributes: { action: { DataType: 'String', diff --git a/packages/core/test/infra-local/elasticmq.conf b/packages/core/test/infra-local/elasticmq.conf index 2095451..c74f59e 100755 --- a/packages/core/test/infra-local/elasticmq.conf +++ b/packages/core/test/infra-local/elasticmq.conf @@ -15,6 +15,11 @@ queues { delay = 5 seconds receiveMessageWait = 0 seconds } + alarm-queue { + defaultVisibilityTimeout = 60 seconds + delay = 5 seconds + receiveMessageWait = 0 seconds + } } aws { diff --git a/packages/core/test/infra-local/serverless.yml b/packages/core/test/infra-local/serverless.yml index f5b59c6..109cfdf 100755 --- a/packages/core/test/infra-local/serverless.yml +++ b/packages/core/test/infra-local/serverless.yml @@ -40,6 +40,10 @@ custom: ['notification-action', 'command-status', 'task-status'], } queue: http://localhost:9324/101010101010/notification-queue + - topic: + topicName: AlarmSnsTopic + rawMessageDelivery: 'true' + queue: http://localhost:9324/101010101010/alarm-queue # host: 0.0.0.0 # Optional, defaults to 127.0.0.1 if not provided to serverless-offline # sns-endpoint: http://127.0.0.1:4567 # Optional. Only if you want to use a custom SNS provider endpoint # sns-subscribe-endpoint: http://127.0.0.1:3000 # Optional. Only if you want to use a custom subscribe endpoint from SNS to send messages back to @@ -93,6 +97,7 @@ provider: region: ap-northeast-1 runtime: nodejs18.x profile: serverless + timeout: 100 httpApi: authorizers: localAuthorizer: diff --git a/packages/core/test/src/testing/handler/test-sync.handler.ts b/packages/core/test/src/testing/handler/test-sync.handler.ts index 57cbbf9..58dbe73 100644 --- a/packages/core/test/src/testing/handler/test-sync.handler.ts +++ b/packages/core/test/src/testing/handler/test-sync.handler.ts @@ -7,9 +7,9 @@ export class TestSyncHandler implements IDataSyncHandler { private readonly logger = new Logger(TestSyncHandler.name) async up(cmd: CommandModel): Promise { - this.logger.log('TestSyncHandler.up', cmd) + this.logger.debug('TestSyncHandler.up', cmd) } async down(cmd: CommandModel): Promise { - this.logger.log('TestSyncHandler.down', cmd) + this.logger.debug('TestSyncHandler.down', cmd) } } diff --git a/packages/sequence/src/sequences.service.ts b/packages/sequence/src/sequences.service.ts index 9744f1b..b58c8d9 100644 --- a/packages/sequence/src/sequences.service.ts +++ b/packages/sequence/src/sequences.service.ts @@ -121,8 +121,8 @@ export class SequencesService implements ISequenceService { const generalMasterPk = masterPk(tenantCode) const generalMasterSk = `SEQ${KEY_SEPARATOR}${typeCode}` - this.logger.log('general master pk: ', generalMasterPk) - this.logger.log('general master sk: ', generalMasterSk) + this.logger.debug('general master pk: ', generalMasterPk) + this.logger.debug('general master sk: ', generalMasterSk) const masterData = await this.masterDataProvider.getData({ pk: generalMasterPk, sk: generalMasterSk, @@ -288,7 +288,7 @@ export class SequencesService implements ISequenceService { const effectiveStartMonth = registerTime ? registerTime.getMonth() + 1 - : startMonth ?? 4 + : (startMonth ?? 4) const referenceYear = registerTime ? registerTime.getFullYear() : 1953 // Reference year // Determine the current fiscal year