From ba6fe38347775dddd091df2fe8b5a27e83006bd6 Mon Sep 17 00:00:00 2001 From: koichimurakami Date: Fri, 6 Dec 2024 09:00:12 +0000 Subject: [PATCH 1/8] v0.1.43-beta.0 --- lerna.json | 2 +- package-lock.json | 16 ++++++++-------- packages/cli/package.json | 2 +- packages/core/package.json | 2 +- packages/sequence/package.json | 4 ++-- packages/task/package.json | 4 ++-- packages/ui-setting/package.json | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lerna.json b/lerna.json index 5ea66a7..cd5c3e9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "packages": ["packages/*"] } diff --git a/package-lock.json b/package-lock.json index 9ff09cf..b2bbcd1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28268,7 +28268,7 @@ }, "packages/cli": { "name": "@mbc-cqrs-serverless/cli", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "license": "MIT", "dependencies": { "commander": "^11.1.0", @@ -28292,7 +28292,7 @@ }, "packages/core": { "name": "@mbc-cqrs-serverless/core", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", @@ -28340,26 +28340,26 @@ }, "packages/sequence": { "name": "@mbc-cqrs-serverless/sequence", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "license": "MIT", "dependencies": { - "@mbc-cqrs-serverless/core": "^0.1.42-beta.0" + "@mbc-cqrs-serverless/core": "^0.1.43-beta.0" } }, "packages/task": { "name": "@mbc-cqrs-serverless/task", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "license": "MIT", "dependencies": { - "@mbc-cqrs-serverless/core": "^0.1.42-beta.0" + "@mbc-cqrs-serverless/core": "^0.1.43-beta.0" } }, "packages/ui-setting": { "name": "@mbc-cqrs-serverless/ui-setting", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "license": "MIT", "dependencies": { - "@mbc-cqrs-serverless/core": "^0.1.42-beta.0" + "@mbc-cqrs-serverless/core": "^0.1.43-beta.0" } } } diff --git a/packages/cli/package.json b/packages/cli/package.json index 9fa384f..466921f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@mbc-cqrs-serverless/cli", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "description": "a CLI to get started with MBC CQRS serverless framework", "keywords": [ "mbc", diff --git a/packages/core/package.json b/packages/core/package.json index bb34cec..b329af5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@mbc-cqrs-serverless/core", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "description": "CQRS and event base core", "keywords": [ "mbc", diff --git a/packages/sequence/package.json b/packages/sequence/package.json index 2791199..d301cbf 100644 --- a/packages/sequence/package.json +++ b/packages/sequence/package.json @@ -1,6 +1,6 @@ { "name": "@mbc-cqrs-serverless/sequence", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "description": "Generate increment sequence with time-rotation", "keywords": [ "mbc", @@ -41,6 +41,6 @@ "access": "public" }, "dependencies": { - "@mbc-cqrs-serverless/core": "^0.1.42-beta.0" + "@mbc-cqrs-serverless/core": "^0.1.43-beta.0" } } diff --git a/packages/task/package.json b/packages/task/package.json index 80b47f6..38f4829 100644 --- a/packages/task/package.json +++ b/packages/task/package.json @@ -1,6 +1,6 @@ { "name": "@mbc-cqrs-serverless/task", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "description": "long-running task", "keywords": [ "mbc", @@ -41,6 +41,6 @@ "access": "public" }, "dependencies": { - "@mbc-cqrs-serverless/core": "^0.1.42-beta.0" + "@mbc-cqrs-serverless/core": "^0.1.43-beta.0" } } diff --git a/packages/ui-setting/package.json b/packages/ui-setting/package.json index c669079..19caa3e 100644 --- a/packages/ui-setting/package.json +++ b/packages/ui-setting/package.json @@ -1,6 +1,6 @@ { "name": "@mbc-cqrs-serverless/ui-setting", - "version": "0.1.42-beta.0", + "version": "0.1.43-beta.0", "description": "Setting master data", "keywords": [ "mbc", @@ -41,6 +41,6 @@ "access": "public" }, "dependencies": { - "@mbc-cqrs-serverless/core": "^0.1.42-beta.0" + "@mbc-cqrs-serverless/core": "^0.1.43-beta.0" } } From ffc666c3db78d97df767ac23a5c13956c2000b77 Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Mon, 16 Dec 2024 15:07:12 +0700 Subject: [PATCH 2/8] the retention period for each table --- .../core/src/command-events/sfn-name.enum.ts | 1 + .../src/commands/command.event.handler.ts | 18 +++++ packages/core/src/commands/command.module.ts | 10 ++- packages/core/src/commands/command.service.ts | 56 +++++++++++++- packages/core/src/commands/data.service.ts | 3 +- packages/core/src/commands/enums/index.ts | 1 + .../core/src/commands/enums/tables.enum.ts | 5 ++ packages/core/src/commands/history.service.ts | 12 ++- packages/core/src/commands/ttl.service.ts | 75 +++++++++++++++++++ .../core/src/data-store/dynamodb.service.ts | 6 +- packages/core/src/helpers/key.ts | 20 ++++- .../interfaces/command-service.interface.ts | 9 +++ packages/core/src/interfaces/index.ts | 1 + .../src/interfaces/ttl-service.interface.ts | 42 +++++++++++ .../sequence/src/sequence-master-factory.ts | 3 +- 15 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 packages/core/src/commands/enums/tables.enum.ts create mode 100644 packages/core/src/commands/ttl.service.ts create mode 100644 packages/core/src/interfaces/ttl-service.interface.ts diff --git a/packages/core/src/command-events/sfn-name.enum.ts b/packages/core/src/command-events/sfn-name.enum.ts index c953417..b085965 100644 --- a/packages/core/src/command-events/sfn-name.enum.ts +++ b/packages/core/src/command-events/sfn-name.enum.ts @@ -1,6 +1,7 @@ export enum DataSyncCommandSfnName { CHECK_VERSION = 'check_version', WAIT_PREV_COMMAND = 'wait_prev_command', + SET_TTL_COMMAND = 'set_ttl_command', HISTORY_COPY = 'history_copy', TRANSFORM_DATA = 'transform_data', SYNC_DATA = 'sync_data', diff --git a/packages/core/src/commands/command.event.handler.ts b/packages/core/src/commands/command.event.handler.ts index bb3172a..0f01b6e 100644 --- a/packages/core/src/commands/command.event.handler.ts +++ b/packages/core/src/commands/command.event.handler.ts @@ -69,6 +69,9 @@ export class CommandEventHandler { case DataSyncCommandSfnName.WAIT_PREV_COMMAND: return await this.waitConfirmToken(event) + case DataSyncCommandSfnName.SET_TTL_COMMAND: + return await this.setTtlCommand(event) + case DataSyncCommandSfnName.HISTORY_COPY: return await this.historyCopy(event) @@ -130,6 +133,21 @@ export class CommandEventHandler { } } + protected async setTtlCommand( + event: DataSyncCommandSfnEvent, + ): Promise { + this.logger.debug('setTtlCommand:: ', event.commandRecord) + + await this.commandService.publishItem({ + pk: event.commandRecord.pk, + sk: event.commandRecord.sk, + }) + + return { + result: 'ok', + } + } + protected async historyCopy( event: DataSyncCommandSfnEvent, ): Promise { diff --git a/packages/core/src/commands/command.module.ts b/packages/core/src/commands/command.module.ts index a199d92..802dccb 100644 --- a/packages/core/src/commands/command.module.ts +++ b/packages/core/src/commands/command.module.ts @@ -10,6 +10,7 @@ import { CommandService } from './command.service' import { DataService } from './data.service' import { DataSyncDdsHandler } from './handlers/data-sync-dds.handler' import { HistoryService } from './history.service' +import { TtlService } from './ttl.service' @Module({ imports: [], @@ -17,11 +18,18 @@ import { HistoryService } from './history.service' ExplorerService, CommandService, DataService, + TtlService, HistoryService, CommandEventHandler, DataSyncDdsHandler, ], - exports: [CommandService, DataService, HistoryService, CommandEventHandler], + exports: [ + CommandService, + DataService, + HistoryService, + CommandEventHandler, + TtlService, + ], }) export class CommandModule extends ConfigurableModuleClass { static register(options: typeof OPTIONS_TYPE): DynamicModule { diff --git a/packages/core/src/commands/command.service.ts b/packages/core/src/commands/command.service.ts index 1b77269..815542e 100644 --- a/packages/core/src/commands/command.service.ts +++ b/packages/core/src/commands/command.service.ts @@ -14,7 +14,12 @@ import { getUserContext } from '../context/user' import { DynamoDbService } from '../data-store/dynamodb.service' import { DATA_SYNC_HANDLER_METADATA } from '../decorators' import { mergeDeep, pickKeys } from '../helpers' -import { addSortKeyVersion, removeSortKeyVersion } from '../helpers/key' +import { + addSortKeyVersion, + getSortKeyVersion, + getTenantCode, + removeSortKeyVersion, +} from '../helpers/key' import { CommandInputModel, CommandModel, @@ -30,7 +35,9 @@ import { SnsService } from '../queue/sns.service' import { ExplorerService } from '../services' import { MODULE_OPTIONS_TOKEN } from './command.module-definition' import { DataService } from './data.service' +import { TableType } from './enums' import { DataSyncDdsHandler } from './handlers/data-sync-dds.handler' +import { TtlService } from './ttl.service' const TABLE_NAME = Symbol('command') const DATA_SYNC_HANDLER = Symbol(DATA_SYNC_HANDLER_METADATA) @@ -52,10 +59,11 @@ export class CommandService implements OnModuleInit, ICommandService { private readonly snsService: SnsService, private readonly dataSyncDdsHandler: DataSyncDdsHandler, private readonly dataService: DataService, + private readonly ttlService: TtlService, ) { this.tableName = this.dynamoDbService.getTableName( this.options.tableName, - 'command', + TableType.COMMAND, ) this.logger = new Logger(`${CommandService.name}:${this.tableName}`) } @@ -122,6 +130,9 @@ export class CommandService implements OnModuleInit, ICommandService { 'The input is not a valid, item not found or version not match', ) } + if (!Object.keys(input).includes('ttl')) { + delete item.ttl + } const fullInput = mergeDeep({}, item, input, { version: item.version }) this.logger.debug('publishPartialUpdateSync::', fullInput) @@ -149,6 +160,9 @@ export class CommandService implements OnModuleInit, ICommandService { 'The input key is not a valid, item not found', ) } + if (!Object.keys(input).includes('ttl')) { + delete item.ttl + } const fullInput = mergeDeep({}, item, input, { version: item.version }) this.logger.debug('publishPartialUpdate::', fullInput) @@ -180,6 +194,9 @@ export class CommandService implements OnModuleInit, ICommandService { 'The input key is not a valid, item not found', ) } + if (!Object.keys(input).includes('ttl')) { + delete item.ttl + } const fullInput = mergeDeep({}, item, input, { version: item.version }) this.logger.debug('publishPartialUpdate::', fullInput) @@ -210,6 +227,10 @@ export class CommandService implements OnModuleInit, ICommandService { const version = (item?.version ?? inputVersion) + 1 const command: CommandModel = { + ttl: await this.ttlService.calculateTtl( + TableType.DATA, + getTenantCode(input.pk), + ), ...input, version, source: options?.source, @@ -271,6 +292,10 @@ export class CommandService implements OnModuleInit, ICommandService { const version = inputVersion + 1 const command: CommandModel = { + ttl: await this.ttlService.calculateTtl( + TableType.DATA, + getTenantCode(input.pk), + ), ...input, sk: addSortKeyVersion(input.sk, version), version, @@ -433,4 +458,31 @@ export class CommandService implements OnModuleInit, ICommandService { structuredClone(pickKeys(input, comparedKeys)), ) } + + async publishItem(key: DetailKey) { + const version = getSortKeyVersion(key.sk) + const sk = removeSortKeyVersion(key.sk) + if (version <= VERSION_FIRST + 1) { + return null + } + + const previousSk = addSortKeyVersion(sk, version - 1) + + const command = await this.dynamoDbService.getItem(this.tableName, { + pk: key.pk, + sk: previousSk, + }) + if (!command) { + return null + } + command.sk = previousSk + const ttl = await this.ttlService.calculateTtl( + TableType.COMMAND, + getTenantCode(key.pk), + ) + command.ttl = ttl + + this.logger.debug('publishItem::', command) + return await this.dynamoDbService.putItem(this.tableName, command) + } } diff --git a/packages/core/src/commands/data.service.ts b/packages/core/src/commands/data.service.ts index 884b8b9..75b4c06 100644 --- a/packages/core/src/commands/data.service.ts +++ b/packages/core/src/commands/data.service.ts @@ -11,6 +11,7 @@ import { DetailKey, } from '../interfaces' import { MODULE_OPTIONS_TOKEN } from './command.module-definition' +import { TableType } from './enums' const TABLE_NAME = Symbol('data') @@ -26,7 +27,7 @@ export class DataService { ) { this.tableName = this.dynamoDbService.getTableName( this.options.tableName, - 'data', + TableType.DATA, ) this.logger = new Logger(`${DataService.name}:${this.tableName}`) } diff --git a/packages/core/src/commands/enums/index.ts b/packages/core/src/commands/enums/index.ts index 59f2d98..1a81e70 100644 --- a/packages/core/src/commands/enums/index.ts +++ b/packages/core/src/commands/enums/index.ts @@ -1 +1,2 @@ export * from './status.enum' +export * from './tables.enum' diff --git a/packages/core/src/commands/enums/tables.enum.ts b/packages/core/src/commands/enums/tables.enum.ts new file mode 100644 index 0000000..05c2b44 --- /dev/null +++ b/packages/core/src/commands/enums/tables.enum.ts @@ -0,0 +1,5 @@ +export enum TableType { + COMMAND = 'command', + HISTORY = 'history', + DATA = 'data', +} diff --git a/packages/core/src/commands/history.service.ts b/packages/core/src/commands/history.service.ts index 67cccbe..0eabecf 100644 --- a/packages/core/src/commands/history.service.ts +++ b/packages/core/src/commands/history.service.ts @@ -1,10 +1,12 @@ import { Inject, Injectable, Logger } from '@nestjs/common' import { DynamoDbService } from '../data-store/dynamodb.service' -import { addSortKeyVersion } from '../helpers' +import { addSortKeyVersion, getTenantCode } from '../helpers' import { CommandModuleOptions, DataModel, DetailKey } from '../interfaces' import { MODULE_OPTIONS_TOKEN } from './command.module-definition' import { DataService } from './data.service' +import { TableType } from './enums' +import { TtlService } from './ttl.service' const TABLE_NAME = Symbol('history') @@ -18,10 +20,11 @@ export class HistoryService { private readonly options: CommandModuleOptions, private readonly dynamoDbService: DynamoDbService, private readonly dataService: DataService, + private readonly ttlService: TtlService, ) { this.tableName = this.dynamoDbService.getTableName( this.options.tableName, - 'history', + TableType.HISTORY, ) this.logger = new Logger(`${HistoryService.name}:${this.tableName}`) } @@ -40,6 +43,11 @@ export class HistoryService { return null } data.sk = addSortKeyVersion(data.sk, data.version) + const ttl = await this.ttlService.calculateTtl( + TableType.HISTORY, + getTenantCode(key.pk), + ) + data.ttl = ttl this.logger.debug('publish::', data) return await this.dynamoDbService.putItem(this.tableName, data) diff --git a/packages/core/src/commands/ttl.service.ts b/packages/core/src/commands/ttl.service.ts new file mode 100644 index 0000000..09a9feb --- /dev/null +++ b/packages/core/src/commands/ttl.service.ts @@ -0,0 +1,75 @@ +import { Inject, Injectable, Logger } from '@nestjs/common' + +import { DynamoDbService } from '../data-store/dynamodb.service' +import { masterPk, ttlSk } from '../helpers' +import { CommandModuleOptions, ITtlService } from '../interfaces' +import { MODULE_OPTIONS_TOKEN } from './command.module-definition' +import { TableType } from './enums' + +@Injectable() +export class TtlService implements ITtlService { + private logger: Logger + + constructor( + @Inject(MODULE_OPTIONS_TOKEN) + private readonly options: CommandModuleOptions, + private readonly dynamoDbService: DynamoDbService, + ) { + this.logger = new Logger(`${TtlService.name}:${this.options.tableName}`) + } + + async calculateTtl( + type: TableType, + tenantCode?: string, + startDate?: Date, + ): Promise { + const numberOfDay = await this.getTtlConfiguration(type, tenantCode) + this.logger.log('numberOfDay', numberOfDay) + return numberOfDay ? this.calculateUnixTime(numberOfDay, startDate) : null + } + + async getTtlConfiguration( + type: TableType, + tenantCode?: string, + ): Promise { + try { + const masterDataTableName = this.dynamoDbService.getTableName( + 'master', + TableType.DATA, + ) + + const pk = masterPk(tenantCode) + const sk = ttlSk( + this.dynamoDbService.getTableName(this.options.tableName, type), + ) + + const ttlData = await this.dynamoDbService.getItem(masterDataTableName, { + pk, + sk, + }) + + const numberOfDay = ttlData?.attributes?.days // pk | sk | attributes {days: 90} + + return numberOfDay ? +numberOfDay : null + } catch (error) { + this.logger.error('error', error) + return null + } + } + + calculateUnixTime(days: number, startDate?: Date): number { + if (days <= 0) { + throw new Error('Number of days must be greater than 0.') + } + + // Determine the base time in seconds + const baseTimeInSeconds = startDate + ? Math.floor(startDate.getTime() / 1000) + : Math.floor(Date.now() / 1000) + + // Convert days to seconds and add to the base time + const result = baseTimeInSeconds + days * 24 * 60 * 60 + + return result + } +} diff --git a/packages/core/src/data-store/dynamodb.service.ts b/packages/core/src/data-store/dynamodb.service.ts index 78f57df..ac4ab71 100644 --- a/packages/core/src/data-store/dynamodb.service.ts +++ b/packages/core/src/data-store/dynamodb.service.ts @@ -12,6 +12,7 @@ import { Injectable, Logger } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { ulid } from 'ulid' +import { TableType } from '../commands' import { isS3AttributeKey, parseS3AttributeKey, @@ -427,10 +428,7 @@ export class DynamoDbService { return ret } - getTableName( - moduleName: string, - type?: 'command' | 'data' | 'history', - ): string { + getTableName(moduleName: string, type?: TableType): string { let tableName = this.tablePrefix + moduleName if (type) { tableName = `${tableName}-${type}` diff --git a/packages/core/src/helpers/key.ts b/packages/core/src/helpers/key.ts index eb40484..e2497ee 100644 --- a/packages/core/src/helpers/key.ts +++ b/packages/core/src/helpers/key.ts @@ -1,10 +1,18 @@ import { DEFAULT_TENANT_CODE } from '../constants' -import { KEY_SEPARATOR, VER_SEPARATOR } from '../constants/key' +import { KEY_SEPARATOR, VER_SEPARATOR, VERSION_LATEST } from '../constants/key' export function addSortKeyVersion(sk: string, version: number) { return `${removeSortKeyVersion(sk)}${VER_SEPARATOR}${version}` } +export function getSortKeyVersion(sk: string) { + const lastDivIdx = sk.lastIndexOf(VER_SEPARATOR) + if (lastDivIdx === -1) { + return VERSION_LATEST + } + return +sk.substring(lastDivIdx + 1) +} + export function removeSortKeyVersion(sk: string) { const lastDivIdx = sk.lastIndexOf(VER_SEPARATOR) if (lastDivIdx === -1) { @@ -17,6 +25,14 @@ export function generateId(pk: string, sk: string) { return `${pk}${KEY_SEPARATOR}${removeSortKeyVersion(sk)}` } +export function getTenantCode(pk: string) { + const lastDivIdx = pk.lastIndexOf(KEY_SEPARATOR) + if (lastDivIdx === -1) { + return + } + return pk.substring(lastDivIdx + 1) +} + const S3_PREFIX = 's3://' const S3_PREFIX_LEN = S3_PREFIX.length @@ -47,3 +63,5 @@ export const masterPk = (tenantCode?: string) => export const seqPk = (tenantCode?: string) => `SEQ${KEY_SEPARATOR}${tenantCode || DEFAULT_TENANT_CODE}` + +export const ttlSk = (tableName: string) => `TTL${KEY_SEPARATOR}${tableName}` diff --git a/packages/core/src/interfaces/command-service.interface.ts b/packages/core/src/interfaces/command-service.interface.ts index c32d8af..bca8b90 100644 --- a/packages/core/src/interfaces/command-service.interface.ts +++ b/packages/core/src/interfaces/command-service.interface.ts @@ -2,6 +2,7 @@ import { CommandInputModel, CommandModel, CommandPartialInputModel, + DetailKey, ICommandOptions, } from '.' @@ -57,4 +58,12 @@ export interface ICommandService { input: CommandPartialInputModel, options: ICommandOptions, ): Promise + + /** + * Publishes an old command by updating its TTL. + * + * @param {DetailKey} key - The primary and sort key details for the item. + * @returns {Promise} A promise resolving to the result of the put operation, or `null` if the operation is not applicable. + */ + publishItem(key: DetailKey): Promise } diff --git a/packages/core/src/interfaces/index.ts b/packages/core/src/interfaces/index.ts index f7b96e5..6118a3e 100644 --- a/packages/core/src/interfaces/index.ts +++ b/packages/core/src/interfaces/index.ts @@ -22,3 +22,4 @@ export * from './master.interface' export * from './notification.interface' export * from './paginate.dto' export * from './search.dto' +export * from './ttl-service.interface' diff --git a/packages/core/src/interfaces/ttl-service.interface.ts b/packages/core/src/interfaces/ttl-service.interface.ts new file mode 100644 index 0000000..697bbf8 --- /dev/null +++ b/packages/core/src/interfaces/ttl-service.interface.ts @@ -0,0 +1,42 @@ +import { TableType } from '../commands' + +/** + * Interface defining the TTL service. + */ +export interface ITtlService { + /** + * Calculates the Unix timestamp TTL based on the given type, tenant code, and optional start date. + * + * @param {TableType} type - The table type. + * @param {string} [tenantCode] - (Optional) The tenant code. + * @param {Date} [startDate] - (Optional) The start date. Defaults to the current date if not provided. + * @returns {Promise} The TTL as a Unix timestamp or `null` if no configuration is found. + */ + calculateTtl( + type: TableType, + tenantCode?: string, + startDate?: Date, + ): Promise + + /** + * Retrieves the TTL configuration for the given table type and tenant code. + * + * @param {TableType} type - The table type. + * @param {string} [tenantCode] - (Optional) The tenant code. + * @returns {Promise} The number of days configured for TTL or `null` if no configuration is found. + */ + getTtlConfiguration( + type: TableType, + tenantCode?: string, + ): Promise + + /** + * Calculates the Unix timestamp for the given number of days from a start date. + * + * @param {number} days - The number of days to calculate the TTL. + * @param {Date} [startDate] - (Optional) The start date. Defaults to the current date if not provided. + * @returns {number} The TTL as a Unix timestamp. + * @throws {Error} An error if the number of days is less than or equal to zero. + */ + calculateUnixTime(days: number, startDate?: Date): number +} diff --git a/packages/sequence/src/sequence-master-factory.ts b/packages/sequence/src/sequence-master-factory.ts index 2a9893d..1b0ad65 100644 --- a/packages/sequence/src/sequence-master-factory.ts +++ b/packages/sequence/src/sequence-master-factory.ts @@ -2,6 +2,7 @@ import { DetailKey, DynamoDbService, IMasterDataProvider, + TableType, } from '@mbc-cqrs-serverless/core' import { Inject, Injectable, Optional } from '@nestjs/common' @@ -16,7 +17,7 @@ export class SequenceMasterDataProvider implements IMasterDataProvider { @Optional() private readonly defaultValue: Record, ) { - this.tableName = dynamoDbService.getTableName('master', 'data') + this.tableName = dynamoDbService.getTableName('master', TableType.DATA) } async getData(key: DetailKey): Promise { try { From 6342fcf85f88c5d8d63b6720ef7de4e846ce302e Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Mon, 16 Dec 2024 17:21:56 +0700 Subject: [PATCH 3/8] add tll test for sync process --- packages/core/.env.example | 1 + .../publish-partial-update-sync.e2e-spec.ts | 188 ++++++++++++++++++ .../core/test/e2e/publish-sync.e2e-spec.ts | 147 +++++++++++++- packages/core/test/infra-local/serverless.yml | 20 +- packages/core/test/infra-local/swagger.json | 13 ++ packages/core/test/prisma/ddb.ts | 2 - packages/core/test/prisma/dynamodbs/cqrs.json | 2 +- .../core/test/src/testing/test.controller.ts | 19 +- 8 files changed, 380 insertions(+), 12 deletions(-) create mode 100644 packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts diff --git a/packages/core/.env.example b/packages/core/.env.example index 65b4598..72c5f03 100644 --- a/packages/core/.env.example +++ b/packages/core/.env.example @@ -53,3 +53,4 @@ DATABASE_URL="mysql://root:RootCqrs@0.0.0.0:3306/cqrs?schema=public&connection_l # serverless dynamodb local stream LOCAL_DDB_TESTING_TABLE_STREAM=arn:aws:dynamodb:ddblocal:000000000000:table/local-demo-testing_table-command/stream/2024-09-20T07:06:05.837 +LOCAL_DDB_MASTER_STREAM=arn:aws:dynamodb:ddblocal:000000000000:table/local-demo-master-command/stream/2024-12-13T07:55:58.001 diff --git a/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts b/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts new file mode 100644 index 0000000..7e70b00 --- /dev/null +++ b/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts @@ -0,0 +1,188 @@ +import request from 'supertest' + +import { config } from './config' +import { + deleteItem, + getItem, + getTableName, + putItem, + TableType, +} from './dynamo-client' + +const API_PATH = '/api/testing/sync' + +describe('publishPartialUpdateSync', () => { + it('should be stored correct updated data in the data DDB table', async () => { + // Arrange + const payload = { + pk: 'TEST#MBC', + sk: 'publish-partial-update-sync#data', + id: 'TEST#MBC#publish-partial-update-sync#data', + name: 'testing#data', + version: 0, + code: 'publish-partial-update-sync#data', + type: 'TEST', + } + + const postRes = await request(config.apiBaseUrl) + .post(API_PATH) + .send(payload) + + expect(postRes.statusCode).toEqual(201) + + const updatePayload = { + pk: 'TEST#MBC', + sk: 'publish-partial-update-sync#data', + name: 'update name', + version: 1, + } + + // Action + const putRes = await request(config.apiBaseUrl) + .put(API_PATH) + .send(updatePayload) + + // Assert + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + + console.log('putRes', putRes.body) + + expect(data).toMatchObject({ + ...payload, + name: updatePayload.name, + version: 2, + }) + }, 40000) + + it('should store the latest data with configuration TTL in DDB (90 days)', async () => { + // Arrange + const ttlData = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-data', + id: 'MASTER#MBC#TTL#local-test-testing_table-data', + code: 'MASTER#MBC#TTL#local-test-testing_table-data', + type: 'TTL', + name: 'TTL#local-test-testing_table-data', + version: 0, + attributes: { + days: 90, + }, + } + const masterDataTableName = getTableName('master', TableType.DATA) + await putItem(masterDataTableName, ttlData) + const ttl = Math.floor( + (new Date().getTime() + 90 * 24 * 60 * 60 * 1000) / 1000, + ) // 90 days + + const payload = { + pk: 'TEST#MBC', + sk: 'publish-partial-update-sync#data90', + id: 'TEST#MBC#publish-partial-update-sync#data90', + name: 'testing#data', + version: 0, + code: 'publish-partial-update-sync#data90', + type: 'TEST', + } + + const postRes = await request(config.apiBaseUrl) + .post(API_PATH) + .send(payload) + + expect(postRes.statusCode).toEqual(201) + + const updatePayload = { + pk: payload.pk, + sk: payload.sk, + name: 'update name', + version: 1, + } + + // Action + const putRes = await request(config.apiBaseUrl) + .put(API_PATH) + .send(updatePayload) + + // Assert + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + + console.log('putRes', putRes.body) + + expect(data?.ttl >= ttl).toBeTruthy() + + deleteItem(masterDataTableName, { + pk: ttlData.pk, + sk: ttlData.sk, + }) + }, 40000) + + it('should store the latest data with input TTL', async () => { + // Arrange + const ttlData = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-data', + id: 'MASTER#MBC#TTL#local-test-testing_table-data', + code: 'MASTER#MBC#TTL#local-test-testing_table-data', + type: 'TTL', + name: 'TTL#local-test-testing_table-data', + version: 0, + attributes: { + days: 90, + }, + } + const masterDataTableName = getTableName('master', TableType.DATA) + await putItem(masterDataTableName, ttlData) + const ttl = Math.floor( + (new Date().getTime() + 30 * 24 * 60 * 60 * 1000) / 1000, + ) // 30 days + + const payload = { + pk: 'TEST#MBC', + sk: 'publish-partial-update-sync#data30', + id: 'TEST#MBC#publish-partial-update-sync#data30', + name: 'testing#data', + version: 0, + code: 'publish-partial-update-sync#data30', + type: 'TEST', + } + + const postRes = await request(config.apiBaseUrl) + .post(API_PATH) + .send(payload) + + expect(postRes.statusCode).toEqual(201) + + const updatePayload = { + pk: payload.pk, + sk: payload.sk, + name: 'update name', + version: 1, + ttl, + } + + // Action + const putRes = await request(config.apiBaseUrl) + .put(API_PATH) + .send(updatePayload) + + // Assert + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + + console.log('putRes', putRes.body) + + expect(data?.ttl == ttl).toBeTruthy() + + deleteItem(masterDataTableName, { + pk: ttlData.pk, + sk: ttlData.sk, + }) + }, 40000) +}) diff --git a/packages/core/test/e2e/publish-sync.e2e-spec.ts b/packages/core/test/e2e/publish-sync.e2e-spec.ts index 780b7e9..89f8cc5 100644 --- a/packages/core/test/e2e/publish-sync.e2e-spec.ts +++ b/packages/core/test/e2e/publish-sync.e2e-spec.ts @@ -1,7 +1,13 @@ import request from 'supertest' import { config } from './config' -import { getItem, getTableName, TableType } from './dynamo-client' +import { + deleteItem, + getItem, + getTableName, + putItem, + TableType, +} from './dynamo-client' const API_PATH = '/api/testing/sync' @@ -9,9 +15,9 @@ describe('publishSync', () => { it('should be stored correct data in the data DDB table', async () => { // Arrange const payload = { - pk: 'TEST', + pk: 'TEST#MBC', sk: 'publish-sync#data', - id: 'TEST#publish-sync#data', + id: 'TEST#MBC#publish-sync#data', name: 'testing#data', version: 0, code: 'publish-sync#data', @@ -29,8 +35,6 @@ describe('publishSync', () => { sk: payload.sk, }) - console.log('data', data) - expect(data).toMatchObject({ ...payload, version: 1, @@ -40,9 +44,9 @@ describe('publishSync', () => { it('should return invalid input version', async () => { // Arrange const payload = { - pk: 'TEST', + pk: 'TEST#MBC', sk: 'publish-sync_1', - id: 'TEST#publish-sync_1', + id: 'TEST#MBC#publish-sync_1', name: 'testing', version: 1, code: 'publish-sync_1', @@ -58,4 +62,133 @@ describe('publishSync', () => { 'Invalid input version. The input version must be equal to the latest version', ) }, 40000) + + it('should store the latest data with default TTL (null)', async () => { + // Arrange + const payload = { + pk: 'TEST#MBC', + sk: 'publish-sync#data1', + id: 'TEST#MBC#publish-sync#data1', + name: 'testing#ttl', + version: 0, + code: 'publish-sync#data1', + type: 'TEST', + } + + // Action + const res = await request(config.apiBaseUrl).post(API_PATH).send(payload) + + // Assert + expect(res.statusCode).toEqual(201) + + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + + expect(data).toMatchObject({ + ...payload, + ttl: null, + version: 1, + }) + }) + + it('should store the latest data with configuration TTL in DDB (90 days)', async () => { + // Arrange + const ttlData = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-data', + id: 'MASTER#MBC#TTL#local-test-testing_table-data', + code: 'MASTER#MBC#TTL#local-test-testing_table-data', + type: 'TTL', + name: 'TTL#local-test-testing_table-data', + version: 0, + attributes: { + days: 90, + }, + } + const masterDataTableName = getTableName('master', TableType.DATA) + await putItem(masterDataTableName, ttlData) + const ttl = Math.floor( + (new Date().getTime() + 90 * 24 * 60 * 60 * 1000) / 1000, + ) // 90 days + + const payload = { + pk: 'TEST#MBC', + sk: 'publish-sync#data90', + id: 'TEST#MBC#publish-sync#data90', + name: 'testing#ttl', + version: 0, + code: 'publish-sync#data90', + type: 'TEST', + } + + // Action + const res = await request(config.apiBaseUrl).post(API_PATH).send(payload) + + // Assert + expect(res.statusCode).toEqual(201) + + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + + expect(data?.ttl >= ttl).toBeTruthy() + + deleteItem(masterDataTableName, { + pk: ttlData.pk, + sk: ttlData.sk, + }) + }) + + it('should store the latest data with input TTL (30 days)', async () => { + // Arrange + const ttlData = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-data', + id: 'MASTER#MBC#TTL#local-test-testing_table-data', + code: 'MASTER#MBC#TTL#local-test-testing_table-data', + type: 'TTL', + name: 'TTL#local-test-testing_table-data', + version: 0, + attributes: { + days: 90, + }, + } + const masterDataTableName = getTableName('master', TableType.DATA) + await putItem(masterDataTableName, ttlData) + const ttl = Math.floor( + (new Date().getTime() + 30 * 24 * 60 * 60 * 1000) / 1000, + ) // 30 days + + const payload = { + pk: 'TEST#MBC', + sk: 'publish-sync#data-in-30', + id: 'TEST#MBC#publish-sync#data-in-30', + name: 'testing#ttl', + version: 0, + code: 'publish-sync#data-in-30', + type: 'TEST', + ttl, + } + + // Action + const res = await request(config.apiBaseUrl).post(API_PATH).send(payload) + + // Assert + expect(res.statusCode).toEqual(201) + + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + + expect(data?.ttt == ttl).toBeTruthy() + + deleteItem(masterDataTableName, { + pk: ttlData.pk, + sk: ttlData.sk, + }) + }) }) diff --git a/packages/core/test/infra-local/serverless.yml b/packages/core/test/infra-local/serverless.yml index 32fbaaa..f5b59c6 100755 --- a/packages/core/test/infra-local/serverless.yml +++ b/packages/core/test/infra-local/serverless.yml @@ -186,7 +186,7 @@ stepFunctions: Choices: - Variable: $.result NumericEquals: 0 - Next: history_copy + Next: set_ttl_command - Variable: $.result NumericEquals: 1 Next: wait_prev_command @@ -212,6 +212,24 @@ stepFunctions: MaxAttempts: 5 BackoffRate: 2 OutputPath: $.Payload[0][0] + Next: set_ttl_command + set_ttl_command: + Type: Task + Resource: arn:aws:states:::lambda:invoke + Parameters: + FunctionName: arn:aws:lambda:ap-northeast-1:101010101010:function:serverless-example-dev-main + Payload: + input.$: $ + context.$: $$ + Retry: + - ErrorEquals: + - Lambda.ServiceException + - Lambda.AWSLambdaException + - Lambda.SdkClientException + IntervalSeconds: 2 + MaxAttempts: 5 + BackoffRate: 2 + OutputPath: $.Payload[0][0] Next: history_copy history_copy: Type: Task diff --git a/packages/core/test/infra-local/swagger.json b/packages/core/test/infra-local/swagger.json index 1c2d153..c9fd9c9 100755 --- a/packages/core/test/infra-local/swagger.json +++ b/packages/core/test/infra-local/swagger.json @@ -60,6 +60,19 @@ }, "responses": { "201": { "description": "" } }, "tags": ["testing"] + }, + "put": { + "operationId": "TestController_publishPartialUpdateSync", + "parameters": [ + { + "in": "header", + "required": false, + "name": "x-tenant-code", + "schema": {} + } + ], + "responses": { "200": { "description": "" } }, + "tags": ["testing"] } } }, diff --git a/packages/core/test/prisma/ddb.ts b/packages/core/test/prisma/ddb.ts index 1851720..740be1b 100644 --- a/packages/core/test/prisma/ddb.ts +++ b/packages/core/test/prisma/ddb.ts @@ -27,8 +27,6 @@ const client = new DynamoDBClient({ region: process.env.DYNAMODB_REGION, }) -console.log('process.env', process.env) - console.log('tablePrefix', tablePrefix) main().catch((e) => { diff --git a/packages/core/test/prisma/dynamodbs/cqrs.json b/packages/core/test/prisma/dynamodbs/cqrs.json index a2c3465..c97b7a0 100644 --- a/packages/core/test/prisma/dynamodbs/cqrs.json +++ b/packages/core/test/prisma/dynamodbs/cqrs.json @@ -1 +1 @@ -["testing_table"] +["testing_table", "master"] diff --git a/packages/core/test/src/testing/test.controller.ts b/packages/core/test/src/testing/test.controller.ts index 90c02a5..47f506c 100644 --- a/packages/core/test/src/testing/test.controller.ts +++ b/packages/core/test/src/testing/test.controller.ts @@ -1,8 +1,9 @@ -import { Body, Controller, Logger, Post } from '@nestjs/common' +import { Body, Controller, Logger, Post, Put } from '@nestjs/common' import { ApiTags } from '@nestjs/swagger' import { CommandDto, + CommandPartialInputModel, CommandService, IInvoke, INVOKE_CONTEXT, @@ -40,4 +41,20 @@ export class TestController { }) return item } + + @Put('/sync') + async publishPartialUpdateSync( + @INVOKE_CONTEXT() invokeContext: IInvoke, + @Body() commandDto: CommandPartialInputModel, + ) { + this.logger.debug('cmd:', commandDto) + this.logger.debug('commandService:' + this.commandService.tableName) + const item = await this.commandService.publishPartialUpdateSync( + commandDto, + { + invokeContext, + }, + ) + return item + } } From bb4bdd35dd06e9c33275c5b78d30f364cdcf69f4 Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Tue, 17 Dec 2024 14:38:02 +0700 Subject: [PATCH 4/8] add tll test for async process --- .../core/test/e2e/publish-async.e2e-spec.ts | 165 +++++++++++++++++- .../publish-partial-update-sync.e2e-spec.ts | 4 +- .../core/test/e2e/publish-sync.e2e-spec.ts | 7 +- packages/core/test/infra-local/swagger.json | 13 ++ .../core/test/src/testing/test.controller.ts | 16 ++ 5 files changed, 199 insertions(+), 6 deletions(-) diff --git a/packages/core/test/e2e/publish-async.e2e-spec.ts b/packages/core/test/e2e/publish-async.e2e-spec.ts index 604b214..59000dc 100644 --- a/packages/core/test/e2e/publish-async.e2e-spec.ts +++ b/packages/core/test/e2e/publish-async.e2e-spec.ts @@ -1,7 +1,13 @@ import request from 'supertest' import { config } from './config' -import { getItem, getTableName, TableType } from './dynamo-client' +import { + deleteItem, + getItem, + getTableName, + putItem, + TableType, +} from './dynamo-client' import { syncDataFinished } from './utils' const API_PATH = '/api/testing' @@ -161,4 +167,161 @@ describe('Publish', () => { 'ConditionalCheckFailedException: The conditional request failed', ) }, 80000) + + it('should store command, data, history with configuration TTL in ddb', async () => { + // Arrange + const ttlCommand = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-command', + id: 'MASTER#MBC#TTL#local-test-testing_table-command', + code: 'MASTER#MBC#TTL#local-test-testing_table-command', + type: 'TTL', + name: 'TTL#local-test-testing_table-command', + version: 0, + attributes: { + days: 30, + }, + } + const ttlHistory = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-history', + id: 'MASTER#MBC#TTL#local-test-testing_table-history', + code: 'MASTER#MBC#TTL#local-test-testing_table-history', + type: 'TTL', + name: 'TTL#local-test-testing_table-history', + version: 0, + attributes: { + days: 45, + }, + } + const ttlData = { + pk: 'MASTER#MBC', + sk: 'TTL#local-test-testing_table-data', + id: 'MASTER#MBC#TTL#local-test-testing_table-data', + code: 'MASTER#MBC#TTL#local-test-testing_table-data', + type: 'TTL', + name: 'TTL#local-test-testing_table-data', + version: 0, + attributes: { + days: 90, + }, + } + + const masterDataTableName = getTableName('master', TableType.DATA) + + await Promise.all([ + putItem(masterDataTableName, ttlCommand), + putItem(masterDataTableName, ttlHistory), + putItem(masterDataTableName, ttlData), + ]) + + // TODO: + const payload = { + pk: 'TEST#MBC', + sk: 'publish-partial-update-async#data', + id: 'TEST#MBC#publish-partial-update-async#data', + name: 'testing#data', + version: 0, + code: 'publish-partial-update-async#data', + type: 'TEST', + } + + const postRes = await request(config.apiBaseUrl) + .post(API_PATH) + .send(payload) + console.log('$#@!$postRes$@#', postRes.body) + expect(postRes.statusCode).toEqual(201) + await syncDataFinished('testing_table', { + pk: payload.pk, + sk: `${payload.sk}@1`, + }) + const validTtl = { + command: Math.floor( + (new Date().getTime() + 30 * 24 * 60 * 60 * 1000) / 1000, + ), // 30 days + history: Math.floor( + (new Date().getTime() + 45 * 24 * 60 * 60 * 1000) / 1000, + ), // 45 days + data: Math.floor( + (new Date().getTime() + 90 * 24 * 60 * 60 * 1000) / 1000, + ), // 90 days + } + + const updatePayload = { + pk: payload.pk, + sk: payload.sk, + name: 'update name', + version: 1, + } + + // Action + const putRes = await request(config.apiBaseUrl) + .put(API_PATH) + .send(updatePayload) + await syncDataFinished('testing_table', { + pk: payload.pk, + sk: `${payload.sk}@2`, + }) + console.log('$@#$@#$putRes$@#$@#', putRes.body) + + // Assert + const data = await getItem(getTableName('testing_table', TableType.DATA), { + pk: payload.pk, + sk: payload.sk, + }) + console.log('🚀 ~ data ~ data:', data) + + expect(data?.ttl >= validTtl.data).toBeTruthy() + + const newestCommand = await getItem( + getTableName('testing_table', TableType.COMMAND), + { + pk: payload.pk, + sk: `${payload.sk}@2`, + }, + ) + + console.log('🚀 ~ newestCommand ~ newestCommand:', newestCommand) + + expect(newestCommand?.ttl >= validTtl.data).toBeTruthy() + + const oldCommand = await getItem( + getTableName('testing_table', TableType.COMMAND), + { + pk: payload.pk, + sk: `${payload.sk}@1`, + }, + ) + + console.log('🚀 ~ oldCommand ~ oldCommand:', oldCommand) + + expect(oldCommand?.ttl >= validTtl.command).toBeTruthy() + + const history = await getItem( + getTableName('testing_table', TableType.HISTORY), + { + pk: payload.pk, + sk: `${payload.sk}@1`, + }, + ) + + console.log('🚀 ~ history ~ history:', history) + + expect(history?.ttl >= validTtl.history).toBeTruthy() + + await Promise.all([ + deleteItem(masterDataTableName, { + pk: ttlData.pk, + sk: ttlData.sk, + }), + deleteItem(masterDataTableName, { + pk: ttlHistory.pk, + sk: ttlHistory.sk, + }), + deleteItem(masterDataTableName, { + pk: ttlHistory.pk, + sk: ttlHistory.sk, + }), + ]) + }, 150000) }) diff --git a/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts b/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts index 7e70b00..6cdb16d 100644 --- a/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts +++ b/packages/core/test/e2e/publish-partial-update-sync.e2e-spec.ts @@ -115,7 +115,7 @@ describe('publishPartialUpdateSync', () => { expect(data?.ttl >= ttl).toBeTruthy() - deleteItem(masterDataTableName, { + await deleteItem(masterDataTableName, { pk: ttlData.pk, sk: ttlData.sk, }) @@ -180,7 +180,7 @@ describe('publishPartialUpdateSync', () => { expect(data?.ttl == ttl).toBeTruthy() - deleteItem(masterDataTableName, { + await deleteItem(masterDataTableName, { pk: ttlData.pk, sk: ttlData.sk, }) diff --git a/packages/core/test/e2e/publish-sync.e2e-spec.ts b/packages/core/test/e2e/publish-sync.e2e-spec.ts index 89f8cc5..0f0539c 100644 --- a/packages/core/test/e2e/publish-sync.e2e-spec.ts +++ b/packages/core/test/e2e/publish-sync.e2e-spec.ts @@ -136,7 +136,7 @@ describe('publishSync', () => { expect(data?.ttl >= ttl).toBeTruthy() - deleteItem(masterDataTableName, { + await deleteItem(masterDataTableName, { pk: ttlData.pk, sk: ttlData.sk, }) @@ -183,10 +183,11 @@ describe('publishSync', () => { pk: payload.pk, sk: payload.sk, }) + console.log('🚀 ~ data ~ data:', data) - expect(data?.ttt == ttl).toBeTruthy() + expect(data?.ttl == ttl).toBeTruthy() - deleteItem(masterDataTableName, { + await deleteItem(masterDataTableName, { pk: ttlData.pk, sk: ttlData.sk, }) diff --git a/packages/core/test/infra-local/swagger.json b/packages/core/test/infra-local/swagger.json index c9fd9c9..e3c4b34 100755 --- a/packages/core/test/infra-local/swagger.json +++ b/packages/core/test/infra-local/swagger.json @@ -37,6 +37,19 @@ }, "responses": { "201": { "description": "" } }, "tags": ["testing"] + }, + "put": { + "operationId": "TestController_publishPartialUpdateAsync", + "parameters": [ + { + "in": "header", + "required": false, + "name": "x-tenant-code", + "schema": {} + } + ], + "responses": { "200": { "description": "" } }, + "tags": ["testing"] } }, "/api/testing/sync": { diff --git a/packages/core/test/src/testing/test.controller.ts b/packages/core/test/src/testing/test.controller.ts index 47f506c..509416c 100644 --- a/packages/core/test/src/testing/test.controller.ts +++ b/packages/core/test/src/testing/test.controller.ts @@ -29,6 +29,22 @@ export class TestController { return item } + @Put('/') + async publishPartialUpdateAsync( + @INVOKE_CONTEXT() invokeContext: IInvoke, + @Body() commandDto: CommandPartialInputModel, + ) { + this.logger.debug('cmd:', commandDto) + this.logger.debug('commandService:' + this.commandService.tableName) + const item = await this.commandService.publishPartialUpdateAsync( + commandDto, + { + invokeContext, + }, + ) + return item + } + @Post('/sync') async publishCommandSync( @INVOKE_CONTEXT() invokeContext: IInvoke, From 49358e44cd51a63f6da82a9f1d5f4db783a764e3 Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Tue, 17 Dec 2024 14:42:34 +0700 Subject: [PATCH 5/8] add set_ttl_command to step function --- .../cli/templates/infra-local/serverless.yml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/cli/templates/infra-local/serverless.yml b/packages/cli/templates/infra-local/serverless.yml index 219f1bc..94155d1 100644 --- a/packages/cli/templates/infra-local/serverless.yml +++ b/packages/cli/templates/infra-local/serverless.yml @@ -187,7 +187,7 @@ stepFunctions: Choices: - Variable: $.result NumericEquals: 0 - Next: history_copy + Next: set_ttl_command - Variable: $.result NumericEquals: 1 Next: wait_prev_command @@ -213,6 +213,24 @@ stepFunctions: MaxAttempts: 5 BackoffRate: 2 OutputPath: $.Payload[0][0] + Next: set_ttl_command + set_ttl_command: + Type: Task + Resource: arn:aws:states:::lambda:invoke + Parameters: + FunctionName: arn:aws:lambda:ap-northeast-1:101010101010:function:serverless-example-dev-main + Payload: + input.$: $ + context.$: $$ + Retry: + - ErrorEquals: + - Lambda.ServiceException + - Lambda.AWSLambdaException + - Lambda.SdkClientException + IntervalSeconds: 2 + MaxAttempts: 5 + BackoffRate: 2 + OutputPath: $.Payload[0][0] Next: history_copy history_copy: Type: Task From 708c8550e444314d059e622a28af45bab8e4bc9b Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Tue, 17 Dec 2024 18:16:29 +0700 Subject: [PATCH 6/8] add unit test for ttl service and set tll command step --- .../commands/command.event.handler.spec.ts | 78 +++++++++- .../src/commands/command.event.handler.ts | 2 +- packages/core/src/commands/command.service.ts | 19 ++- .../core/src/commands/ttl.service.spec.ts | 133 ++++++++++++++++++ .../interfaces/command-service.interface.ts | 4 +- 5 files changed, 224 insertions(+), 12 deletions(-) create mode 100644 packages/core/src/commands/ttl.service.spec.ts diff --git a/packages/core/src/commands/command.event.handler.spec.ts b/packages/core/src/commands/command.event.handler.spec.ts index 6d74766..56e05d4 100644 --- a/packages/core/src/commands/command.event.handler.spec.ts +++ b/packages/core/src/commands/command.event.handler.spec.ts @@ -26,6 +26,7 @@ import { CommandService } from './command.service' import { DataSyncDdsHandler } from './handlers/data-sync-dds.handler' import { HistoryService } from './history.service' import { DataSyncCommandSfnName } from '../command-events/sfn-name.enum' +import { TtlService } from './ttl.service' export class MockedHandler implements IDataSyncHandler { async up(cmd: CommandModel): Promise { @@ -98,8 +99,15 @@ const createEvent = ( const sfnCheckVersionEvent = createEvent(DataSyncCommandSfnName.CHECK_VERSION) +const sfnSetTtlCommandEvent = createEvent( + DataSyncCommandSfnName.SET_TTL_COMMAND, + { + result: 0, + }, +) + const sfnHistoryCopyEvent = createEvent(DataSyncCommandSfnName.HISTORY_COPY, { - result: 0, + result: 'ok', }) const sfnTransformDataEvent = createEvent( @@ -138,6 +146,7 @@ describe('DataSyncCommandSfnEventHandler', () => { DynamoDbService, SnsService, MockedHandler, + TtlService, { provide: MODULE_OPTIONS_TOKEN, useValue: { @@ -268,6 +277,54 @@ describe('DataSyncCommandSfnEventHandler', () => { }) }) + it('should return result = ok when executing the correct set ttl command event', async () => { + // Arrange + dynamoDBMock.on(UpdateItemCommand).resolves({} as any) + snsMock.on(PublishCommand).resolves({} as any) + dynamoDBMock + .on(GetItemCommand) + .resolves({ Item: { sk: { S: '1726027976' }, version: { N: '1' } } }) + + // Action + const result = await commandEventHandler.execute(sfnSetTtlCommandEvent) + + // Assert + expect(result).toEqual({ result: 'ok' }) + }) + + it('should call the AWS service with the correct parameters when executing the set ttl command event', async () => { + // Arrange + dynamoDBMock.on(UpdateItemCommand).resolves({} as any) + snsMock.on(PublishCommand).resolves({} as any) + dynamoDBMock + .on(GetItemCommand) + .resolves({ Item: { sk: { S: '1726027976' }, version: { N: '1' } } }) + + // Action + await commandEventHandler.execute(sfnSetTtlCommandEvent) + + // Assert + expect(dynamoDBMock).toHaveReceivedCommandTimes(UpdateItemCommand, 2) + + expect(snsMock).toHaveReceivedCommandWith(PublishCommand, { + Message: expect.stringContaining('set_ttl_command'), + }) + expect(dynamoDBMock).toHaveReceivedNthCommandWith(1, UpdateItemCommand, { + TableName: 'env-app_name-table_name-command', + Key: { pk: { S: 'tenantCode#test' }, sk: { S: '1726027976@1' } }, + ExpressionAttributeValues: expect.objectContaining({ + ':status': { S: 'set_ttl_command:STARTED' }, + }), + }) + expect(dynamoDBMock).toHaveReceivedNthCommandWith(2, UpdateItemCommand, { + TableName: 'env-app_name-table_name-command', + Key: { pk: { S: 'tenantCode#test' }, sk: { S: '1726027976@1' } }, + ExpressionAttributeValues: expect.objectContaining({ + ':status': { S: 'set_ttl_command:FINISHED' }, + }), + }) + }) + it('should return result = ok when executing the correct history copy event', async () => { // Arrange dynamoDBMock.on(UpdateItemCommand).resolves({} as any) @@ -296,7 +353,7 @@ describe('DataSyncCommandSfnEventHandler', () => { // Assert expect(dynamoDBMock).toHaveReceivedCommandTimes(UpdateItemCommand, 2) - expect(dynamoDBMock).toHaveReceivedCommandTimes(GetItemCommand, 1) + expect(dynamoDBMock).toHaveReceivedCommandTimes(GetItemCommand, 2) expect(snsMock).toHaveReceivedCommandTimes(PublishCommand, 2) expect(snsMock).toHaveReceivedCommandWith(PublishCommand, { @@ -313,11 +370,22 @@ describe('DataSyncCommandSfnEventHandler', () => { TableName: 'env-app_name-table_name-data', Key: { pk: { S: 'tenantCode#test' }, sk: { S: '1726027976' } }, }) - expect(dynamoDBMock).toHaveReceivedNthCommandWith(3, PutItemCommand, { + expect(dynamoDBMock).toHaveReceivedNthCommandWith(3, GetItemCommand, { + Key: { + pk: { S: 'MASTER#test' }, + sk: { S: 'TTL#env-app_name-table_name-history' }, + }, + TableName: 'env-app_name-master-data', + }) + expect(dynamoDBMock).toHaveReceivedNthCommandWith(4, PutItemCommand, { TableName: 'env-app_name-table_name-history', - Item: { sk: { S: '1726027976@1' }, version: { N: '1' } }, + Item: { + sk: { S: '1726027976@1' }, + version: { N: '1' }, + ttl: { NULL: true }, + }, }) - expect(dynamoDBMock).toHaveReceivedNthCommandWith(4, UpdateItemCommand, { + expect(dynamoDBMock).toHaveReceivedNthCommandWith(5, UpdateItemCommand, { TableName: 'env-app_name-table_name-command', Key: { pk: { S: 'tenantCode#test' }, sk: { S: '1726027976@1' } }, ExpressionAttributeValues: expect.objectContaining({ diff --git a/packages/core/src/commands/command.event.handler.ts b/packages/core/src/commands/command.event.handler.ts index 0f01b6e..3f257e9 100644 --- a/packages/core/src/commands/command.event.handler.ts +++ b/packages/core/src/commands/command.event.handler.ts @@ -138,7 +138,7 @@ export class CommandEventHandler { ): Promise { this.logger.debug('setTtlCommand:: ', event.commandRecord) - await this.commandService.publishItem({ + await this.commandService.updateTtl({ pk: event.commandRecord.pk, sk: event.commandRecord.sk, }) diff --git a/packages/core/src/commands/command.service.ts b/packages/core/src/commands/command.service.ts index 815542e..9f84139 100644 --- a/packages/core/src/commands/command.service.ts +++ b/packages/core/src/commands/command.service.ts @@ -67,6 +67,9 @@ export class CommandService implements OnModuleInit, ICommandService { ) this.logger = new Logger(`${CommandService.name}:${this.tableName}`) } + publishItem(key: DetailKey): Promise { + throw new Error('Method not implemented.') + } onModuleInit() { if (!this.options.disableDefaultHandler) { @@ -459,7 +462,7 @@ export class CommandService implements OnModuleInit, ICommandService { ) } - async publishItem(key: DetailKey) { + async updateTtl(key: DetailKey) { const version = getSortKeyVersion(key.sk) const sk = removeSortKeyVersion(key.sk) if (version <= VERSION_FIRST + 1) { @@ -480,9 +483,17 @@ export class CommandService implements OnModuleInit, ICommandService { TableType.COMMAND, getTenantCode(key.pk), ) - command.ttl = ttl - this.logger.debug('publishItem::', command) - return await this.dynamoDbService.putItem(this.tableName, command) + this.logger.debug('updateTtl::', command) + return await this.dynamoDbService.updateItem( + this.tableName, + { + pk: key.pk, + sk: previousSk, + }, + { + set: { ttl }, + }, + ) } } diff --git a/packages/core/src/commands/ttl.service.spec.ts b/packages/core/src/commands/ttl.service.spec.ts new file mode 100644 index 0000000..7e666a5 --- /dev/null +++ b/packages/core/src/commands/ttl.service.spec.ts @@ -0,0 +1,133 @@ +import { Test, TestingModule } from '@nestjs/testing' +import { TtlService } from './ttl.service' +import { DynamoDbService } from '../data-store/dynamodb.service' +import { MODULE_OPTIONS_TOKEN } from './command.module-definition' +import { CommandModuleOptions } from '../interfaces' +import { TableType } from './enums' + +describe('TtlService', () => { + let service: TtlService + let dynamoDbService: jest.Mocked + const tableName = 'test-table' + + const mockOptions: CommandModuleOptions = { + tableName, + } + + const mockDynamoDbService = { + getTableName: jest.fn(), + getItem: jest.fn(), + } + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + TtlService, + { provide: MODULE_OPTIONS_TOKEN, useValue: mockOptions }, + { provide: DynamoDbService, useValue: mockDynamoDbService }, + ], + }).compile() + + service = module.get(TtlService) + dynamoDbService = module.get(DynamoDbService) + }) + + describe('calculateTtl', () => { + it('should return null if getTtlConfiguration returns null', async () => { + jest.spyOn(service, 'getTtlConfiguration').mockResolvedValue(null) + const result = await service.calculateTtl(TableType.DATA) + expect(result).toBeNull() + }) + + it('should return calculated TTL if days are retrieved', async () => { + jest.spyOn(service, 'getTtlConfiguration').mockResolvedValue(5) + jest.spyOn(service, 'calculateUnixTime').mockReturnValue(1698796800) + + const result = await service.calculateTtl(TableType.DATA) + expect(result).toBe(1698796800) + expect(service.getTtlConfiguration).toHaveBeenCalledWith( + TableType.DATA, + undefined, + ) + expect(service.calculateUnixTime).toHaveBeenCalledWith(5, undefined) + }) + }) + + describe('getTtlConfiguration', () => { + it('should return null if no TTL data is found', async () => { + dynamoDbService.getTableName + .mockReturnValueOnce('local-app-name-master-data') // For master data table + .mockReturnValueOnce('local-app-test-table-data') // For TTL key + dynamoDbService.getItem.mockResolvedValue(null) + + const result = await service.getTtlConfiguration( + TableType.DATA, + 'tenantCode', + ) + + expect(result).toBeNull() + expect(dynamoDbService.getItem).toHaveBeenCalledWith( + 'local-app-name-master-data', + { + pk: expect.any(String), + sk: expect.any(String), + }, + ) + }) + + it('should return days if TTL data is found', async () => { + dynamoDbService.getTableName + .mockReturnValueOnce('local-app-name-master-data') // For master data table + .mockReturnValueOnce('local-app-test-table-data') // For TTL key + dynamoDbService.getItem.mockResolvedValue({ + attributes: { days: 90 }, + }) + + const result = await service.getTtlConfiguration( + TableType.DATA, + 'tenantCode', + ) + + expect(result).toBe(90) + expect(dynamoDbService.getItem).toHaveBeenCalledWith( + 'local-app-name-master-data', + { + pk: expect.any(String), + sk: expect.any(String), + }, + ) + }) + + it('should return null and log error if an exception occurs', async () => { + dynamoDbService.getTableName.mockReturnValue('local-app-name-master-data') + dynamoDbService.getItem.mockRejectedValue(new Error('DB Error')) + + const result = await service.getTtlConfiguration( + TableType.DATA, + 'tenantCode', + ) + + expect(result).toBeNull() + }) + }) + + describe('calculateUnixTime', () => { + it('should throw an error if days are less than or equal to 0', () => { + expect(() => service.calculateUnixTime(0)).toThrow( + 'Number of days must be greater than 0.', + ) + }) + + it('should calculate UNIX time based on current time if no startDate is provided', () => { + jest.spyOn(Date, 'now').mockReturnValue(1734423525000) // Mock current time + const result = service.calculateUnixTime(1) // Add 1 day + expect(result).toBe(1734509925) // Next day in UNIX timestamp + }) + + it('should calculate UNIX time based on provided startDate', () => { + const startDate = new Date('2023-11-01T00:00:00Z') + const result = service.calculateUnixTime(2, startDate) + expect(result).toBe(1698796800 + 2 * 86400) // StartDate + 2 days + }) + }) +}) diff --git a/packages/core/src/interfaces/command-service.interface.ts b/packages/core/src/interfaces/command-service.interface.ts index bca8b90..269b56c 100644 --- a/packages/core/src/interfaces/command-service.interface.ts +++ b/packages/core/src/interfaces/command-service.interface.ts @@ -60,10 +60,10 @@ export interface ICommandService { ): Promise /** - * Publishes an old command by updating its TTL. + * Update the ttl for the previous command. * * @param {DetailKey} key - The primary and sort key details for the item. * @returns {Promise} A promise resolving to the result of the put operation, or `null` if the operation is not applicable. */ - publishItem(key: DetailKey): Promise + updateTtl(key: DetailKey): Promise } From 051c23c112ee7bab242000f992e04c6a7ae9d328 Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Wed, 18 Dec 2024 11:28:29 +0700 Subject: [PATCH 7/8] correct custom attributes --- .../infra-local/cognito-local/db/local_2G7noHgW.json | 4 ++-- packages/cli/templates/infra/libs/infra-stack.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/templates/infra-local/cognito-local/db/local_2G7noHgW.json b/packages/cli/templates/infra-local/cognito-local/db/local_2G7noHgW.json index 6f60848..b578927 100644 --- a/packages/cli/templates/infra-local/cognito-local/db/local_2G7noHgW.json +++ b/packages/cli/templates/infra-local/cognito-local/db/local_2G7noHgW.json @@ -11,7 +11,7 @@ "Value": "admin@test.com" }, { - "Name": "custom:tenant_code", + "Name": "custom:tenant", "Value": "mbc" }, { @@ -261,7 +261,7 @@ } }, { - "Name": "custom:tenant_code", + "Name": "custom:tenant", "AttributeDataType": "String", "DeveloperOnlyAttribute": false, "Mutable": true, diff --git a/packages/cli/templates/infra/libs/infra-stack.ts b/packages/cli/templates/infra/libs/infra-stack.ts index 5199136..b2bff4d 100644 --- a/packages/cli/templates/infra/libs/infra-stack.ts +++ b/packages/cli/templates/infra/libs/infra-stack.ts @@ -72,7 +72,7 @@ export class InfraStack extends cdk.Stack { mfa: cdk.aws_cognito.Mfa.OFF, accountRecovery: cdk.aws_cognito.AccountRecovery.NONE, customAttributes: { - tenant_code: new cdk.aws_cognito.StringAttribute({ + tenant: new cdk.aws_cognito.StringAttribute({ mutable: true, maxLen: 50, }), From 4e941a28a3409b6bf75a9e96b77e665c4d6e4103 Mon Sep 17 00:00:00 2001 From: Quan Nguyen Ba Date: Wed, 18 Dec 2024 16:26:14 +0700 Subject: [PATCH 8/8] add set_ttl_command step for infra --- packages/cli/templates/infra/libs/infra-stack.ts | 10 ++++++++-- .../infra/test/__snapshots__/infra.test.ts.snap | 15 +++++++++++++-- .../core/src/commands/command.service.spec.ts | 1 - packages/core/test/e2e/publish-async.e2e-spec.ts | 4 ++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/cli/templates/infra/libs/infra-stack.ts b/packages/cli/templates/infra/libs/infra-stack.ts index b2bff4d..adcd583 100644 --- a/packages/cli/templates/infra/libs/infra-stack.ts +++ b/packages/cli/templates/infra/libs/infra-stack.ts @@ -697,9 +697,15 @@ export class InfraStack extends cdk.Stack { transformData, cdk.aws_stepfunctions.IntegrationPattern.REQUEST_RESPONSE, ) + const setTtlCommand = lambdaInvoke( + 'set_ttl_command', + historyCopy, + cdk.aws_stepfunctions.IntegrationPattern.REQUEST_RESPONSE, + ) + const waitPrevCommand = lambdaInvoke( 'wait_prev_command', - historyCopy, + setTtlCommand, cdk.aws_stepfunctions.IntegrationPattern.WAIT_FOR_TASK_TOKEN, ) @@ -713,7 +719,7 @@ export class InfraStack extends cdk.Stack { ) .when( cdk.aws_stepfunctions.Condition.numberEquals('$.result', 0), - historyCopy, + setTtlCommand, ) .when( cdk.aws_stepfunctions.Condition.numberEquals('$.result', 1), diff --git a/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap b/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap index b64cc8d..f076409 100644 --- a/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap +++ b/packages/cli/templates/infra/test/__snapshots__/infra.test.ts.snap @@ -242,7 +242,7 @@ exports[`snapshot test for InfraStack 1`] = ` "Arn", ], }, - "","Payload":{"input.$":"$","context.$":"$$"}}},"check_version_result":{"Type":"Choice","Choices":[{"Variable":"$.result","NumericEquals":0,"Next":"history_copy"},{"Variable":"$.result","NumericEquals":1,"Next":"wait_prev_command"},{"Variable":"$.result","NumericEquals":-1,"Next":"fail"}],"Default":"wait_prev_command"},"wait_prev_command":{"Next":"history_copy","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload[0][0]","Resource":"arn:", + "","Payload":{"input.$":"$","context.$":"$$"}}},"check_version_result":{"Type":"Choice","Choices":[{"Variable":"$.result","NumericEquals":0,"Next":"set_ttl_command"},{"Variable":"$.result","NumericEquals":1,"Next":"wait_prev_command"},{"Variable":"$.result","NumericEquals":-1,"Next":"fail"}],"Default":"wait_prev_command"},"wait_prev_command":{"Next":"set_ttl_command","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload[0][0]","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -253,7 +253,18 @@ exports[`snapshot test for InfraStack 1`] = ` "Arn", ], }, - "","Payload":{"input.$":"$","context.$":"$$","taskToken.$":"$$.Task.Token"}}},"history_copy":{"Next":"transform_data","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload[0][0]","Resource":"arn:", + "","Payload":{"input.$":"$","context.$":"$$","taskToken.$":"$$.Task.Token"}}},"set_ttl_command":{"Next":"history_copy","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload[0][0]","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "lambdaapi893CD94E", + "Arn", + ], + }, + "","Payload":{"input.$":"$","context.$":"$$"}}},"history_copy":{"Next":"transform_data","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload[0][0]","Resource":"arn:", { "Ref": "AWS::Partition", }, diff --git a/packages/core/src/commands/command.service.spec.ts b/packages/core/src/commands/command.service.spec.ts index 57a15d7..9b76a67 100644 --- a/packages/core/src/commands/command.service.spec.ts +++ b/packages/core/src/commands/command.service.spec.ts @@ -261,7 +261,6 @@ describe('CommandService', () => { const item = await commandService.publishSync(inputItem, { invokeContext: {}, }) - console.log('$@#$@', item) expect(item).toBeDefined() expect(item).toMatchObject({ ...inputItem, diff --git a/packages/core/test/e2e/publish-async.e2e-spec.ts b/packages/core/test/e2e/publish-async.e2e-spec.ts index 59000dc..338f9ee 100644 --- a/packages/core/test/e2e/publish-async.e2e-spec.ts +++ b/packages/core/test/e2e/publish-async.e2e-spec.ts @@ -229,7 +229,7 @@ describe('Publish', () => { const postRes = await request(config.apiBaseUrl) .post(API_PATH) .send(payload) - console.log('$#@!$postRes$@#', postRes.body) + console.log('postRes', postRes.body) expect(postRes.statusCode).toEqual(201) await syncDataFinished('testing_table', { pk: payload.pk, @@ -262,7 +262,7 @@ describe('Publish', () => { pk: payload.pk, sk: `${payload.sk}@2`, }) - console.log('$@#$@#$putRes$@#$@#', putRes.body) + console.log('putRes', putRes.body) // Assert const data = await getItem(getTableName('testing_table', TableType.DATA), {