Merge pull request #54 from ckb-cell/fix/query-performance

backend/.eslintrc.js

@@ -5,11 +5,8 @@ module.exports = {
     tsconfigRootDir: __dirname,
     sourceType: 'module',
   },
-  plugins: ['@typescript-eslint/eslint-plugin'],
-  extends: [
-    'plugin:@typescript-eslint/recommended',
-    'plugin:prettier/recommended',
-  ],
+  plugins: ['@typescript-eslint/eslint-plugin', 'import'],
+  extends: ['plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'],
   root: true,
   env: {
     node: true,
@@ -21,5 +18,13 @@ module.exports = {
     '@typescript-eslint/explicit-function-return-type': 'off',
     '@typescript-eslint/explicit-module-boundary-types': 'off',
     '@typescript-eslint/no-explicit-any': 'off',
+    'no-restricted-imports': [
+      'error',
+      {
+        name: 'nestjs-cacheable',
+        importNames: ['Cacheable'],
+        message: "Please use 'src/decorators/cacheable.decorator' instead",
+      },
+    ],
   },
 };

backend/.husky/pre-commit

@@ -0,0 +1 @@
+npm

backend/package.json

@@ -18,7 +18,12 @@
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./test/jest-e2e.json",
-    "postinstall": "npx prisma generate"
+    "postinstall": "npx prisma generate",
+    "precommit": "lint-staged",
+    "prepare": "husky"
+  },
+  "lint-staged": {
+    "*.ts": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix"
   },
   "dependencies": {
     "@applifting-io/nestjs-dataloader": "^1.1.5",
@@ -36,12 +41,12 @@
     "@nestjs/platform-express": "^10.0.0",
     "@nestjs/platform-fastify": "^10.3.10",
     "@nestjs/schedule": "^4.1.0",
-    "@ntegral/nestjs-sentry": "^4.0.1",
     "@prisma/client": "^5.16.2",
     "@rgbpp-sdk/btc": "^0.0.0-snap-20240727021715",
     "@rgbpp-sdk/ckb": "^0.0.0-snap-20240727021715",
-    "@sentry/node": "^8.17.0",
-    "@sentry/profiling-node": "^8.17.0",
+    "@ntegral/nestjs-sentry": "^4.0.1",
+    "@sentry/node": "^7.116.0",
+    "@sentry/profiling-node": "^7.116.0",
     "@types/ws": "^8.5.11",
     "axios": "^1.7.2",
     "bullmq": "^5.11.0",
@@ -70,13 +75,16 @@
     "@types/jest": "^29.5.2",
     "@types/lodash": "^4.17.7",
     "@types/node": "^20.3.1",
+    "@types/serialize-javascript": "^5.0.4",
     "@types/supertest": "^6.0.0",
     "@typescript-eslint/eslint-plugin": "^7.0.0",
     "@typescript-eslint/parser": "^7.0.0",
     "eslint": "^8.42.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-prettier": "^5.0.0",
+    "husky": "^9.1.4",
     "jest": "^29.5.0",
+    "lint-staged": "^15.2.7",
     "prettier": "^3.0.0",
     "prisma": "^5.16.2",
     "source-map-support": "^0.5.21",
@@ -94,7 +102,9 @@
       "json",
       "ts"
     ],
-    "modulePaths": ["."],
+    "modulePaths": [
+      "."
+    ],
     "testRegex": ".*\\.(spec|test)\\.ts$",
     "transform": {
       "^.+\\.(t|j)s$": "ts-jest"

backend/src/app.module.ts

@@ -22,8 +22,8 @@ import configModule from './config';
       useFactory: async (configService: ConfigService<Env>) => ({
         dsn: configService.get('SENTRY_DSN'),
         environment: configService.get('NODE_ENV'),
-        tracesSampleRate: 0.1,
-        profilesSampleRate: 0.1,
+        tracesSampleRate: 0.5,
+        profilesSampleRate: 0.5,
         integrations: [nodeProfilingIntegration()],
         logLevels:
           configService.get('NODE_ENV') === 'production'

backend/src/common/date.ts

@@ -1,3 +1,4 @@
 export const ONE_MONTH_MS = 30 * 24 * 60 * 60 * 1000;
+export const ONE_HOUR_MS = 60 * 60 * 1000;
 export const TEN_MINUTES_MS = 10 * 60 * 1000;
 export const ONE_MINUTE_MS = 60 * 1000;

backend/src/constants.ts

@@ -1,4 +1,3 @@
-import { Script } from '@ckb-lumos/lumos';
 import * as RgbppBtc from '@rgbpp-sdk/btc';
 import { BTCTestnetType } from '@rgbpp-sdk/ckb';
 

backend/src/core/bitcoin-api/bitcoin-api.service.ts

@@ -7,9 +7,9 @@ import { Env } from 'src/env';
 import { IBitcoinDataProvider } from './bitcoin-api.interface';
 import { ElectrsService } from './provider/electrs.service';
 import { MempoolService } from './provider/mempool.service';
-import { ChainInfo } from './bitcoin-api.schema';
-import { Cacheable } from 'nestjs-cacheable';
-import { ONE_MONTH_MS, TEN_MINUTES_MS } from 'src/common/date';
+import { ChainInfo, Transaction } from './bitcoin-api.schema';
+import { ONE_HOUR_MS, ONE_MONTH_MS, TEN_MINUTES_MS } from 'src/common/date';
+import { Cacheable } from 'src/decorators/cacheable.decorator';
 
 type MethodParameters<T, K extends keyof T> = T[K] extends (...args: infer P) => any ? P : never;
 type MethodReturnType<T, K extends keyof T> = T[K] extends (...args: any[]) => infer R ? R : never;
@@ -185,9 +185,13 @@ export class BitcoinApiService {
   }
 
   @Cacheable({
-    key: ({ txid }) => `getTx:${txid}`,
     namespace: 'bitcoinApiService',
+    key: ({ txid }) => `getTx:${txid}`,
     ttl: ONE_MONTH_MS,
+    shouldCache: (tx: Transaction) =>
+      tx.status.confirmed &&
+      !!tx.status.block_time &&
+      tx.status.block_time - Date.now() < ONE_HOUR_MS,
   })
   public async getTx({ txid }: { txid: string }) {
     return this.call('getTx', { txid });
@@ -202,35 +206,35 @@ export class BitcoinApiService {
   }
 
   @Cacheable({
-    key: ({ hash }) => `getBlock:${hash}`,
     namespace: 'bitcoinApiService',
+    key: ({ hash }) => `getBlock:${hash}`,
     ttl: ONE_MONTH_MS,
   })
   public async getBlock({ hash }: { hash: string }) {
     return this.call('getBlock', { hash });
   }
 
   @Cacheable({
-    key: ({ hash, startIndex }) => `getBlockTxs:${hash}:${startIndex}`,
     namespace: 'bitcoinApiService',
+    key: ({ hash, startIndex }) => `getBlockTxs:${hash}:${startIndex}`,
     ttl: ONE_MONTH_MS,
   })
   public async getBlockTxs({ hash, startIndex }: { hash: string; startIndex?: number }) {
     return this.call('getBlockTxs', { hash, startIndex });
   }
 
   @Cacheable({
-    key: ({ height }) => `getBlockHeight:${height}`,
     namespace: 'bitcoinApiService',
+    key: ({ height }) => `getBlockHeight:${height}`,
     ttl: TEN_MINUTES_MS,
   })
   public async getBlockHeight({ height }: { height: number }) {
     return this.call('getBlockHeight', { height });
   }
 
   @Cacheable({
-    key: ({ hash }) => `getBlockTxids:${hash}`,
     namespace: 'bitcoinApiService',
+    key: ({ hash }) => `getBlockTxids:${hash}`,
     ttl: ONE_MONTH_MS,
   })
   public async getBlockTxids({ hash }: { hash: string }) {

backend/src/core/ckb-explorer/ckb-explorer.interface.ts

@@ -52,7 +52,7 @@ export interface CkbExplorerResponse<T, IsPaginated extends boolean = false> {
   meta: IsPaginated extends true ? PaginationMeta : never;
 }
 
-export type NonPaginatedResponse<T extends {}> = CkbExplorerResponse<
+export type NonPaginatedResponse<T extends object> = CkbExplorerResponse<
   {
     id: string;
     type: string;
@@ -61,7 +61,7 @@ export type NonPaginatedResponse<T extends {}> = CkbExplorerResponse<
   false
 >;
 
-export type PaginatedResponse<T extends {}> = CkbExplorerResponse<
+export type PaginatedResponse<T extends object> = CkbExplorerResponse<
   {
     id: string;
     type: string;

backend/src/core/ckb-explorer/ckb-explorer.service.ts

@@ -1,4 +1,4 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import axios, { Axios } from 'axios';
 import { Env } from 'src/env';
@@ -22,6 +22,10 @@ import {
   TransactionFeesStatistic,
   TransactionListSortType,
 } from './ckb-explorer.interface';
+import { ONE_HOUR_MS, ONE_MONTH_MS } from 'src/common/date';
+import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+import { toNumber } from 'lodash';
+import { Cacheable } from 'src/decorators/cacheable.decorator';
 
 type BasePaginationParams = {
   page?: number;
@@ -62,7 +66,10 @@ export class CkbExplorerService {
   private logger = new Logger(CkbExplorerService.name);
   private request: Axios;
 
-  constructor(private configService: ConfigService<Env>) {
+  constructor(
+    private configService: ConfigService<Env>,
+    @Inject(CACHE_MANAGER) protected cacheManager: Cache,
+  ) {
     this.request = axios.create({
       baseURL: this.configService.get('CKB_EXPLORER_API_URL'),
       headers: {
@@ -118,11 +125,22 @@ export class CkbExplorerService {
     return response.data;
   }
 
+  @Cacheable({
+    namespace: 'CkbExplorerService',
+    key: (heightOrHash: string) => `getBlock:${heightOrHash}`,
+    ttl: ONE_MONTH_MS,
+  })
   public async getBlock(heightOrHash: string): Promise<NonPaginatedResponse<Block>> {
     const response = await this.request.get(`/v1/blocks/${heightOrHash}`);
     return response.data;
   }
 
+  @Cacheable({
+    namespace: 'CkbExplorerService',
+    key: (heightOrHash: string, { page = 1, pageSize = 10 }: BasePaginationParams = {}) =>
+      `getBlockTransactions:${heightOrHash},${page},${pageSize}`,
+    ttl: ONE_MONTH_MS,
+  })
   public async getBlockTransactions(
     blockHash: string,
     { page = 1, pageSize = 10 }: BasePaginationParams = {},
@@ -173,11 +191,26 @@ export class CkbExplorerService {
     return response.data;
   }
 
+  @Cacheable({
+    namespace: 'CkbExplorerService',
+    key: (txHash: string) => `getTransaction:${txHash}`,
+    ttl: ONE_HOUR_MS,
+    shouldCache: async (tx: NonPaginatedResponse<DetailTransaction>) => {
+      // cache tx for 1 month if it's committed and older than 1 hour
+      const { tx_status, block_timestamp } = tx.data.attributes;
+      return tx_status === 'committed' && Date.now() - toNumber(block_timestamp) > ONE_HOUR_MS;
+    },
+  })
   public async getTransaction(txHash: string): Promise<NonPaginatedResponse<DetailTransaction>> {
     const response = await this.request.get(`/v1/transactions/${txHash}`);
     return response.data;
   }
 
+  @Cacheable({
+    namespace: 'CkbExplorerService',
+    key: (txHash: string) => `getRgbppDigest:${txHash}`,
+    ttl: ONE_MONTH_MS,
+  })
   public async getRgbppDigest(txHash: string): Promise<CkbExplorerResponse<RgbppDigest>> {
     const response = await this.request.get(`/v2/ckb_transactions/${txHash}/rgb_digest`);
     return response.data;

backend/src/core/ckb-rpc/ckb-rpc-websocket.service.ts

@@ -11,6 +11,8 @@ import {
   SearchKey,
   TransactionWithStatusResponse,
 } from './ckb-rpc.interface';
+import { Cacheable } from 'src/decorators/cacheable.decorator';
+import { ONE_MONTH_MS } from 'src/common/date';
 
 @Injectable()
 export class CkbRpcWebsocketService {
@@ -19,11 +21,26 @@ export class CkbRpcWebsocketService {
 
   constructor(private configService: ConfigService<Env>) {
     this.websocket = new RpcWebsocketsClient(this.configService.get('CKB_RPC_WEBSOCKET_URL'));
-    this.websocket.on('error', (error) => {
-      this.logger.error(error.message);
+
+    this.websocket.on('open', () => {
+      this.websocket.on('error', (error) => {
+        this.logger.error(error.message);
+      });
     });
   }
 
+  @Cacheable({
+    namespace: 'CkbRpcWebsocketService',
+    key: (txHash: string) => `getTransaction:${txHash}`,
+    ttl: ONE_MONTH_MS,
+    shouldCache: async (tx: TransactionWithStatusResponse, that: CkbRpcWebsocketService) => {
+      if (tx.tx_status.status !== 'committed') {
+        return false;
+      }
+      const block = await that.getTipBlockNumber();
+      return BI.from(tx.tx_status.block_number).lt(BI.from(block));
+    },
+  })
   public async getTransaction(txHash: string): Promise<TransactionWithStatusResponse> {
     this.logger.debug(`get_transaction - txHash: ${txHash}`);
     const tx = await this.websocket.call('get_transaction', [txHash]);
@@ -50,6 +67,12 @@ export class CkbRpcWebsocketService {
     return blockEconomicState as BlockEconomicState;
   }
 
+  @Cacheable({
+    namespace: 'CkbRpcWebsocketService',
+    key: 'getTipBlockNumber',
+    // just cache for 1 second to avoid too many requests
+    ttl: 1000,
+  })
   public async getTipBlockNumber(): Promise<number> {
     this.logger.debug('get_tip_block_number');
     const tipBlockNumber = await this.websocket.call('get_tip_block_number', []);

backend/src/core/ckb-rpc/ckb-rpc.interface.ts

@@ -1,5 +1,3 @@
-import { RPC } from '@ckb-lumos/rpc';
-
 export interface CellDep {
   dep_type: string;
   out_point: {

backend/src/decorators/cacheable.decorator.ts

@@ -0,0 +1,62 @@
+// eslint-disable-next-line no-restricted-imports
+import { Cache, CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Inject, Logger } from '@nestjs/common';
+import { CacheableRegisterOptions, Cacheable as _Cacheable } from 'nestjs-cacheable';
+import { cacheableHandle, generateComposedKey } from 'nestjs-cacheable/dist/cacheable.helper';
+
+export interface CustomCacheableRegisterOptions extends CacheableRegisterOptions {
+  shouldCache?: (result: any, target: any) => boolean | Promise<boolean>;
+}
+
+const logger = new Logger('Cacheable');
+
+/**
+ * Cacheable decorator with custom options, based on the original Cacheable decorator from the nestjs-cacheable package.
+ * Adds a shouldCache option to determine whether the result should be cached.
+ *
+ * @example
+ * @Cacheable({
+ *   ttl: 1000,
+ *   key: (args: any[]) => args[0],
+ *   shouldCache: (result: any) => result !== null,
+ * });
+ */
+export function Cacheable(options: CustomCacheableRegisterOptions): MethodDecorator {
+  const injectCacheService = Inject(CACHE_MANAGER);
+
+  return function(target, propertyKey, descriptor) {
+    // eslint-disable-next-line @typescript-eslint/ban-types
+    const originalMethod = descriptor.value as unknown as Function;
+
+    injectCacheService(target, '__cacheManager');
+    return {
+      ...descriptor,
+      value: async function(...args: any[]) {
+        const cacheManager = this.__cacheManager as Cache;
+        if (!cacheManager) return originalMethod.apply(this, args);
+        const composeOptions: Parameters<typeof generateComposedKey>[0] = {
+          methodName: String(propertyKey),
+          key: options.key,
+          namespace: options.namespace,
+          args,
+        };
+        const [key] = generateComposedKey(composeOptions);
+        const returnVal = await cacheableHandle(
+          key,
+          () => originalMethod.apply(this, args),
+          options.ttl,
+        );
+
+        // Remove the cache if shouldCache returns false
+        const shouldCache = options.shouldCache
+          ? await options.shouldCache(returnVal, this)
+          : true;
+        if (!shouldCache) {
+          logger.debug(`Removing cache for key: ${key}`);
+          await cacheManager.del(key);
+        }
+        return returnVal;
+      } as any,
+    };
+  };
+}

backend/src/decorators/parent-field.decorator.ts

@@ -6,4 +6,3 @@ export const ParentField = createParamDecorator((param: string, ctx: ExecutionCo
   const value = root[param];
   return value;
 });
-