feat: add bitcoin dataloader cache and optimize transactionsCountIn24…

…Hours field

backend/package.json

@@ -42,7 +42,9 @@
     "@sentry/profiling-node": "^8.17.0",
     "@types/ws": "^8.5.11",
     "axios": "^1.7.2",
-    "cache-manager": "^5.7.1",
+    "cache-manager": "^5.2.3",
+    "cache-manager-redis-yet": "^4.1.2",
+    "redis": "^4.6.7",
     "dataloader": "^2.2.2",
     "fastify": "^4.28.1",
     "graphql": "^16.9.0",

backend/src/app.module.ts

@@ -1,10 +1,12 @@
 import { APP_INTERCEPTOR } from '@nestjs/core';
-import { CacheModule } from '@nestjs/cache-manager';
+import { CacheModule, CacheStore } from '@nestjs/cache-manager';
 import { HttpException, Module } from '@nestjs/common';
 import { ConfigModule, ConfigService } from '@nestjs/config';
 import { SentryInterceptor, SentryModule } from '@ntegral/nestjs-sentry';
 import { nodeProfilingIntegration } from '@sentry/profiling-node';
-import { envSchema } from './env';
+import type { RedisClientOptions } from 'redis';
+import { redisStore } from 'cache-manager-redis-yet';
+import { Env, envSchema } from './env';
 import { CoreModule } from './core/core.module';
 import { ApiModule } from './modules/api.module';
 
@@ -20,7 +22,7 @@ import { ApiModule } from './modules/api.module';
     }),
     SentryModule.forRootAsync({
       imports: [ConfigModule],
-      useFactory: async (configService: ConfigService) => ({
+      useFactory: async (configService: ConfigService<Env>) => ({
         dsn: configService.get('SENTRY_DSN'),
         environment: configService.get('NODE_ENV'),
         tracesSampleRate: 0.1,
@@ -33,8 +35,18 @@ import { ApiModule } from './modules/api.module';
       }),
       inject: [ConfigService],
     }),
-    CacheModule.register({
+    CacheModule.registerAsync<RedisClientOptions>({
       isGlobal: true,
+      imports: [ConfigModule],
+      useFactory: async (configService: ConfigService<Env>) => {
+        const store = await redisStore({
+          url: configService.get('REDIS_URL'),
+        }) as unknown as CacheStore;
+        return {
+          store,
+        };
+      },
+      inject: [ConfigService],
     }),
     CoreModule,
     ApiModule,

backend/src/env.ts

@@ -3,6 +3,7 @@ import { NetworkType } from './constants';
 
 export const envSchema = z
   .object({
+    NODE_ENV: z.enum(['development', 'production', 'test']).default('development'),
     NETWORK: z.enum([NetworkType.mainnet, NetworkType.testnet]).default(NetworkType.testnet),
     ENABLED_GRAPHQL_PLAYGROUND: z.boolean().default(true),
 
@@ -13,6 +14,8 @@ export const envSchema = z
 
     CKB_EXPLORER_API_URL: z.string(),
     CKB_RPC_WEBSOCKET_URL: z.string(),
+
+    SENTRY_DSN: z.string().optional(),
   })
   .and(
     z.union([

backend/src/modules/bitcoin/bitcoin.resolver.ts

@@ -2,10 +2,7 @@ import { Float, Parent, Query, ResolveField, Resolver } from '@nestjs/graphql';
 import { BitcoinApiService } from 'src/core/bitcoin-api/bitcoin-api.service';
 import { BitcoinBaseChainInfo, BitcoinChainInfo, BitcoinFees } from './bitcoin.model';
 import { Loader } from '@applifting-io/nestjs-dataloader';
-import {
-  BitcoinBlockTransactionsLoader,
-  BitcoinBlockTransactionsLoaderType,
-} from './block/block.dataloader';
+import { BitcoinBlockTxidsLoader, BitcoinBlockTxidsLoaderType } from './block/dataloader/block-txids.loader';
 
 // 60 * 24 = 1440 minutes
 const BLOCK_NUMBER_OF_24_HOURS = 144;
@@ -23,16 +20,16 @@ export class BitcoinResolver {
   @ResolveField(() => Float)
   public async transactionsCountIn24Hours(
     @Parent() chainInfo: BitcoinBaseChainInfo,
-    @Loader(BitcoinBlockTransactionsLoader) blockTxsLoader: BitcoinBlockTransactionsLoaderType,
+    @Loader(BitcoinBlockTxidsLoader) blockTxidsLoader: BitcoinBlockTxidsLoaderType,
   ): Promise<number> {
     const blockNumbers = Array.from(
       { length: BLOCK_NUMBER_OF_24_HOURS },
       (_, i) => chainInfo.tipBlockHeight - i,
     );
-    const transactions = await blockTxsLoader.loadMany(
+    const txidsCollection = await blockTxidsLoader.loadMany(
       blockNumbers.map((blockNumber) => ({ height: blockNumber })),
     );
-    const count = transactions
+    const count = txidsCollection
       .map((txs) => (txs instanceof Array ? txs : []))
       .reduce((acc, txs) => acc + txs?.length ?? 0, 0);
     return count;

backend/src/modules/bitcoin/block/block.module.ts

@@ -1,11 +1,18 @@
 import { Module } from '@nestjs/common';
 import { BitcoinApiModule } from 'src/core/bitcoin-api/bitcoin-api.module';
-import { BitcoinBlockLoader, BitcoinBlockTransactionsLoader } from './block.dataloader';
 import { BitcoinBlockResolver } from './block.resolver';
+import { BitcoinBlockLoader } from './dataloader/block.loader';
+import { BitcoinBlockTransactionsLoader } from './dataloader/block-transactions.loader';
+import { BitcoinBlockTxidsLoader } from './dataloader/block-txids.loader';
 
 @Module({
   imports: [BitcoinApiModule],
-  providers: [BitcoinBlockResolver, BitcoinBlockLoader, BitcoinBlockTransactionsLoader],
-  exports: [BitcoinBlockLoader, BitcoinBlockTransactionsLoader],
+  providers: [
+    BitcoinBlockResolver,
+    BitcoinBlockLoader,
+    BitcoinBlockTransactionsLoader,
+    BitcoinBlockTxidsLoader,
+  ],
+  exports: [BitcoinBlockLoader, BitcoinBlockTransactionsLoader, BitcoinBlockTxidsLoader],
 })
 export class BitcoinBlockModule {}

backend/src/modules/bitcoin/block/block.resolver.ts

@@ -4,12 +4,8 @@ import { Args, Float, Parent, Query, ResolveField, Resolver } from '@nestjs/grap
 import { BitcoinBaseTransaction, BitcoinTransaction } from '../transaction/transaction.model';
 import { BitcoinAddress, BitcoinBaseAddress } from '../address/address.model';
 import { BitcoinBaseBlock, BitcoinBlock, FeeRateRange } from './block.model';
-import {
-  BitcoinBlockLoader,
-  BitcoinBlockLoaderType,
-  BitcoinBlockTransactionsLoader,
-  BitcoinBlockTransactionsLoaderType,
-} from './block.dataloader';
+import { BitcoinBlockLoader, BitcoinBlockLoaderType } from './dataloader/block.loader';
+import { BitcoinBlockTransactionsLoader, BitcoinBlockTransactionsLoaderType } from './dataloader/block-transactions.loader';
 
 @Resolver(() => BitcoinBlock)
 export class BitcoinBlockResolver {

backend/src/modules/bitcoin/block/dataloader/base.ts

@@ -0,0 +1,94 @@
+import { Logger } from '@nestjs/common';
+import { BitcoinApiService } from 'src/core/bitcoin-api/bitcoin-api.service';
+import * as BitcoinApi from 'src/core/bitcoin-api/bitcoin-api.schema';
+import { Cache } from '@nestjs/cache-manager';
+
+export abstract class BitcoinBaseLoader {
+  protected logger = new Logger(BitcoinBaseLoader.name);
+  abstract bitcoinApiService: BitcoinApiService;
+  abstract cacheManager: Cache;
+
+  private async getBlockHashByHeight(height: number): Promise<string> {
+    const cacheKey = `bitcoinApiService#getBlockHeight:${height}`;
+    const cached = await this.cacheManager.get<string>(cacheKey);
+    if (cached) {
+      return cached;
+    }
+    const hash = await this.bitcoinApiService.getBlockHeight({ height });
+    await this.cacheManager.set(cacheKey, hash);
+    return hash;
+  }
+
+  private async getBlockByHash(hash: string): Promise<BitcoinApi.Block> {
+    const cacheKey = `bitcoinApiService#getBlock:${hash}`;
+    const cached = await this.cacheManager.get<BitcoinApi.Block>(cacheKey);
+    if (cached) {
+      return cached;
+    }
+    const block = await this.bitcoinApiService.getBlock({ hash });
+    const ttl = Date.now() - block.timestamp * 1000;
+    await this.cacheManager.set(cacheKey, block, ttl);
+    return block;
+  }
+
+  private async getBlockTxsByHash(
+    hash: string,
+    startIndex: number,
+  ): Promise<BitcoinApi.Transaction[]> {
+    const cacheKey = `bitcoinApiService#getBlockTxs:${hash}:${startIndex}`;
+    const cached = await this.cacheManager.get<BitcoinApi.Transaction[]>(cacheKey);
+    if (cached) {
+      return cached;
+    }
+    const txs = await this.bitcoinApiService.getBlockTxs({ hash, startIndex });
+    await this.cacheManager.set(cacheKey, txs);
+    return txs;
+  }
+
+  private async getBlockTxidsByHash(hash: string): Promise<string[]> {
+    const cacheKey = `bitcoinApiService#getBlockTxids:${hash}`;
+    const cached = await this.cacheManager.get<string[]>(cacheKey);
+    if (cached) {
+      return cached;
+    }
+    const txids = await this.bitcoinApiService.getBlockTxids({ hash });
+    await this.cacheManager.set(cacheKey, txids);
+    return txids;
+  }
+
+  protected async getBlock(hashOrHeight: string): Promise<BitcoinApi.Block> {
+    if (hashOrHeight.startsWith('0')) {
+      const block = await this.getBlockByHash(hashOrHeight);
+      return block;
+    }
+    const height = parseInt(hashOrHeight, 10);
+    const hash = await this.getBlockHashByHeight(height);
+    const block = await this.getBlockByHash(hash);
+    return block;
+  }
+
+  protected async getBlockTxs(
+    hashOrHeight: string,
+    startIndex: number,
+  ): Promise<BitcoinApi.Transaction[]> {
+    if (hashOrHeight.startsWith('0')) {
+      return this.getBlockTxsByHash(hashOrHeight, startIndex);
+    }
+    const height = parseInt(hashOrHeight, 10);
+    const hash = await this.getBlockHashByHeight(height);
+    const txs = await this.getBlockTxsByHash(hash, startIndex);
+    return txs;
+  }
+
+  protected async getBlockTxids(hashOrHeight: string): Promise<string[]> {
+    if (hashOrHeight.startsWith('0')) {
+      const txs = await this.getBlockTxsByHash(hashOrHeight, 0);
+      return txs.map((tx) => tx.txid);
+    }
+    const height = parseInt(hashOrHeight, 10);
+    const hash = await this.getBlockHashByHeight(height);
+    const txids = await this.getBlockTxidsByHash(hash);
+    return txids;
+  }
+}
+

backend/src/modules/bitcoin/block/dataloader/block-transactions.loader.ts

@@ -0,0 +1,48 @@
+import DataLoader from 'dataloader';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { NestDataLoader } from '@applifting-io/nestjs-dataloader';
+import { DataLoaderResponse } from 'src/common/type/dataloader';
+import { BitcoinApiService } from 'src/core/bitcoin-api/bitcoin-api.service';
+import * as BitcoinApi from 'src/core/bitcoin-api/bitcoin-api.schema';
+import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+import { BitcoinBaseLoader } from './base';
+
+export interface BitcoinBlockTransactionsLoaderParams {
+  hash?: string;
+  height?: number;
+  startIndex?: number;
+}
+
+@Injectable()
+export class BitcoinBlockTransactionsLoader
+  extends BitcoinBaseLoader
+  implements NestDataLoader<BitcoinBlockTransactionsLoaderParams, BitcoinApi.Transaction[] | null>
+{
+  protected logger = new Logger(BitcoinBlockTransactionsLoader.name);
+
+  constructor(
+    public bitcoinApiService: BitcoinApiService,
+    @Inject(CACHE_MANAGER) public cacheManager: Cache,
+  ) {
+    super();
+  }
+
+  public getBatchFunction() {
+    return async (batchProps: BitcoinBlockTransactionsLoaderParams[]) => {
+      this.logger.debug(`Loading bitcoin block transactions`);
+      const results = await Promise.allSettled(
+        batchProps.map(async ({ hash, height, startIndex }) =>
+          this.getBlockTxs(hash || height.toString(), startIndex),
+        ),
+      );
+      return results.map((result) => (result.status === 'fulfilled' ? result.value : null));
+    };
+  }
+}
+export type BitcoinBlockTransactionsLoaderType = DataLoader<
+  BitcoinBlockTransactionsLoaderParams,
+  BitcoinApi.Transaction[] | null
+>;
+export type BitcoinBlockTransactionsLoaderResponse =
+  DataLoaderResponse<BitcoinBlockTransactionsLoader>;
+

backend/src/modules/bitcoin/block/dataloader/block-txids.loader.ts

@@ -0,0 +1,39 @@
+import DataLoader from 'dataloader';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { NestDataLoader } from '@applifting-io/nestjs-dataloader';
+import { DataLoaderResponse } from 'src/common/type/dataloader';
+import { BitcoinApiService } from 'src/core/bitcoin-api/bitcoin-api.service';
+import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+import { BitcoinBaseLoader } from './base';
+
+export interface BitcoinBlockTxidsLoaderParams {
+  hash?: string;
+  height?: number;
+}
+
+@Injectable()
+export class BitcoinBlockTxidsLoader
+  extends BitcoinBaseLoader
+  implements NestDataLoader<BitcoinBlockTxidsLoaderParams, string[] | null>
+{
+  protected logger = new Logger(BitcoinBlockTxidsLoader.name);
+
+  constructor(
+    public bitcoinApiService: BitcoinApiService,
+    @Inject(CACHE_MANAGER) public cacheManager: Cache,
+  ) {
+    super();
+  }
+
+  public getBatchFunction() {
+    return async (keys: BitcoinBlockTxidsLoaderParams[]) => {
+      this.logger.debug(`Loading bitcoin block transactions`);
+      const results = await Promise.allSettled(
+        keys.map(async ({ hash, height }) => this.getBlockTxids(hash || height.toString())),
+      );
+      return results.map((result) => (result.status === 'fulfilled' ? result.value : null));
+    };
+  }
+}
+export type BitcoinBlockTxidsLoaderType = DataLoader<BitcoinBlockTxidsLoaderParams, string[]>;
+export type BitcoinBlockTxidsLoaderResponse = DataLoaderResponse<BitcoinBlockTxidsLoader>;