Merge pull request #169 from utxostack/feat/block-indexing

feat: rgbpp transactions indexing

Dockerfile

@@ -1,6 +1,6 @@
 ARG NODE_VERSION=20
 
-FROM node:${NODE_VERSION}-slim AS base
+FROM node:${NODE_VERSION} AS base
 ARG GIT_BRANCH
 
 ENV PNPM_HOME="/pnpm"
@@ -15,7 +15,7 @@ FROM base AS prod-deps
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile --ignore-scripts
 RUN pnpm run --filter backend postinstall
 
-FROM base as build
+FROM base AS build
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 RUN pnpm run --filter backend build
 

backend/prisma/migrations/20240916084317_transaction/migration.sql

@@ -0,0 +1,35 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `difficulty` on the `Block` table. All the data in the column will be lost.
+  - You are about to drop the column `maxFee` on the `Block` table. All the data in the column will be lost.
+  - You are about to drop the column `minFee` on the `Block` table. All the data in the column will be lost.
+  - You are about to drop the column `size` on the `Block` table. All the data in the column will be lost.
+  - You are about to drop the column `totalFee` on the `Block` table. All the data in the column will be lost.
+  - You are about to drop the column `fee` on the `Transaction` table. All the data in the column will be lost.
+  - You are about to drop the column `size` on the `Transaction` table. All the data in the column will be lost.
+  - You are about to drop the column `timestamp` on the `Transaction` table. All the data in the column will be lost.
+  - You are about to drop the `Output` table. If the table is not empty, all the data it contains will be lost.
+  - Changed the type of `index` on the `Transaction` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required.
+
+*/
+-- DropForeignKey
+ALTER TABLE "Output" DROP CONSTRAINT "Output_chainId_txHash_fkey";
+
+-- AlterTable
+ALTER TABLE "Block" DROP COLUMN "difficulty",
+DROP COLUMN "maxFee",
+DROP COLUMN "minFee",
+DROP COLUMN "size",
+DROP COLUMN "totalFee";
+
+-- AlterTable
+ALTER TABLE "Transaction" DROP COLUMN "fee",
+DROP COLUMN "size",
+DROP COLUMN "timestamp",
+ADD COLUMN     "btcTxid" TEXT,
+DROP COLUMN "index",
+ADD COLUMN     "index" INTEGER NOT NULL;
+
+-- DropTable
+DROP TABLE "Output";

backend/prisma/schema.prisma

@@ -39,11 +39,6 @@ model Block {
   number            Int
   timestamp         DateTime
   transactionsCount Int
-  size              Int
-  totalFee          BigInt
-  minFee            BigInt
-  maxFee            BigInt
-  difficulty        BigInt
   createdAt         DateTime @default(now())
   updatedAt         DateTime @updatedAt
 
@@ -58,13 +53,11 @@ model Transaction {
   id            Int            @id @default(autoincrement())
   chainId       Int
   hash          String
-  index         String
+  index         Int
   blockNumber   Int
-  timestamp     DateTime
-  fee           BigInt
-  size          Int
   isCellbase    Boolean        @default(false)
   isRgbpp       Boolean        @default(false)
+  btcTxid       String?
   leapDirection LeapDirection?
   inputCount    Int
   outputCount   Int
@@ -73,33 +66,10 @@ model Transaction {
 
   chain   Chain    @relation(fields: [chainId], references: [id])
   block   Block    @relation(fields: [chainId, blockNumber], references: [chainId, number])
-  outputs Output[]
 
   @@unique([chainId, hash])
 }
 
-model Output {
-  id               Int      @id @default(autoincrement())
-  chainId          Int
-  txHash           String
-  index            String
-  consumedByTxHash String?
-  consumedByIndex  String?
-  capacity         BigInt
-  lockScriptHash   String   @db.Char(66)
-  typeScriptHash   String?  @db.Char(66)
-  isLive           Boolean  @default(true)
-  rgbppBound       Boolean  @default(false)
-  boundBtcTxId     String?
-  boundBtcTxIndex  Int?
-  createdAt        DateTime @default(now())
-  updatedAt        DateTime @updatedAt
-
-  transaction Transaction @relation(fields: [chainId, txHash], references: [chainId, hash])
-
-  @@unique([chainId, txHash, index])
-}
-
 model LockScript {
   id            Int      @id @default(autoincrement())
   chainId       Int

backend/src/bootstrap.service.ts

@@ -16,7 +16,7 @@ export class BootstrapService {
     for (const chain of chains) {
       this.logger.log(`Indexing assets for chain ${chain.name}`);
       const indexerService = await this.IndexerServiceFactory.getService(chain.id);
-      await indexerService.startAssetsIndexing();
+      await indexerService.start();
     }
   }
 }

backend/src/core/blockchain/blockchain.service.ts

@@ -96,8 +96,17 @@ export class BlockchainService {
   }
 
   @Cacheable({
-    namespace: 'CkbRpcWebsocketService',
-    key: (txHash: string) => `getTransaction:${txHash}`,
+    namespace: 'BlockchainService',
+    key: (txHash: string, withData: boolean, withWitness: boolean) => {
+      let key = `getTransaction:${txHash}`;
+      if (withData) {
+        key += ':withData';
+      }
+      if (withWitness) {
+        key += ':withWitness';
+      }
+      return key;
+    },
     ttl: ONE_MONTH_MS,
     shouldCache: async (tx: TransactionWithStatusResponse, that: BlockchainService) => {
       if (tx.tx_status.status !== 'committed' || !tx.tx_status.block_number) {
@@ -106,36 +115,132 @@ export class BlockchainService {
       return that.isSafeConfirmations(tx.tx_status.block_number);
     },
   })
-  public async getTransaction(txHash: string): Promise<TransactionWithStatusResponse> {
+  public async getTransaction(
+    txHash: string,
+    withData: boolean = false,
+    withWitness: boolean = false,
+  ): Promise<TransactionWithStatusResponse> {
     await this.websocketReady;
     this.logger.debug(`get_transaction - txHash: ${txHash}`);
-    const tx = await this.websocket.call('get_transaction', [txHash]);
-    return tx as TransactionWithStatusResponse;
+    const response = await this.websocket.call('get_transaction', [txHash]);
+    const tx = response as TransactionWithStatusResponse;
+    // XXX: we don't need these fields by default, remove them to save cache/memory space
+    if (!withData) {
+      tx.transaction.outputs_data = [];
+    }
+    if (!withWitness) {
+      tx.transaction.witnesses = [];
+    }
+    return tx;
   }
 
-  public async getBlock(blockHash: string): Promise<Block> {
+  @Cacheable({
+    namespace: 'BlockchainService',
+    key: (blockHash: string, withTxData: boolean, withTxWitness: boolean) => {
+      let key = `getBlock:${blockHash}`;
+      if (withTxData) {
+        key += ':withTxData';
+      }
+      if (withTxWitness) {
+        key += ':withTxWitness';
+      }
+      return key;
+    },
+    ttl: ONE_MONTH_MS,
+    shouldCache: async (block: Block, that: BlockchainService) => {
+      if (!block?.header) {
+        return false;
+      }
+      const { number } = block.header;
+      return that.isSafeConfirmations(number);
+    },
+  })
+  public async getBlock(
+    blockHash: string,
+    withTxData: boolean = false,
+    withTxWitness: boolean = false,
+  ): Promise<Block> {
     await this.websocketReady;
     this.logger.debug(`get_block - blockHash: ${blockHash}`);
-    const block = await this.websocket.call('get_block', [blockHash]);
-    return block as Block;
+    const response = await this.websocket.call('get_block', [blockHash]);
+    const block = response as Block;
+    if (!withTxData) {
+      block.transactions = block.transactions.map((tx) => {
+        tx.outputs_data = [];
+        return tx;
+      });
+    }
+    if (!withTxWitness) {
+      block.transactions = block.transactions.map((tx) => {
+        tx.witnesses = [];
+        return tx;
+      });
+    }
+    return block;
   }
 
-  public async getBlockByNumber(blockNumber: string): Promise<Block> {
+  @Cacheable({
+    namespace: 'BlockchainService',
+    key: (blockNumber: string, withTxData: boolean, withTxWitness: boolean) => {
+      let key = `getBlockByNumber:${blockNumber}`;
+      if (withTxData) {
+        key += ':withTxData';
+      }
+      if (withTxWitness) {
+        key += ':withTxWitness';
+      }
+      return key;
+    },
+    ttl: ONE_MONTH_MS,
+    shouldCache: async (block: Block, that: BlockchainService) => {
+      const { number } = block.header;
+      return that.isSafeConfirmations(number);
+    },
+  })
+  public async getBlockByNumber(
+    blockNumber: string,
+    withTxData: boolean = false,
+    withTxWitness: boolean = false,
+  ): Promise<Block> {
     await this.websocketReady;
     this.logger.debug(`get_block_by_number - blockNumber: ${blockNumber}`);
-    const block = await this.websocket.call('get_block_by_number', [
+    const response = await this.websocket.call('get_block_by_number', [
       BI.from(blockNumber).toHexString(),
     ]);
-    return block as Block;
+    const block = response as Block;
+    if (!withTxData) {
+      block.transactions = block.transactions.map((tx) => {
+        tx.outputs_data = [];
+        return tx;
+      });
+    }
+    if (!withTxWitness) {
+      block.transactions = block.transactions.map((tx) => {
+        tx.witnesses = [];
+        return tx;
+      });
+    }
+    return block;
   }
 
+  @Cacheable({
+    namespace: 'BlockchainService',
+    key: (blockHash: string) => `getBlockEconomicState:${blockHash}`,
+    ttl: ONE_MONTH_MS,
+  })
   public async getBlockEconomicState(blockHash: string): Promise<BlockEconomicState> {
     await this.websocketReady;
     this.logger.debug(`get_block_economic_state - blockHash: ${blockHash}`);
     const blockEconomicState = await this.websocket.call('get_block_economic_state', [blockHash]);
     return blockEconomicState as BlockEconomicState;
   }
 
+  @Cacheable({
+    namespace: 'BlockchainService',
+    key: 'getTipBlockNumber',
+    // just cache for 1 second to avoid too many requests
+    ttl: 1000,
+  })
   public async getTipBlockNumber(): Promise<number> {
     await this.websocketReady;
     this.logger.debug('get_tip_block_number');
@@ -153,26 +258,35 @@ export class BlockchainService {
     this.logger.debug(
       `get_transactions - searchKey: ${JSON.stringify(searchKey)}, order: ${order}, limit: ${limit}, after: ${after}`,
     );
-    const transactions = await this.websocket.call('get_transactions', [
+    const result = await this.websocket.call('get_transactions', [
       searchKey,
       order,
       limit,
       after,
     ]);
-    return transactions as GetTransactionsResult;
+    const transactions = result as GetTransactionsResult;
+    return transactions;
   }
 
   public async getCells(
     searchKey: SearchKey,
     order: 'asc' | 'desc',
     limit: string,
     after?: string,
+    withData: boolean = false,
   ): Promise<GetCellsResult> {
     await this.websocketReady;
     this.logger.debug(
       `get_cells - searchKey: ${JSON.stringify(searchKey)}, order: ${order}, limit: ${limit}, after: ${after}`,
     );
-    const cells = await this.websocket.call('get_cells', [searchKey, order, limit, after]);
-    return cells as GetCellsResult;
+    const result = await this.websocket.call('get_cells', [searchKey, order, limit, after]);
+    const cells = result as GetCellsResult;
+    cells.objects = cells.objects.map((cell) => {
+      if (!withData) {
+        cell.output_data = '';
+      }
+      return cell;
+    });
+    return cells;
   }
 }