config dictionary query size

packages/node-core/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Add `dictionaryQuerySize` to nodeConfig, so the block range in dictionary can be configurable. (#2139)
 ### Fixed
 - `processedBlockCount` and `schemaMigrationCount` metadata fields incrementing exponentially (#2136)
 

packages/node-core/src/configure/NodeConfig.ts

@@ -33,6 +33,7 @@ export interface IConfig {
   readonly proofOfIndex: boolean;
   readonly ipfs?: string;
   readonly dictionaryTimeout: number;
+  readonly dictionaryQuerySize: number;
   readonly workers?: number;
   readonly profiler?: boolean;
   readonly unsafe?: boolean;
@@ -68,6 +69,7 @@ const DEFAULT_CONFIG = {
   timestampField: true,
   proofOfIndex: false,
   dictionaryTimeout: 30,
+  dictionaryQuerySize: 10000,
   profiler: false,
   subscription: false,
   disableHistorical: false,
@@ -213,6 +215,10 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     return this._config.dictionaryTimeout;
   }
 
+  get dictionaryQuerySize(): number {
+    return this._config.dictionaryQuerySize;
+  }
+
   get ipfs(): string | undefined {
     return this._config.ipfs;
   }

packages/node-core/src/indexer/fetch.service.ts

@@ -17,7 +17,6 @@ import {DynamicDsService} from './dynamic-ds.service';
 import {IProjectService} from './types';
 
 const logger = getLogger('FetchService');
-const DICTIONARY_MAX_QUERY_SIZE = 10000;
 const CHECK_MEMORY_INTERVAL = 60000;
 
 export abstract class BaseFetchService<
@@ -272,7 +271,10 @@ export abstract class BaseFetchService<
         /* queryEndBlock needs to be limited by the latest height or the maximum value of endBlock in datasources.
          * Dictionaries could be in the future depending on if they index unfinalized blocks or the node is using an RPC endpoint that is behind.
          */
-        const queryEndBlock = Math.min(startBlockHeight + DICTIONARY_MAX_QUERY_SIZE, this.latestFinalizedHeight);
+        const queryEndBlock = Math.min(
+          startBlockHeight + this.nodeConfig.dictionaryQuerySize,
+          this.latestFinalizedHeight
+        );
         try {
           const dictionary = await this.dictionaryService.scopedDictionaryEntries(
             startBlockHeight,

packages/node/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- With `dictionary-query-size` now dictionary can config the query block range(#2139)
 
 ## [3.2.0] - 2023-10-31
 ### Fixed

packages/node/src/yargs.ts

@@ -85,6 +85,12 @@ export const yargsOptions = yargs(hideBin(process.argv))
             describe: 'Max timeout for dictionary query',
             type: 'number',
           },
+          'dictionary-query-size': {
+            demandOption: false,
+            describe:
+              'Dictionary query max block size, this specify the block height range of the dictionary query',
+            type: 'number',
+          },
           'disable-historical': {
             demandOption: false,
             describe: 'Disable storing historical state entities',