config dictionary query size

subquery · Nov 4, 2023 · 48f5bf5 · 48f5bf5
1 parent ad2a065
commit 48f5bf5
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 2 deletions.
diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts
@@ -33,6 +33,7 @@ export interface IConfig {
   readonly proofOfIndex: boolean;
   readonly ipfs?: string;
   readonly dictionaryTimeout: number;
+  readonly dictionaryQuerySize: number;
   readonly workers?: number;
   readonly profiler?: boolean;
   readonly unsafe?: boolean;
@@ -68,6 +69,7 @@ const DEFAULT_CONFIG = {
   timestampField: true,
   proofOfIndex: false,
   dictionaryTimeout: 30,
+  dictionaryQuerySize: 10000,
   profiler: false,
   subscription: false,
   disableHistorical: false,
@@ -213,6 +215,10 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     return this._config.dictionaryTimeout;
   }
 
+  get dictionaryQuerySize(): number {
+    return this._config.dictionaryQuerySize;
+  }
+
   get ipfs(): string | undefined {
     return this._config.ipfs;
   }

diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts
@@ -17,7 +17,6 @@ import {DynamicDsService} from './dynamic-ds.service';
 import {IProjectService} from './types';
 
 const logger = getLogger('FetchService');
-const DICTIONARY_MAX_QUERY_SIZE = 10000;
 const CHECK_MEMORY_INTERVAL = 60000;
 
 export abstract class BaseFetchService<
@@ -272,7 +271,11 @@ export abstract class BaseFetchService<
         /* queryEndBlock needs to be limited by the latest height or the maximum value of endBlock in datasources.
          * Dictionaries could be in the future depending on if they index unfinalized blocks or the node is using an RPC endpoint that is behind.
          */
-        const queryEndBlock = Math.min(startBlockHeight + DICTIONARY_MAX_QUERY_SIZE, this.latestFinalizedHeight);
+        this.nodeConfig.dictionaryTimeout;
+        const queryEndBlock = Math.min(
+          startBlockHeight + this.nodeConfig.dictionaryQuerySize,
+          this.latestFinalizedHeight
+        );
         try {
           const dictionary = await this.dictionaryService.scopedDictionaryEntries(
             startBlockHeight,

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -85,6 +85,12 @@ export const yargsOptions = yargs(hideBin(process.argv))
             describe: 'Max timeout for dictionary query',
             type: 'number',
           },
+          'dictionary-query-size': {
+            demandOption: false,
+            describe:
+              'Dictionary query max block size, this specify the block height range of the dictionary query',
+            type: 'number',
+          },
           'disable-historical': {
             demandOption: false,
             describe: 'Disable storing historical state entities',