diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index 0f20656729..8e6f74d5db 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Add `dictionaryQuerySize` to nodeConfig, so the block range in dictionary can be configurable. (#2139) ### Fixed - `processedBlockCount` and `schemaMigrationCount` metadata fields incrementing exponentially (#2136) diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts index f7abbf6d01..0a27beefc0 100644 --- a/packages/node-core/src/configure/NodeConfig.ts +++ b/packages/node-core/src/configure/NodeConfig.ts @@ -33,6 +33,7 @@ export interface IConfig { readonly proofOfIndex: boolean; readonly ipfs?: string; readonly dictionaryTimeout: number; + readonly dictionaryQuerySize: number; readonly workers?: number; readonly profiler?: boolean; readonly unsafe?: boolean; @@ -68,6 +69,7 @@ const DEFAULT_CONFIG = { timestampField: true, proofOfIndex: false, dictionaryTimeout: 30, + dictionaryQuerySize: 10000, profiler: false, subscription: false, disableHistorical: false, @@ -213,6 +215,10 @@ export class NodeConfig implements IConfig { return this._config.dictionaryTimeout; } + get dictionaryQuerySize(): number { + return this._config.dictionaryQuerySize; + } + get ipfs(): string | undefined { return this._config.ipfs; } diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index ec9a579222..a138ed3562 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -17,7 +17,6 @@ import {DynamicDsService} from './dynamic-ds.service'; import {IProjectService} from './types'; const logger = getLogger('FetchService'); -const DICTIONARY_MAX_QUERY_SIZE = 10000; const CHECK_MEMORY_INTERVAL = 60000; export abstract class BaseFetchService< @@ -272,7 +271,10 @@ export abstract class BaseFetchService< /* queryEndBlock needs to be limited by the latest height or the maximum value of endBlock in datasources. * Dictionaries could be in the future depending on if they index unfinalized blocks or the node is using an RPC endpoint that is behind. */ - const queryEndBlock = Math.min(startBlockHeight + DICTIONARY_MAX_QUERY_SIZE, this.latestFinalizedHeight); + const queryEndBlock = Math.min( + startBlockHeight + this.nodeConfig.dictionaryQuerySize, + this.latestFinalizedHeight + ); try { const dictionary = await this.dictionaryService.scopedDictionaryEntries( startBlockHeight, diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index f3abf88eaa..3ec245c87b 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- With `dictionary-query-size` now dictionary can config the query block range(#2139) ## [3.2.0] - 2023-10-31 ### Fixed diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index ccd91e9467..360ba95677 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -85,6 +85,12 @@ export const yargsOptions = yargs(hideBin(process.argv)) describe: 'Max timeout for dictionary query', type: 'number', }, + 'dictionary-query-size': { + demandOption: false, + describe: + 'Dictionary query max block size, this specify the block height range of the dictionary query', + type: 'number', + }, 'disable-historical': { demandOption: false, describe: 'Disable storing historical state entities',