From 4f56615d3984c229d79a20d5ecf8d388e377b53c Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 6 Nov 2024 11:55:28 +1300 Subject: [PATCH 1/2] Allow lazy messages for monitor service --- .../node-core/src/indexer/indexer.manager.ts | 4 ++-- .../node-core/src/indexer/monitor.service.ts | 10 +++++----- packages/node-core/src/indexer/store/store.ts | 20 ++++++++++--------- packages/node-core/src/process.ts | 15 +++++--------- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/node-core/src/indexer/indexer.manager.ts b/packages/node-core/src/indexer/indexer.manager.ts index 92ee79566e..24647b8cae 100644 --- a/packages/node-core/src/indexer/indexer.manager.ts +++ b/packages/node-core/src/indexer/indexer.manager.ts @@ -191,7 +191,7 @@ export abstract class BaseIndexerManager< const parsedData = await this.prepareFilteredData(kind, data, ds); - monitorWrite(`- Handler: ${handler.handler}, args:${handledStringify(data)}`); + monitorWrite(() => `- Handler: ${handler.handler}, args:${handledStringify(data)}`); this.nodeConfig.profiler ? await profilerWrap( vm.securedExec.bind(vm), @@ -210,7 +210,7 @@ export abstract class BaseIndexerManager< for (const handler of handlers) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion vm = vm! ?? (await getVM(ds)); - monitorWrite(`- Handler: ${handler.handler}, args:${handledStringify(data)}`); + monitorWrite(() => `- Handler: ${handler.handler}, args:${handledStringify(data)}`); await this.transformAndExecuteCustomDs(ds, vm, handler, data); } } diff --git a/packages/node-core/src/indexer/monitor.service.ts b/packages/node-core/src/indexer/monitor.service.ts index 45b3d84889..91b716b4f0 100644 --- a/packages/node-core/src/indexer/monitor.service.ts +++ b/packages/node-core/src/indexer/monitor.service.ts @@ -35,7 +35,7 @@ interface FileStat { } export interface MonitorServiceInterface { - write(blockData: string): void; + write(blockData: string | (() => string)): void; createBlockFork(blockHeight: number): void; createBlockStart(blockHeight: number): void; } @@ -212,16 +212,16 @@ export class MonitorService implements MonitorServiceInterface { /** * Write block record data to file - * @param blockData + * @param blockData string or function that returns a string, this allows lazy interpolation for any heavy stringification */ - write(blockData: string): void { + write(blockData: string | (() => string)): void { if (this.monitorFileSize <= 0) { return; } this.checkAndSwitchFile(); - const escapedBlockData = blockData.replace(/\n/g, '\\n'); + const escapedBlockData = (typeof blockData === 'string' ? blockData : blockData()).replace(/\n/g, '\\n'); fs.appendFileSync(this.getFilePath(this.currentFile), `${escapedBlockData}\n`); - this.currentFileSize += Buffer.byteLength(blockData) + 1; // + 1 for the new line + this.currentFileSize += Buffer.byteLength(escapedBlockData) + 1; // + 1 for the new line this.currentFileLastLine += 1; } diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index d7bf92516b..be13122cf6 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -40,7 +40,7 @@ export class Store implements IStore { async get(entity: string, id: string): Promise { try { const raw = await this.#storeCache.getModel(entity).get(id); - monitorWrite(`-- [Store][get] Entity ${entity} ID ${id}, data: ${handledStringify(raw)}`); + monitorWrite(() => `-- [Store][get] Entity ${entity} ID ${id}, data: ${handledStringify(raw)}`); return EntityClass.create(entity, raw, this); } catch (e) { throw new Error(`Failed to get Entity ${entity} with id ${id}: ${e}`); @@ -60,7 +60,7 @@ export class Store implements IStore { this.#queryLimitCheck('getByField', entity, options); const raw = await this.#storeCache.getModel(entity).getByField(field, value, options); - monitorWrite(`-- [Store][getByField] Entity ${entity}, data: ${handledStringify(raw)}`); + monitorWrite(() => `-- [Store][getByField] Entity ${entity}, data: ${handledStringify(raw)}`); return raw.map((v) => EntityClass.create(entity, v, this)) as T[]; } catch (e) { throw new Error(`Failed to getByField Entity ${entity} with field ${String(field)}: ${e}`); @@ -84,7 +84,7 @@ export class Store implements IStore { this.#queryLimitCheck('getByFields', entity, options); const raw = await this.#storeCache.getModel(entity).getByFields(filter, options); - monitorWrite(`-- [Store][getByFields] Entity ${entity}, data: ${handledStringify(raw)}`); + monitorWrite(() => `-- [Store][getByFields] Entity ${entity}, data: ${handledStringify(raw)}`); return raw.map((v) => EntityClass.create(entity, v, this)) as T[]; } catch (e) { throw new Error(`Failed to getByFields Entity ${entity}: ${e}`); @@ -96,7 +96,7 @@ export class Store implements IStore { const indexed = this.#context.isIndexedHistorical(entity, field as string); assert(indexed, `to query by field ${String(field)}, a unique index must be created on model ${entity}`); const raw = await this.#storeCache.getModel(entity).getOneByField(field, value); - monitorWrite(`-- [Store][getOneByField] Entity ${entity}, data: ${handledStringify(raw)}`); + monitorWrite(() => `-- [Store][getOneByField] Entity ${entity}, data: ${handledStringify(raw)}`); return EntityClass.create(entity, raw, this); } catch (e) { throw new Error(`Failed to getOneByField Entity ${entity} with field ${String(field)}: ${e}`); @@ -108,7 +108,7 @@ export class Store implements IStore { try { this.#storeCache.getModel(entity).set(_id, data, this.#context.blockHeight); monitorWrite( - `-- [Store][set] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` + () => `-- [Store][set] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` ); this.#context.operationStack?.put(OperationType.Set, entity, data); } catch (e) { @@ -123,7 +123,8 @@ export class Store implements IStore { this.#context.operationStack?.put(OperationType.Set, entity, item); } monitorWrite( - `-- [Store][bulkCreate] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` + () => + `-- [Store][bulkCreate] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` ); } catch (e) { throw new Error(`Failed to bulkCreate Entity ${entity}: ${e}`); @@ -138,7 +139,8 @@ export class Store implements IStore { this.#context.operationStack?.put(OperationType.Set, entity, item); } monitorWrite( - `-- [Store][bulkUpdate] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` + () => + `-- [Store][bulkUpdate] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` ); } catch (e) { throw new Error(`Failed to bulkCreate Entity ${entity}: ${e}`); @@ -149,7 +151,7 @@ export class Store implements IStore { try { this.#storeCache.getModel(entity).remove(id, this.#context.blockHeight); this.#context.operationStack?.put(OperationType.Remove, entity, id); - monitorWrite(`-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, id: ${id}`); + monitorWrite(() => `-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, id: ${id}`); } catch (e) { throw new Error(`Failed to remove Entity ${entity} with id ${id}: ${e}`); } @@ -163,7 +165,7 @@ export class Store implements IStore { this.#context.operationStack?.put(OperationType.Remove, entity, id); } monitorWrite( - `-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, ids: ${handledStringify(ids)}` + () => `-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, ids: ${handledStringify(ids)}` ); } catch (e) { throw new Error(`Failed to bulkRemove Entity ${entity}: ${e}`); diff --git a/packages/node-core/src/process.ts b/packages/node-core/src/process.ts index 3630034d54..0c4eef78da 100644 --- a/packages/node-core/src/process.ts +++ b/packages/node-core/src/process.ts @@ -17,20 +17,15 @@ export function exitWithError(error: Error | string, logger?: Pino.Logger, code process.exit(code); } -export function monitorWrite(blockData: string): void { - if (monitorService) { - monitorService.write(blockData); - } +// Function argument is to allow for lazy evauluation only if monitor service is enabled +export function monitorWrite(blockData: string | (() => string)): void { + monitorService?.write(blockData); } export function monitorCreateBlockStart(blockNumber: number): void { - if (monitorService) { - monitorService.createBlockStart(blockNumber); - } + monitorService?.createBlockStart(blockNumber); } export function monitorCreateBlockFork(blockNumber: number): void { - if (monitorService) { - monitorService.createBlockFork(blockNumber); - } + monitorService?.createBlockFork(blockNumber); } From aabdb2c01f2b6db80b2dc22dc867feed82ed3f29 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 6 Nov 2024 12:00:41 +1300 Subject: [PATCH 2/2] Update changelog --- packages/node-core/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index d6964f56d8..e1c9e57db7 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- lazy loading for monitor service (#2583) ## [14.1.7] - 2024-10-30 ### Changed