From a58f4e11cee207649eec8eec753e6c6cdb93398d Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 30 Sep 2024 09:00:21 +1300 Subject: [PATCH 01/39] WIP implment non cache metadata --- .../src/indexer/dynamic-ds.service.ts | 21 ++-- .../node-core/src/indexer/store.service.ts | 96 +++++++++---------- .../node-core/src/indexer/storeCache/index.ts | 2 +- .../{ => metadata}/cacheMetadata.spec.ts | 0 .../{ => metadata}/cacheMetadata.test.ts | 46 ++++----- .../{ => metadata}/cacheMetadata.ts | 62 +++++------- .../src/indexer/storeCache/metadata/index.ts | 5 + .../indexer/storeCache/metadata/metadata.ts | 93 ++++++++++++++++++ .../src/indexer/storeCache/metadata/utils.ts | 37 +++++++ .../indexer/storeCache/storeCache.service.ts | 36 +++---- 10 files changed, 258 insertions(+), 140 deletions(-) rename packages/node-core/src/indexer/storeCache/{ => metadata}/cacheMetadata.spec.ts (100%) rename packages/node-core/src/indexer/storeCache/{ => metadata}/cacheMetadata.test.ts (65%) rename packages/node-core/src/indexer/storeCache/{ => metadata}/cacheMetadata.ts (77%) create mode 100644 packages/node-core/src/indexer/storeCache/metadata/index.ts create mode 100644 packages/node-core/src/indexer/storeCache/metadata/metadata.ts create mode 100644 packages/node-core/src/indexer/storeCache/metadata/utils.ts diff --git a/packages/node-core/src/indexer/dynamic-ds.service.ts b/packages/node-core/src/indexer/dynamic-ds.service.ts index 8203dc96f9..169b11c849 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.ts @@ -1,11 +1,11 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {cloneDeep} from 'lodash'; -import {getLogger} from '../logger'; -import {exitWithError} from '../process'; -import {CacheMetadataModel} from './storeCache/cacheMetadata'; -import {ISubqueryProject} from './types'; +import { cloneDeep } from 'lodash'; +import { getLogger } from '../logger'; +import { exitWithError } from '../process'; +import { CacheMetadataModel } from './storeCache'; +import { ISubqueryProject } from './types'; const logger = getLogger('dynamic-ds'); @@ -24,8 +24,7 @@ export interface IDynamicDsService { } export abstract class DynamicDsService - implements IDynamicDsService -{ + implements IDynamicDsService { private _metadata?: CacheMetadataModel; private _datasources?: DS[]; private _datasourceParams?: DatasourceParams[]; @@ -81,7 +80,7 @@ export abstract class DynamicDsService[number], 'name'> & {startBlock?: number}>( + protected getTemplate[number], 'name'> & { startBlock?: number }>( templateName: string, startBlock?: number ): T { @@ -121,7 +120,7 @@ export abstract class DynamicDsService { const sql = `SELECT * FROM information_schema.columns WHERE table_schema = ? AND table_name = '_poi' AND column_name = 'projectId'`; - const [result] = await this.sequelize.query(sql, {replacements: [schema]}); + const [result] = await this.sequelize.query(sql, { replacements: [schema] }); return !!result.length; } async getHistoricalStateEnabled(schema: string): Promise { - const {disableHistorical, multiChain} = this.config; + const { disableHistorical, multiChain } = this.config; try { const tableRes = await this.sequelize.query>( `SELECT table_name FROM information_schema.tables where table_schema='${schema}'`, - {type: QueryTypes.SELECT} + { type: QueryTypes.SELECT } ); const metadataTableNames = flatten(tableRes).filter( @@ -311,9 +311,9 @@ export class StoreService { } if (metadataTableNames.length === 1) { - const res = await this.sequelize.query<{key: string; value: boolean | string}>( + const res = await this.sequelize.query<{ key: string; value: boolean | string }>( `SELECT key, value FROM "${schema}"."${metadataTableNames[0]}" WHERE (key = 'historicalStateEnabled' OR key = 'genesisHash')`, - {type: QueryTypes.SELECT} + { type: QueryTypes.SELECT } ); const store = res.reduce( @@ -321,7 +321,7 @@ export class StoreService { total[current.key] = current.value; return total; }, - {} as {[key: string]: string | boolean} + {} as { [key: string]: string | boolean } ); const useHistorical = @@ -478,33 +478,33 @@ async function batchDeleteAndThenUpdate( destroyCompleted ? 0 : model.destroy({ - transaction, - hooks: false, - limit: batchSize, - where: sequelize.where(sequelize.fn('lower', sequelize.col('_block_range')), Op.gt, targetBlockHeight), - }), + transaction, + hooks: false, + limit: batchSize, + where: sequelize.where(sequelize.fn('lower', sequelize.col('_block_range')), Op.gt, targetBlockHeight), + }), updateCompleted ? [0] : model.update( - { - __block_range: sequelize.fn('int8range', sequelize.fn('lower', sequelize.col('_block_range')), null), - }, - { - transaction, - limit: batchSize, - hooks: false, - where: { - [Op.and]: [ - { - __block_range: { - [Op.contains]: targetBlockHeight, - }, + { + __block_range: sequelize.fn('int8range', sequelize.fn('lower', sequelize.col('_block_range')), null), + }, + { + transaction, + limit: batchSize, + hooks: false, + where: { + [Op.and]: [ + { + __block_range: { + [Op.contains]: targetBlockHeight, }, - sequelize.where(sequelize.fn('upper', sequelize.col('_block_range')), Op.not, null), - ], - }, - } - ), + }, + sequelize.where(sequelize.fn('upper', sequelize.col('_block_range')), Op.not, null), + ], + }, + } + ), ]); logger.debug(`${model.name} deleted ${numDestroyRows} records, updated ${numUpdatedRows} records`); if (numDestroyRows === 0) { diff --git a/packages/node-core/src/indexer/storeCache/index.ts b/packages/node-core/src/indexer/storeCache/index.ts index 2cda5d9c25..330386025f 100644 --- a/packages/node-core/src/indexer/storeCache/index.ts +++ b/packages/node-core/src/indexer/storeCache/index.ts @@ -4,4 +4,4 @@ export * from './storeCache.service'; export * from './types'; export * from './cacheModel'; -export * from './cacheMetadata'; +export * from './metadata'; diff --git a/packages/node-core/src/indexer/storeCache/cacheMetadata.spec.ts b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/cacheMetadata.spec.ts rename to packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/cacheMetadata.test.ts b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.test.ts similarity index 65% rename from packages/node-core/src/indexer/storeCache/cacheMetadata.test.ts rename to packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.test.ts index e1025fc87c..3255e07bbf 100644 --- a/packages/node-core/src/indexer/storeCache/cacheMetadata.test.ts +++ b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.test.ts @@ -1,10 +1,10 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Sequelize} from '@subql/x-sequelize'; -import {MetadataFactory, MetadataKeys, MetadataRepo} from '../'; -import {DbOption} from '../../'; -import {CacheMetadataModel} from './cacheMetadata'; +import { Sequelize } from '@subql/x-sequelize'; +import { MetadataFactory, MetadataKeys, MetadataRepo } from '../../'; +import { DbOption } from '../../../'; +import { CacheMetadataModel } from './cacheMetadata'; const option: DbOption = { host: process.env.DB_HOST ?? '127.0.0.1', @@ -49,7 +49,7 @@ describe('cacheMetadata integration', () => { }; afterAll(async () => { - await sequelize.dropSchema(schema, {logging: false}); + await sequelize.dropSchema(schema, { logging: false }); await sequelize.close(); }); @@ -70,18 +70,18 @@ describe('cacheMetadata integration', () => { describe('dynamicDatasources', () => { beforeEach(async () => { // Ensure value exits so we can update it - await metaDataRepo.bulkCreate([{key: 'dynamicDatasources', value: []}], {updateOnDuplicate: ['key', 'value']}); + await metaDataRepo.bulkCreate([{ key: 'dynamicDatasources', value: [] }], { updateOnDuplicate: ['key', 'value'] }); }); it('Appends dynamicDatasources correctly', async () => { - cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); - cacheMetadataModel.setNewDynamicDatasource({templateName: 'bar', startBlock: 2}); - cacheMetadataModel.setNewDynamicDatasource({templateName: 'baz', startBlock: 3}); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'bar', startBlock: 2 }); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'baz', startBlock: 3 }); const expected = [ - {templateName: 'foo', startBlock: 1}, - {templateName: 'bar', startBlock: 2}, - {templateName: 'baz', startBlock: 3}, + { templateName: 'foo', startBlock: 1 }, + { templateName: 'bar', startBlock: 2 }, + { templateName: 'baz', startBlock: 3 }, ]; await flush(); @@ -94,40 +94,40 @@ describe('cacheMetadata integration', () => { }); it('Allows overriding all dynamicDatasources', async () => { - cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); - cacheMetadataModel.set('dynamicDatasources', [{templateName: 'bar', startBlock: 2}]); + cacheMetadataModel.set('dynamicDatasources', [{ templateName: 'bar', startBlock: 2 }]); await flush(); const v = await queryMeta('dynamicDatasources'); - expect(v).toEqual([{templateName: 'bar', startBlock: 2}]); + expect(v).toEqual([{ templateName: 'bar', startBlock: 2 }]); }); it('Caches the dynamicDatasources correctly after using set', async () => { - cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); await flush(); const cacheV = await cacheMetadataModel.find('dynamicDatasources'); - expect(cacheV).toEqual([{templateName: 'foo', startBlock: 1}]); + expect(cacheV).toEqual([{ templateName: 'foo', startBlock: 1 }]); - cacheMetadataModel.setNewDynamicDatasource({templateName: 'bar', startBlock: 2}); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'bar', startBlock: 2 }); // await flush(); const cacheV2 = await cacheMetadataModel.find('dynamicDatasources'); expect(cacheV2).toEqual([ - {templateName: 'foo', startBlock: 1}, - {templateName: 'bar', startBlock: 2}, + { templateName: 'foo', startBlock: 1 }, + { templateName: 'bar', startBlock: 2 }, ]); }); it('Uses the correct cache values when using new and set', async () => { - cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); + cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); - cacheMetadataModel.set('dynamicDatasources', [{templateName: 'bar', startBlock: 2}]); + cacheMetadataModel.set('dynamicDatasources', [{ templateName: 'bar', startBlock: 2 }]); const cacheV = await cacheMetadataModel.find('dynamicDatasources'); - expect(cacheV).toEqual([{templateName: 'bar', startBlock: 2}]); + expect(cacheV).toEqual([{ templateName: 'bar', startBlock: 2 }]); }); }); }); diff --git a/packages/node-core/src/indexer/storeCache/cacheMetadata.ts b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts similarity index 77% rename from packages/node-core/src/indexer/storeCache/cacheMetadata.ts rename to packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts index 781f3a4067..65bf263bea 100644 --- a/packages/node-core/src/indexer/storeCache/cacheMetadata.ts +++ b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts @@ -2,20 +2,22 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Transaction} from '@subql/x-sequelize'; -import {hasValue} from '../../utils'; -import {DatasourceParams} from '../dynamic-ds.service'; -import {Metadata, MetadataKeys, MetadataRepo} from '../entities'; -import {Cacheable} from './cacheable'; -import {ICachedModelControl} from './types'; - -type MetadataKey = keyof MetadataKeys; -const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; -type IncrementalMetadataKey = 'processedBlockCount' | 'schemaMigrationCount'; +import { Transaction } from '@subql/x-sequelize'; +import { hasValue } from '../../../utils'; +import { DatasourceParams } from '../../dynamic-ds.service'; +import { Metadata, MetadataKeys, MetadataRepo } from '../../entities'; +import { Cacheable } from '../cacheable'; +import { ICachedModelControl } from '../types'; +import { IMetadata } from './metadata'; +import { MetadataKey, incrementKeys, IncrementalMetadataKey, INCREMENT_QUERY, APPEND_DS_QUERY } from './utils'; + +// type MetadataKey = keyof MetadataKeys; +// const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; +// type IncrementalMetadataKey = 'processedBlockCount' | 'schemaMigrationCount'; const specialKeys: MetadataKey[] = [...incrementKeys, 'dynamicDatasources']; -export class CacheMetadataModel extends Cacheable implements ICachedModelControl { +export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedModelControl { private setCache: Partial = {}; // Needed for dynamic datasources private getCache: Partial = {}; @@ -112,14 +114,8 @@ export class CacheMetadataModel extends Cacheable implements ICachedModelControl assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); await this.model.sequelize.query( - ` - INSERT INTO ${schemaTable} (key, value, "createdAt", "updatedAt") - VALUES ('${key}', '0'::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - ON CONFLICT (key) DO - UPDATE SET value = (COALESCE(${schemaTable}.value->>0)::int + '${amount}')::text::jsonb, - "updatedAt" = CURRENT_TIMESTAMP - WHERE ${schemaTable}.key = '${key}';`, - tx && {transaction: tx} + INCREMENT_QUERY(schemaTable, key, amount), + tx && { transaction: tx } ); } @@ -128,21 +124,9 @@ export class CacheMetadataModel extends Cacheable implements ICachedModelControl assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); - const VALUE = '"value"'; - - const makeSet = (item: DatasourceParams, value: string, index = 1): string => - `jsonb_set(${value}, array[(jsonb_array_length(${VALUE}) + ${index})::text], '${JSON.stringify( - item - )}'::jsonb, true)`; - await this.model.sequelize.query( - ` - UPDATE ${schemaTable} - SET ${VALUE} = ${items.reduce((acc, item, index) => makeSet(item, acc, index + 1), VALUE)}, - "updatedAt" = CURRENT_TIMESTAMP - WHERE ${schemaTable}.key = 'dynamicDatasources'; - `, - tx && {transaction: tx} + APPEND_DS_QUERY(schemaTable, items), + tx && { transaction: tx } ); } @@ -158,7 +142,7 @@ export class CacheMetadataModel extends Cacheable implements ICachedModelControl **/ const val = this.setCache[key]; if (val !== undefined) { - await this.model.bulkCreate([{key, value: val}], {transaction: tx, updateOnDuplicate: ['key', 'value']}); + await this.model.bulkCreate([{ key, value: val }], { transaction: tx, updateOnDuplicate: ['key', 'value'] }); } else if (this.datasourceUpdates.length) { await this.appendDynamicDatasources(this.datasourceUpdates, tx); } @@ -185,7 +169,7 @@ export class CacheMetadataModel extends Cacheable implements ICachedModelControl async runFlush(tx: Transaction, blockHeight?: number): Promise { const ops = Object.entries(this.setCache) .filter(([key]) => !specialKeys.includes(key as MetadataKey)) - .map(([key, value]) => ({key, value} as Metadata)); + .map(([key, value]) => ({ key, value } as Metadata)); const lastProcessedHeightIdx = ops.findIndex((k) => k.key === 'lastProcessedHeight'); if (blockHeight !== undefined && lastProcessedHeightIdx >= 0) { @@ -203,13 +187,13 @@ export class CacheMetadataModel extends Cacheable implements ICachedModelControl updateOnDuplicate: ['key', 'value'], }), this.handleSpecialKeys(tx), - this.model.destroy({where: {key: this.removeCache}}), + this.model.destroy({ where: { key: this.removeCache } }), ]); } // This is current only use for migrate Poi // If concurrent change to cache, please add mutex if needed - bulkRemove(keys: K[]): void { + async bulkRemove(keys: K[], tx: Transaction): Promise { this.removeCache.push(...keys); for (const key of keys) { delete this.setCache[key]; @@ -228,8 +212,8 @@ export class CacheMetadataModel extends Cacheable implements ICachedModelControl newSetCache.lastProcessedHeight = this.setCache.lastProcessedHeight; this.flushableRecordCounter = 1; } - this.setCache = {...newSetCache}; - this.getCache = {...newSetCache}; + this.setCache = { ...newSetCache }; + this.getCache = { ...newSetCache }; this.datasourceUpdates = []; } } diff --git a/packages/node-core/src/indexer/storeCache/metadata/index.ts b/packages/node-core/src/indexer/storeCache/metadata/index.ts new file mode 100644 index 0000000000..961566f4d0 --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/metadata/index.ts @@ -0,0 +1,5 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +export * from './cacheMetadata'; +export { IMetadata } from './metadata'; diff --git a/packages/node-core/src/indexer/storeCache/metadata/metadata.ts b/packages/node-core/src/indexer/storeCache/metadata/metadata.ts new file mode 100644 index 0000000000..a277d52707 --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/metadata/metadata.ts @@ -0,0 +1,93 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import assert from "assert"; +import { hasValue } from "@subql/node-core/utils"; +import { Op, Transaction } from '@subql/x-sequelize'; +import { DatasourceParams } from "../../dynamic-ds.service"; +import { Metadata, MetadataKeys, MetadataRepo } from "../../entities"; +import { APPEND_DS_QUERY, INCREMENT_QUERY } from "./utils"; + + +export type MetadataKey = keyof MetadataKeys; +export const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; +export type IncrementalMetadataKey = 'processedBlockCount' | 'schemaMigrationCount'; + +export interface IMetadata { + find(key: K, fallback?: MetadataKeys[K]): Promise; + findMany(keys: readonly K[]): Promise>; + + set(key: K, value: MetadataKeys[K], tx?: Transaction): Promise; + setBulk(metadata: Metadata[], tx?: Transaction): Promise; + setIncrement(key: IncrementalMetadataKey, amount?: number, tx?: Transaction): Promise; + setNewDynamicDatasource(item: DatasourceParams, tx?: Transaction): Promise; + + bulkRemove(keys: K[], tx?: Transaction): Promise; +} + + + +export class MetadataModel implements IMetadata { + + constructor(readonly model: MetadataRepo) {} + + + async find(key: K, fallback?: MetadataKeys[K]): Promise { + const record = await this.model.findByPk(key); + + return hasValue(record) + ? record.toJSON().value as MetadataKeys[K] + : fallback; + } + + async findMany(keys: readonly K[]): Promise> { + const entries = await this.model.findAll({ + where: { + key: keys, + }, + }); + + return entries.reduce((arr, curr) => { + arr[curr.key as K] = curr.value as MetadataKeys[K]; + return arr; + }, {} as Partial); + } + + async set(key: K, value: MetadataKeys[K], tx?: Transaction): Promise { + throw new Error("Method not implemented."); + } + + async setBulk(metadata: Metadata[], tx?: Transaction): Promise { + throw new Error("Method not implemented."); + } + + async setIncrement(key: IncrementalMetadataKey, amount = 1, tx?: Transaction): Promise { + const schemaTable = this.model.getTableName(); + + assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); + assert(incrementKeys.includes(key), `Key ${key} is not incrementable`); + + await this.model.sequelize.query( + INCREMENT_QUERY(schemaTable, key, amount), + tx && { transaction: tx } + ); + } + + async setNewDynamicDatasource(item: DatasourceParams, tx?: Transaction): Promise { + const schemaTable = this.model.getTableName(); + + assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); + + await this.model.sequelize.query( + APPEND_DS_QUERY(schemaTable, [item]), + tx && { transaction: tx }, + ); + } + + async bulkRemove(keys: K[], tx: Transaction): Promise { + await this.model.destroy({ + where: { key: { [Op.in]: keys } }, + transaction: tx + }); + } +} diff --git a/packages/node-core/src/indexer/storeCache/metadata/utils.ts b/packages/node-core/src/indexer/storeCache/metadata/utils.ts new file mode 100644 index 0000000000..5b22ccb72a --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/metadata/utils.ts @@ -0,0 +1,37 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { DatasourceParams } from "../../dynamic-ds.service"; +import { MetadataKeys } from "../../entities"; + +export type MetadataKey = keyof MetadataKeys; +export const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; +export type IncrementalMetadataKey = 'processedBlockCount' | 'schemaMigrationCount'; + + +type SchemaTable = string | { tableName: string, schema: string, delimiter: string }; + +export function INCREMENT_QUERY(schemaTable: SchemaTable, key: MetadataKey, amount = 1): string { + return `INSERT INTO ${schemaTable} (key, value, "createdAt", "updatedAt") + VALUES ('${key}', '0'::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (key) DO + UPDATE SET value = (COALESCE(${schemaTable}.value->>0)::int + '${amount}')::text::jsonb, + "updatedAt" = CURRENT_TIMESTAMP + WHERE ${schemaTable}.key = '${key}';` +} + +export function APPEND_DS_QUERY(schemaTable: SchemaTable, items: DatasourceParams[]): string { + const VALUE = '"value"'; + + const makeSet = (item: DatasourceParams, value: string, index = 1): string => + `jsonb_set(${value}, array[(jsonb_array_length(${VALUE}) + ${index})::text], '${JSON.stringify( + item + )}'::jsonb, true)`; + + return ` + UPDATE ${schemaTable} + SET ${VALUE} = ${items.reduce((acc, item, index) => makeSet(item, acc, index + 1), VALUE)}, + "updatedAt" = CURRENT_TIMESTAMP + WHERE ${schemaTable}.key = 'dynamicDatasources'; + `; +} diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.ts b/packages/node-core/src/indexer/storeCache/storeCache.service.ts index feefb2c0c7..8807248faf 100644 --- a/packages/node-core/src/indexer/storeCache/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeCache.service.ts @@ -2,23 +2,23 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Injectable} from '@nestjs/common'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {SchedulerRegistry} from '@nestjs/schedule'; -import {DatabaseError, Deferrable, ModelStatic, Sequelize, Transaction} from '@subql/x-sequelize'; -import {sum} from 'lodash'; -import {NodeConfig} from '../../configure'; -import {IndexerEvent} from '../../events'; -import {getLogger} from '../../logger'; -import {exitWithError} from '../../process'; -import {profiler} from '../../profiler'; -import {MetadataRepo, PoiRepo} from '../entities'; -import {BaseCacheService} from './baseCache.service'; -import {CacheMetadataModel} from './cacheMetadata'; -import {CachedModel} from './cacheModel'; -import {CachePoiModel} from './cachePoi'; -import {CsvStoreService} from './csvStore.service'; -import {Exporter, ICachedModel, ICachedModelControl} from './types'; +import { Injectable } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { DatabaseError, Deferrable, ModelStatic, Sequelize, Transaction } from '@subql/x-sequelize'; +import { sum } from 'lodash'; +import { NodeConfig } from '../../configure'; +import { IndexerEvent } from '../../events'; +import { getLogger } from '../../logger'; +import { exitWithError } from '../../process'; +import { profiler } from '../../profiler'; +import { MetadataRepo, PoiRepo } from '../entities'; +import { BaseCacheService } from './baseCache.service'; +import { CachedModel } from './cacheModel'; +import { CachePoiModel } from './cachePoi'; +import { CsvStoreService } from './csvStore.service'; +import { CacheMetadataModel } from './metadata'; +import { Exporter, ICachedModel, ICachedModelControl } from './types'; const logger = getLogger('StoreCacheService'); @@ -119,7 +119,7 @@ export class StoreCacheService extends BaseCacheService { await Promise.all(this.exports.map((f) => f.shutdown())); } - updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void { + updateModels({ modifiedModels, removedModels }: { modifiedModels: ModelStatic[]; removedModels: string[] }): void { modifiedModels.forEach((m) => { this.cachedModels[m.name] = this.createModel(m.name); }); From 58672be69888c22c2d626cca16f6d0d06219434e Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 7 Oct 2024 08:53:30 +1300 Subject: [PATCH 02/39] WIP extracting out interfaces and implementing non-cache versions --- .../configure/ProjectUpgrade.service.spec.ts | 3 +- .../src/configure/ProjectUpgrade.service.ts | 44 +++--- .../SchemaMigration.service.ts | 2 +- .../blockDispatcher/base-block-dispatcher.ts | 94 +++++++----- .../worker-block-dispatcher.ts | 6 +- .../src/indexer/dynamic-ds.service.spec.ts | 4 +- .../src/indexer/dynamic-ds.service.ts | 38 ++--- packages/node-core/src/indexer/poi/index.ts | 1 - .../node-core/src/indexer/poi/poi.service.ts | 22 ++- .../src/indexer/poi/poiSync.service.spec.ts | 3 +- .../src/indexer/poi/poiSync.service.ts | 2 +- .../node-core/src/indexer/store.service.ts | 134 ++++++++++-------- packages/node-core/src/indexer/store/store.ts | 38 ++--- .../indexer/storeCache/baseCache.service.ts | 8 +- .../storeCache/baseStoreModel.service.ts | 45 ++++++ .../node-core/src/indexer/storeCache/index.ts | 3 +- .../storeCache/metadata/cacheMetadata.ts | 51 ++++--- .../indexer/storeCache/metadata/metadata.ts | 39 ++--- .../src/indexer/storeCache/metadata/utils.ts | 9 +- .../storeCache/{ => model}/cacheModel.spec.ts | 74 +++++----- .../storeCache/{ => model}/cacheModel.test.ts | 21 +-- .../storeCache/{ => model}/cacheModel.ts | 123 ++++++---------- .../src/indexer/storeCache/model/index.ts | 5 + .../src/indexer/storeCache/model/model.ts | 79 +++++++++++ .../src/indexer/storeCache/model/utils.ts | 26 ++++ .../storeCache/{ => poi}/cachePoi.spec.ts | 2 +- .../indexer/storeCache/{ => poi}/cachePoi.ts | 15 +- .../src/indexer/storeCache/poi/index.ts | 5 + .../poiModel.ts => storeCache/poi/poi.ts} | 16 ++- .../storeCache/storeCache.service.spec.ts | 51 +++---- .../indexer/storeCache/storeCache.service.ts | 110 +++++++------- .../indexer/storeCache/storeModel.service.ts | 90 ++++++++++++ .../node-core/src/indexer/storeCache/types.ts | 13 -- .../indexer/unfinalizedBlocks.service.spec.ts | 6 +- .../src/indexer/unfinalizedBlocks.service.ts | 40 +++--- .../worker.unfinalizedBlocks.service.ts | 6 +- packages/node-core/src/meta/health.service.ts | 2 +- .../src/subcommands/reindex.service.ts | 6 +- packages/node-core/src/utils/reindex.ts | 6 +- 39 files changed, 744 insertions(+), 498 deletions(-) create mode 100644 packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts rename packages/node-core/src/indexer/storeCache/{ => model}/cacheModel.spec.ts (88%) rename packages/node-core/src/indexer/storeCache/{ => model}/cacheModel.test.ts (96%) rename packages/node-core/src/indexer/storeCache/{ => model}/cacheModel.ts (85%) create mode 100644 packages/node-core/src/indexer/storeCache/model/index.ts create mode 100644 packages/node-core/src/indexer/storeCache/model/model.ts create mode 100644 packages/node-core/src/indexer/storeCache/model/utils.ts rename packages/node-core/src/indexer/storeCache/{ => poi}/cachePoi.spec.ts (99%) rename packages/node-core/src/indexer/storeCache/{ => poi}/cachePoi.ts (86%) create mode 100644 packages/node-core/src/indexer/storeCache/poi/index.ts rename packages/node-core/src/indexer/{poi/poiModel.ts => storeCache/poi/poi.ts} (90%) create mode 100644 packages/node-core/src/indexer/storeCache/storeModel.service.ts diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts b/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts index 36f87a7f07..d1cda85ead 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts @@ -4,6 +4,7 @@ import {SchedulerRegistry} from '@nestjs/schedule'; import {Sequelize} from '@subql/x-sequelize'; import {CacheMetadataModel, ISubqueryProject, StoreCacheService, StoreService} from '../indexer'; +import {IStoreModelService} from '../indexer/storeCache/storeModel.service'; import {NodeConfig} from './NodeConfig'; import {IProjectUpgradeService, ProjectUpgradeService, upgradableSubqueryProject} from './ProjectUpgrade.service'; @@ -289,7 +290,7 @@ describe('Project Upgrades', () => { describe('Upgradable subquery project', () => { let upgradeService: ProjectUpgradeService; let project: ISubqueryProject & IProjectUpgradeService; - let storeCache: StoreCacheService; + let storeCache: IStoreModelService; beforeEach(async () => { storeCache = new StoreCacheService({} as any, {} as any, {} as any, new SchedulerRegistry()); diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.ts b/packages/node-core/src/configure/ProjectUpgrade.service.ts index 6bd66e80e7..571d185c00 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.ts @@ -7,7 +7,14 @@ import {ParentProject} from '@subql/types-core'; import {Sequelize, Transaction} from '@subql/x-sequelize'; import {findLast, last, parseInt} from 'lodash'; import {SchemaMigrationService} from '../db'; -import {CacheMetadataModel, ISubqueryProject, StoreCacheService, StoreService} from '../indexer'; +import { + CacheMetadataModel, + IMetadata, + IStoreModelService, + ISubqueryProject, + StoreCacheService, + StoreService, +} from '../indexer'; import {getLogger} from '../logger'; import {exitWithError, monitorWrite} from '../process'; import {getStartHeight, mainThreadOnly} from '../utils'; @@ -107,27 +114,20 @@ export class ProjectUpgradeService

; private migrationService?: SchemaMigrationService; - private constructor( - private _projects: BlockHeightMap

, - currentHeight: number, - private _isRewindable = true - ) { + private constructor(private _projects: BlockHeightMap

, currentHeight: number, private _isRewindable = true) { logger.info( `Projects: ${JSON.stringify( - [..._projects.getAll().entries()].reduce( - (acc, curr) => { - acc[curr[0]] = curr[1].id; - return acc; - }, - {} as Record - ), + [..._projects.getAll().entries()].reduce((acc, curr) => { + acc[curr[0]] = curr[1].id; + return acc; + }, {} as Record), undefined, 2 )}` @@ -172,7 +172,9 @@ export class ProjectUpgradeService

{ + async removeIndexedDeployments(blockHeight: number, tx?: Transaction): Promise { const deployments = await this.getDeploymentsMetadata(); // remove all future block heights @@ -466,6 +468,6 @@ export class ProjectUpgradeService

(sortedSchemaModels, addedModels); const sortedModifiedModels = alignModelOrder(sortedSchemaModels, modifiedModels); - await this.flushCache(true); + await this.storeService.storeCache._flushCache(true); const migrationAction = await Migration.create( this.sequelize, this.storeService, diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index 5ddb106d14..02f8caaec7 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -5,17 +5,18 @@ import assert from 'assert'; import {EventEmitter2, OnEvent} from '@nestjs/event-emitter'; import {hexToU8a, u8aEq} from '@subql/utils'; +import {Transaction} from '@subql/x-sequelize'; import {NodeConfig, IProjectUpgradeService} from '../../configure'; import {AdminEvent, IndexerEvent, PoiEvent, TargetBlockPayload} from '../../events'; import {getLogger} from '../../logger'; -import {exitWithError, monitorCreateBlockFork, monitorCreateBlockStart, monitorWrite} from '../../process'; +import {monitorCreateBlockFork, monitorCreateBlockStart, monitorWrite} from '../../process'; import {IQueue, mainThreadOnly} from '../../utils'; import {MonitorServiceInterface} from '../monitor.service'; import {PoiBlock, PoiSyncService} from '../poi'; import {SmartBatchService} from '../smartBatch.service'; import {StoreService} from '../store.service'; -import {StoreCacheService} from '../storeCache'; -import {CachePoiModel} from '../storeCache/cachePoi'; +import {IStoreModelService} from '../storeCache'; +import {IPoi} from '../storeCache/poi'; import {IBlock, IProjectService, ISubqueryProject} from '../types'; const logger = getLogger('BaseBlockDispatcherService'); @@ -63,7 +64,7 @@ export abstract class BaseBlockDispatcher implements IB private projectUpgradeService: IProjectUpgradeService, protected queue: Q, protected storeService: StoreService, - private storeCacheService: StoreCacheService, + private storeModelService: IStoreModelService, private poiSyncService: PoiSyncService, protected monitorService?: MonitorServiceInterface ) { @@ -75,7 +76,7 @@ export abstract class BaseBlockDispatcher implements IB async init(onDynamicDsCreated: (height: number) => void): Promise { this._onDynamicDsCreated = onDynamicDsCreated; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.setProcessedBlockCount((await this.storeCacheService.metadata.find('processedBlockCount', 0))!); + this.setProcessedBlockCount((await this.storeModelService.metadata.find('processedBlockCount', 0))!); } get queueSize(): number { @@ -198,10 +199,10 @@ export abstract class BaseBlockDispatcher implements IB throw e; } } else { - this.updateStoreMetadata(height); + await this.updateStoreMetadata(height, undefined, this.storeService.transaction); const operationHash = this.storeService.getOperationMerkleRoot(); - this.createPOI(height, blockHash, operationHash); + await this.createPOI(height, blockHash, operationHash, this.storeService.transaction); if (dynamicDsCreated) { this.onDynamicDsCreated(height); @@ -215,21 +216,38 @@ export abstract class BaseBlockDispatcher implements IB this.setLatestProcessedHeight(height); } - if (this.nodeConfig.storeCacheAsync) { - // Flush all completed block data and don't wait - await this.storeCacheService.flushAndWaitForCapacity(false)?.catch((e) => { - exitWithError(new Error(`Flushing cache failed`, {cause: e}), logger); - }); - } else { - // Flush all data from cache and wait - await this.storeCacheService.flushCache(false); - } - - if (!this.projectService.hasDataSourcesAfterHeight(height)) { - const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; - await this.storeCacheService.flushCache(false); - exitWithError(msg, logger, 0); - } + await this.storeModelService.applyPendingChanges(height, !this.projectService.hasDataSourcesAfterHeight(height)); + + // if (this.storeModelService instanceof StoreCacheService) { + // if (this.nodeConfig.storeCacheAsync) { + // // Flush all completed block data and don't wait + // await this.storeModelService.flushAndWaitForCapacity(false)?.catch((e) => { + // exitWithError(new Error(`Flushing cache failed`, { cause: e }), logger); + // }); + // } else { + // // Flush all data from cache and wait + // await this.storeModelService.flushCache(false); + // } + + // if (!this.projectService.hasDataSourcesAfterHeight(height)) { + // const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; + // await this.storeModelService.flushCache(false); + // exitWithError(msg, logger, 0); + // } + // } else if (this.storeModelService instanceof PlainStoreModelService) { + // const tx = this.storeService.transaction; + // if (!tx) { + // exitWithError(new Error('Transaction not found'), logger, 1); + // } + // await tx.commit(); + + // if (!this.projectService.hasDataSourcesAfterHeight(height)) { + // const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; + // exitWithError(msg, logger, 0); + // } + // } else { + // exitWithError(new Error('Unknown store model service'), logger, 1); + // } } @OnEvent(AdminEvent.rewindTarget) @@ -258,7 +276,12 @@ export abstract class BaseBlockDispatcher implements IB * @param operationHash * @private */ - private createPOI(height: number, blockHash: string, operationHash: Uint8Array): void { + private async createPOI( + height: number, + blockHash: string, + operationHash: Uint8Array, + tx?: Transaction + ): Promise { if (!this.nodeConfig.proofOfIndex) { return; } @@ -268,8 +291,8 @@ export abstract class BaseBlockDispatcher implements IB const poiBlock = PoiBlock.create(height, blockHash, operationHash, this.project.id); // This is the first creation of POI - this.poi.bulkUpsert([poiBlock]); - this.storeCacheService.metadata.setBulk([{key: 'lastCreatedPoiHeight', value: height}]); + await this.poi.bulkUpsert([poiBlock], tx); + await this.storeModelService.metadata.setBulk([{key: 'lastCreatedPoiHeight', value: height}], tx); this.eventEmitter.emit(PoiEvent.PoiTarget, { height, timestamp: Date.now(), @@ -277,21 +300,24 @@ export abstract class BaseBlockDispatcher implements IB } @mainThreadOnly() - private updateStoreMetadata(height: number, updateProcessed = true): void { - const meta = this.storeCacheService.metadata; + private async updateStoreMetadata(height: number, updateProcessed = true, tx?: Transaction): Promise { + const meta = this.storeModelService.metadata; // Update store metadata - meta.setBulk([ - {key: 'lastProcessedHeight', value: height}, - {key: 'lastProcessedTimestamp', value: Date.now()}, - ]); + await meta.setBulk( + [ + {key: 'lastProcessedHeight', value: height}, + {key: 'lastProcessedTimestamp', value: Date.now()}, + ], + tx + ); // Db Metadata increase BlockCount, in memory ref to block-dispatcher _processedBlockCount if (updateProcessed) { - meta.setIncrement('processedBlockCount'); + await meta.setIncrement('processedBlockCount', undefined, tx); } } - private get poi(): CachePoiModel { - const poi = this.storeCacheService.poi; + private get poi(): IPoi { + const poi = this.storeModelService.poi; if (!poi) { throw new Error('Poi service expected poi repo but it was not found'); } diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index 00b1e86c71..a271d1e8c2 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -15,7 +15,7 @@ import {monitorWrite} from '../../process'; import {AutoQueue, isTaskFlushedError} from '../../utils'; import {MonitorServiceInterface} from '../monitor.service'; import {StoreService} from '../store.service'; -import {StoreCacheService} from '../storeCache'; +import {IStoreModelService, StoreCacheService} from '../storeCache'; import {ISubqueryProject, IProjectService} from '../types'; import {isBlockUnavailableError} from '../worker/utils'; import {BaseBlockDispatcher} from './base-block-dispatcher'; @@ -58,7 +58,7 @@ export abstract class WorkerBlockDispatcher projectService: IProjectService, projectUpgradeService: IProjectUpgradeService, storeService: StoreService, - storeCacheService: StoreCacheService, + storeModelService: IStoreModelService, poiSyncService: PoiSyncService, project: ISubqueryProject, private createIndexerWorker: () => Promise, @@ -72,7 +72,7 @@ export abstract class WorkerBlockDispatcher projectUpgradeService, initAutoQueue(nodeConfig.workers, nodeConfig.batchSize, nodeConfig.timeout, 'Worker'), storeService, - storeCacheService, + storeModelService, poiSyncService, monitorService ); diff --git a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts index 3d242cda57..fc3b0807bb 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts @@ -69,7 +69,7 @@ describe('DynamicDsService', () => { const meta = mockMetadata([testParam1, testParam2, testParam3, testParam4]); await service.init(meta); - await service.resetDynamicDatasource(2); + await service.resetDynamicDatasource(2, null as any); await expect(meta.find('dynamicDatasources')).resolves.toEqual([testParam1, testParam2]); await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1, testParam2]); @@ -79,7 +79,7 @@ describe('DynamicDsService', () => { const meta = mockMetadata([testParam1, testParam2]); await service.init(meta); - meta.set('dynamicDatasources', [testParam1, testParam2, testParam3, testParam4]); + await meta.set('dynamicDatasources', [testParam1, testParam2, testParam3, testParam4]); await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1, testParam2]); await expect(service.getDynamicDatasources(true)).resolves.toEqual([ diff --git a/packages/node-core/src/indexer/dynamic-ds.service.ts b/packages/node-core/src/indexer/dynamic-ds.service.ts index 169b11c849..5e214ebd49 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.ts @@ -1,11 +1,12 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { cloneDeep } from 'lodash'; -import { getLogger } from '../logger'; -import { exitWithError } from '../process'; -import { CacheMetadataModel } from './storeCache'; -import { ISubqueryProject } from './types'; +import {Transaction} from '@subql/x-sequelize'; +import {cloneDeep} from 'lodash'; +import {getLogger} from '../logger'; +import {exitWithError} from '../process'; +import {IMetadata} from './storeCache'; +import {ISubqueryProject} from './types'; const logger = getLogger('dynamic-ds'); @@ -24,8 +25,9 @@ export interface IDynamicDsService { } export abstract class DynamicDsService - implements IDynamicDsService { - private _metadata?: CacheMetadataModel; + implements IDynamicDsService +{ + private _metadata?: IMetadata; private _datasources?: DS[]; private _datasourceParams?: DatasourceParams[]; @@ -33,7 +35,7 @@ export abstract class DynamicDsService { + async init(metadata: IMetadata): Promise { this._metadata = metadata; await this.getDynamicDatasources(true); @@ -46,7 +48,7 @@ export abstract class DynamicDsService { + async resetDynamicDatasource(targetHeight: number, tx: Transaction): Promise { if (this._datasourceParams && this._datasourceParams.length !== 0) { const filteredDs = this._datasourceParams.filter((ds) => ds.startBlock <= targetHeight); - this.metadata.set(METADATA_KEY, filteredDs); + await this.metadata.set(METADATA_KEY, filteredDs, tx); await this.loadDynamicDatasources(filteredDs); } } - async createDynamicDatasource(params: DatasourceParams): Promise { + // TODO make tx required + async createDynamicDatasource(params: DatasourceParams, tx?: Transaction): Promise { try { const ds = await this.getDatasource(params); - this.metadata.setNewDynamicDatasource(params); + await this.metadata.setNewDynamicDatasource(params, tx); logger.info(`Created new dynamic datasource from template: "${params.templateName}"`); @@ -80,7 +82,7 @@ export abstract class DynamicDsService[number], 'name'> & { startBlock?: number }>( + protected getTemplate[number], 'name'> & {startBlock?: number}>( templateName: string, startBlock?: number ): T { @@ -120,7 +122,7 @@ export abstract class DynamicDsService { - let completed = false; // eslint-disable-next-line no-constant-condition - while (!completed) { + while (true) { try { const recordsToDelete = await model.findAll({ transaction, @@ -207,7 +202,6 @@ async function batchDeletePoi( }); if (recordsToDelete.length === 0) { break; - completed = true; } logger.debug(`Found Poi recordsToDelete ${recordsToDelete.length}`); if (recordsToDelete.length) { diff --git a/packages/node-core/src/indexer/poi/poiSync.service.spec.ts b/packages/node-core/src/indexer/poi/poiSync.service.spec.ts index 580ca00b31..d154b22777 100644 --- a/packages/node-core/src/indexer/poi/poiSync.service.spec.ts +++ b/packages/node-core/src/indexer/poi/poiSync.service.spec.ts @@ -6,8 +6,9 @@ import {delay} from '@subql/common'; import {Sequelize} from '@subql/x-sequelize'; import {range} from 'lodash'; import {NodeConfig} from '../../configure'; -import {MetadataFactory, PlainPoiModel, PoiFactory, ProofOfIndex} from '../../indexer'; +import {MetadataFactory, PoiFactory, ProofOfIndex} from '../../indexer'; import {Queue} from '../../utils'; +import {PlainPoiModel} from '../storeCache/poi'; import {ISubqueryProject} from '../types'; import {PoiSyncService} from './poiSync.service'; diff --git a/packages/node-core/src/indexer/poi/poiSync.service.ts b/packages/node-core/src/indexer/poi/poiSync.service.ts index 75f59c4c51..fb4e3451d3 100644 --- a/packages/node-core/src/indexer/poi/poiSync.service.ts +++ b/packages/node-core/src/indexer/poi/poiSync.service.ts @@ -14,9 +14,9 @@ import {exitWithError} from '../../process'; import {hasValue, Queue} from '../../utils'; import {Metadata, MetadataFactory, MetadataRepo} from '../entities'; import {PoiFactory, ProofOfIndex, SyncedProofOfIndex} from '../entities/Poi.entity'; +import {PlainPoiModel} from '../storeCache/poi'; import {ISubqueryProject} from '../types'; import {PoiBlock} from './PoiBlock'; -import {PlainPoiModel} from './poiModel'; const GENESIS_PARENT_HASH = hexToU8a('0x00'); const logger = getLogger('PoiSyncService'); diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index f997ec5b73..37b568cce2 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -2,9 +2,9 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import { Inject, Injectable } from '@nestjs/common'; -import { getDbType, SUPPORT_DB } from '@subql/common'; -import { IProjectNetworkConfig } from '@subql/types-core'; +import {Inject, Injectable} from '@nestjs/common'; +import {getDbType, SUPPORT_DB} from '@subql/common'; +import {IProjectNetworkConfig} from '@subql/types-core'; import { GraphQLModelsRelationsEnums, hashName, @@ -13,9 +13,18 @@ import { hexToU8a, GraphQLModelsType, } from '@subql/utils'; -import { IndexesOptions, ModelAttributes, ModelStatic, Op, QueryTypes, Sequelize, Transaction } from '@subql/x-sequelize'; -import { camelCase, flatten, last, upperFirst } from 'lodash'; -import { NodeConfig } from '../configure'; +import { + IndexesOptions, + ModelAttributes, + ModelStatic, + Op, + QueryTypes, + Sequelize, + Transaction, + Deferrable, +} from '@subql/x-sequelize'; +import {camelCase, flatten, last, upperFirst} from 'lodash'; +import {NodeConfig} from '../configure'; import { BTREE_GIST_EXTENSION_EXIST_QUERY, createSchemaTrigger, @@ -24,15 +33,14 @@ import { getTriggers, SchemaMigrationService, } from '../db'; -import { getLogger } from '../logger'; -import { exitWithError } from '../process'; -import { camelCaseObjectKey, customCamelCaseGraphqlKey } from '../utils'; -import { MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo } from './entities'; -import { Store } from './store'; -import { IMetadata } from './storeCache'; -import { StoreCacheService } from './storeCache/storeCache.service'; -import { StoreOperations } from './StoreOperations'; -import { ISubqueryProject } from './types'; +import {getLogger} from '../logger'; +import {exitWithError} from '../process'; +import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils'; +import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities'; +import {Store} from './store'; +import {IMetadata, IStoreModelService} from './storeCache'; +import {StoreOperations} from './StoreOperations'; +import {ISubqueryProject} from './types'; const logger = getLogger('StoreService'); const NULL_MERKEL_ROOT = hexToU8a('0x00'); @@ -66,10 +74,12 @@ export class StoreService { private _operationStack?: StoreOperations; private _lastTimeDbSizeChecked?: number; + #transaction?: Transaction; + constructor( private sequelize: Sequelize, private config: NodeConfig, - readonly storeCache: StoreCacheService, + readonly storeCache: IStoreModelService, @Inject('ISubqueryProject') private subqueryProject: ISubqueryProject ) {} @@ -106,6 +116,10 @@ export class StoreService { return this._historical; } + get transaction(): Transaction | undefined { + return this.#transaction; + } + async syncDbSize(): Promise { if (!this._lastTimeDbSizeChecked || Date.now() - this._lastTimeDbSizeChecked > DB_SIZE_CACHE_TIMEOUT) { this._lastTimeDbSizeChecked = Date.now(); @@ -169,7 +183,7 @@ export class StoreService { await this.initHotSchemaReloadQueries(schema); - this.metadataModel.set('historicalStateEnabled', this.historical); + await this.metadataModel.set('historicalStateEnabled', this.historical); } async init(schema: string): Promise { @@ -205,10 +219,10 @@ export class StoreService { last(Object.values(deployments)) !== this.subqueryProject.id ) { // TODO this should run with the same db transaction as the migration - this.metadataModel.setIncrement('schemaMigrationCount'); + await this.metadataModel.setIncrement('schemaMigrationCount'); } } catch (e: any) { - exitWithError(new Error(`Having a problem when syncing schema`, { cause: e }), logger); + exitWithError(new Error(`Having a problem when syncing schema`, {cause: e}), logger); } } @@ -242,7 +256,7 @@ export class StoreService { try { this._modelIndexedFields = await this.getAllIndexFields(schema); } catch (e: any) { - exitWithError(new Error(`Having a problem when getting indexed fields`, { cause: e }), logger); + exitWithError(new Error(`Having a problem when getting indexed fields`, {cause: e}), logger); } } @@ -286,17 +300,17 @@ export class StoreService { private async useDeprecatePoi(schema: string): Promise { const sql = `SELECT * FROM information_schema.columns WHERE table_schema = ? AND table_name = '_poi' AND column_name = 'projectId'`; - const [result] = await this.sequelize.query(sql, { replacements: [schema] }); + const [result] = await this.sequelize.query(sql, {replacements: [schema]}); return !!result.length; } async getHistoricalStateEnabled(schema: string): Promise { - const { disableHistorical, multiChain } = this.config; + const {disableHistorical, multiChain} = this.config; try { const tableRes = await this.sequelize.query>( `SELECT table_name FROM information_schema.tables where table_schema='${schema}'`, - { type: QueryTypes.SELECT } + {type: QueryTypes.SELECT} ); const metadataTableNames = flatten(tableRes).filter( @@ -311,18 +325,15 @@ export class StoreService { } if (metadataTableNames.length === 1) { - const res = await this.sequelize.query<{ key: string; value: boolean | string }>( + const res = await this.sequelize.query<{key: string; value: boolean | string}>( `SELECT key, value FROM "${schema}"."${metadataTableNames[0]}" WHERE (key = 'historicalStateEnabled' OR key = 'genesisHash')`, - { type: QueryTypes.SELECT } + {type: QueryTypes.SELECT} ); - const store = res.reduce( - function (total, current) { - total[current.key] = current.value; - return total; - }, - {} as { [key: string]: string | boolean } - ); + const store = res.reduce(function (total, current) { + total[current.key] = current.value; + return total; + }, {} as {[key: string]: string | boolean}); const useHistorical = store.historicalStateEnabled === undefined ? !disableHistorical : (store.historicalStateEnabled as boolean); @@ -345,8 +356,15 @@ export class StoreService { return !disableHistorical; } } - setBlockHeight(blockHeight: number): void { + async setBlockHeight(blockHeight: number): Promise { this._blockHeight = blockHeight; + + // TODO do we need to set hooks for block height? + this.#transaction = await this.sequelize.transaction({ + deferrable: this._historical || this.dbType === SUPPORT_DB.cockRoach ? undefined : Deferrable.SET_DEFERRED(), + }); + this.#transaction.afterCommit(() => (this.#transaction = undefined)); + if (this.config.proofOfIndex) { this.operationStack = new StoreOperations(this.modelsRelations.models); } @@ -422,14 +440,14 @@ group by // This should only been called from CLI, blockHeight in storeService never been set and is required for`beforeFind` hook // Height no need to change for rewind during indexing if (this._blockHeight === undefined) { - this.setBlockHeight(targetBlockHeight); + await this.setBlockHeight(targetBlockHeight); } for (const model of Object.values(this.sequelize.models)) { if ('__block_range' in model.getAttributes()) { await batchDeleteAndThenUpdate(this.sequelize, model, transaction, targetBlockHeight); } } - this.metadataModel.set('lastProcessedHeight', targetBlockHeight); + await this.metadataModel.set('lastProcessedHeight', targetBlockHeight, transaction); // metadataModel will be flushed in reindex.ts#reindex() } @@ -478,33 +496,33 @@ async function batchDeleteAndThenUpdate( destroyCompleted ? 0 : model.destroy({ - transaction, - hooks: false, - limit: batchSize, - where: sequelize.where(sequelize.fn('lower', sequelize.col('_block_range')), Op.gt, targetBlockHeight), - }), + transaction, + hooks: false, + limit: batchSize, + where: sequelize.where(sequelize.fn('lower', sequelize.col('_block_range')), Op.gt, targetBlockHeight), + }), updateCompleted ? [0] : model.update( - { - __block_range: sequelize.fn('int8range', sequelize.fn('lower', sequelize.col('_block_range')), null), - }, - { - transaction, - limit: batchSize, - hooks: false, - where: { - [Op.and]: [ - { - __block_range: { - [Op.contains]: targetBlockHeight, - }, - }, - sequelize.where(sequelize.fn('upper', sequelize.col('_block_range')), Op.not, null), - ], + { + __block_range: sequelize.fn('int8range', sequelize.fn('lower', sequelize.col('_block_range')), null), }, - } - ), + { + transaction, + limit: batchSize, + hooks: false, + where: { + [Op.and]: [ + { + __block_range: { + [Op.contains]: targetBlockHeight, + }, + }, + sequelize.where(sequelize.fn('upper', sequelize.col('_block_range')), Op.not, null), + ], + }, + } + ), ]); logger.debug(`${model.name} deleted ${numDestroyRows} records, updated ${numUpdatedRows} records`); if (numDestroyRows === 0) { diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index 4b335ef98d..173f6c99a6 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -3,11 +3,12 @@ import assert from 'assert'; import {Store as IStore, Entity, FieldsExpression, GetOptions} from '@subql/types-core'; +import {Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {getLogger} from '../../logger'; import {monitorWrite} from '../../process'; import {handledStringify} from '../../utils'; -import {StoreCacheService} from '../storeCache'; +import {IStoreModelService} from '../storeCache'; import {StoreOperations} from '../StoreOperations'; import {OperationType} from '../types'; import {EntityClass} from './entity'; @@ -17,6 +18,7 @@ const logger = getLogger('Store'); /* A context is provided to allow it to be updated by the owner of the class instance */ type Context = { blockHeight: number; + transaction?: Transaction; operationStack?: StoreOperations; isIndexed: (entity: string, field: string) => boolean; isIndexedHistorical: (entity: string, field: string) => boolean; @@ -25,10 +27,10 @@ type Context = { export class Store implements IStore { /* These need to explicily be private using JS style private properties in order to not leak these in the sandbox */ #config: NodeConfig; - #storeCache: StoreCacheService; + #storeCache: IStoreModelService; #context: Context; - constructor(config: NodeConfig, storeCache: StoreCacheService, context: Context) { + constructor(config: NodeConfig, storeCache: IStoreModelService, context: Context) { this.#config = config; this.#storeCache = storeCache; this.#context = context; @@ -70,7 +72,9 @@ export class Store implements IStore { this.#queryLimitCheck('getByField', entity, options); - const raw = await this.#storeCache.getModel(entity).getByField(field, value, options); + const raw = await this.#storeCache + .getModel(entity) + .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], options); monitorWrite(`-- [Store][getByField] Entity ${entity}, data: ${handledStringify(raw)}`); return raw.map((v) => EntityClass.create(entity, v, this)) as T[]; } catch (e) { @@ -81,7 +85,7 @@ export class Store implements IStore { async getByFields( entity: string, filter: FieldsExpression[], - options?: GetOptions + options: GetOptions ): Promise { try { // Check that the fields are indexed @@ -106,7 +110,9 @@ export class Store implements IStore { try { const indexed = this.#context.isIndexedHistorical(entity, field as string); assert(indexed, `to query by field ${String(field)}, a unique index must be created on model ${entity}`); - const raw = await this.#storeCache.getModel(entity).getOneByField(field, value); + const [raw] = await this.#storeCache + .getModel(entity) + .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], {limit: 1}); monitorWrite(`-- [Store][getOneByField] Entity ${entity}, data: ${handledStringify(raw)}`); return EntityClass.create(entity, raw, this); } catch (e) { @@ -114,10 +120,9 @@ export class Store implements IStore { } } - // eslint-disable-next-line @typescript-eslint/require-await async set(entity: string, _id: string, data: Entity): Promise { try { - this.#storeCache.getModel(entity).set(_id, data, this.#context.blockHeight); + await this.#storeCache.getModel(entity).set(_id, data, this.#context.blockHeight, this.#context.transaction); monitorWrite( `-- [Store][set] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` ); @@ -126,10 +131,10 @@ export class Store implements IStore { throw new Error(`Failed to set Entity ${entity} with _id ${_id}: ${e}`); } } - // eslint-disable-next-line @typescript-eslint/require-await + async bulkCreate(entity: string, data: Entity[]): Promise { try { - this.#storeCache.getModel(entity).bulkCreate(data, this.#context.blockHeight); + await this.#storeCache.getModel(entity).bulkCreate(data, this.#context.blockHeight, this.#context.transaction); for (const item of data) { this.#context.operationStack?.put(OperationType.Set, entity, item); } @@ -141,10 +146,11 @@ export class Store implements IStore { } } - // eslint-disable-next-line @typescript-eslint/require-await async bulkUpdate(entity: string, data: Entity[], fields?: string[]): Promise { try { - this.#storeCache.getModel(entity).bulkUpdate(data, this.#context.blockHeight, fields); + await this.#storeCache + .getModel(entity) + .bulkUpdate(data, this.#context.blockHeight, fields, this.#context.transaction); for (const item of data) { this.#context.operationStack?.put(OperationType.Set, entity, item); } @@ -155,20 +161,20 @@ export class Store implements IStore { throw new Error(`Failed to bulkCreate Entity ${entity}: ${e}`); } } - // eslint-disable-next-line @typescript-eslint/require-await + async remove(entity: string, id: string): Promise { try { - this.#storeCache.getModel(entity).remove(id, this.#context.blockHeight); + await this.#storeCache.getModel(entity).bulkRemove([id], this.#context.blockHeight, this.#context.transaction); this.#context.operationStack?.put(OperationType.Remove, entity, id); monitorWrite(`-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, id: ${id}`); } catch (e) { throw new Error(`Failed to remove Entity ${entity} with id ${id}: ${e}`); } } - // eslint-disable-next-line @typescript-eslint/require-await + async bulkRemove(entity: string, ids: string[]): Promise { try { - this.#storeCache.getModel(entity).bulkRemove(ids, this.#context.blockHeight); + await this.#storeCache.getModel(entity).bulkRemove(ids, this.#context.blockHeight, this.#context.transaction); for (const id of ids) { this.#context.operationStack?.put(OperationType.Remove, entity, id); diff --git a/packages/node-core/src/indexer/storeCache/baseCache.service.ts b/packages/node-core/src/indexer/storeCache/baseCache.service.ts index bc01a30ea9..b0c25cd142 100644 --- a/packages/node-core/src/indexer/storeCache/baseCache.service.ts +++ b/packages/node-core/src/indexer/storeCache/baseCache.service.ts @@ -6,9 +6,14 @@ import Pino from 'pino'; import {getLogger} from '../../logger'; import {profiler} from '../../profiler'; import {timeout} from '../../utils/promise'; +import {BaseStoreModelService} from './baseStoreModel.service'; +import {ICachedModelControl} from './types'; @Injectable() -export abstract class BaseCacheService implements BeforeApplicationShutdown { +export abstract class BaseCacheService + extends BaseStoreModelService + implements BeforeApplicationShutdown +{ private pendingFlush?: Promise; private queuedFlush?: Promise; protected logger: Pino.Logger; @@ -20,6 +25,7 @@ export abstract class BaseCacheService implements BeforeApplicationShutdown { abstract flushExportStores(): Promise; protected constructor(loggerName: string) { + super(); this.logger = getLogger(loggerName); } diff --git a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts new file mode 100644 index 0000000000..e28957b7f1 --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts @@ -0,0 +1,45 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import {ModelStatic} from '@subql/x-sequelize'; +import {MetadataRepo, PoiRepo} from '../entities'; +import {METADATA_ENTITY_NAME} from './metadata/utils'; +import {BaseEntity, IModel} from './model'; +import {POI_ENTITY_NAME} from './poi'; + +export abstract class BaseStoreModelService> { + protected historical = true; + protected poiRepo?: PoiRepo; + protected metadataRepo?: MetadataRepo; + protected cachedModels: Record = {}; + protected useCockroachDb?: boolean; + + protected abstract createModel(entity: string): M; + + init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { + this.historical = historical; + this.metadataRepo = meta; + this.poiRepo = poi; + this.useCockroachDb = useCockroachDb; + } + + getModel(entity: string): IModel { + if (entity === METADATA_ENTITY_NAME) { + throw new Error('Please use getMetadataModel instead'); + } + if (entity === POI_ENTITY_NAME) { + throw new Error('Please use getPoiModel instead'); + } + if (!this.cachedModels[entity]) { + this.cachedModels[entity] = this.createModel(entity); + } + return this.cachedModels[entity] as IModel; + } + + updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void { + modifiedModels.forEach((m) => { + this.cachedModels[m.name] = this.createModel(m.name); + }); + removedModels.forEach((r) => delete this.cachedModels[r]); + } +} diff --git a/packages/node-core/src/indexer/storeCache/index.ts b/packages/node-core/src/indexer/storeCache/index.ts index 330386025f..c70f266067 100644 --- a/packages/node-core/src/indexer/storeCache/index.ts +++ b/packages/node-core/src/indexer/storeCache/index.ts @@ -3,5 +3,6 @@ export * from './storeCache.service'; export * from './types'; -export * from './cacheModel'; +export * from './model'; export * from './metadata'; +export * from './storeModel.service'; diff --git a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts index 65bf263bea..fcac1df9eb 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts +++ b/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts @@ -2,14 +2,14 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import { Transaction } from '@subql/x-sequelize'; -import { hasValue } from '../../../utils'; -import { DatasourceParams } from '../../dynamic-ds.service'; -import { Metadata, MetadataKeys, MetadataRepo } from '../../entities'; -import { Cacheable } from '../cacheable'; -import { ICachedModelControl } from '../types'; -import { IMetadata } from './metadata'; -import { MetadataKey, incrementKeys, IncrementalMetadataKey, INCREMENT_QUERY, APPEND_DS_QUERY } from './utils'; +import {Transaction} from '@subql/x-sequelize'; +import {hasValue} from '../../../utils'; +import {DatasourceParams} from '../../dynamic-ds.service'; +import {Metadata, MetadataKeys, MetadataRepo} from '../../entities'; +import {Cacheable} from '../cacheable'; +import {ICachedModelControl} from '../types'; +import {IMetadata} from './metadata'; +import {MetadataKey, incrementKeys, IncrementalMetadataKey, INCREMENT_QUERY, APPEND_DS_QUERY} from './utils'; // type MetadataKey = keyof MetadataKeys; // const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; @@ -85,7 +85,8 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM return result; } - set(key: K, value: MetadataKeys[K]): void { + // eslint-disable-next-line @typescript-eslint/require-await + async set(key: K, value: MetadataKeys[K], tx?: Transaction): Promise { if (this.setCache[key] === undefined) { this.flushableRecordCounter += 1; } @@ -96,15 +97,18 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM } } - setBulk(metadata: Metadata[]): void { + // eslint-disable-next-line @typescript-eslint/require-await + async setBulk(metadata: Metadata[]): Promise { metadata.map((m) => this.set(m.key, m.value)); } - setIncrement(key: IncrementalMetadataKey, amount = 1): void { + // eslint-disable-next-line @typescript-eslint/require-await + async setIncrement(key: IncrementalMetadataKey, amount = 1): Promise { this.setCache[key] = (this.setCache[key] ?? 0) + amount; } - setNewDynamicDatasource(item: DatasourceParams): void { + // eslint-disable-next-line @typescript-eslint/require-await + async setNewDynamicDatasource(item: DatasourceParams): Promise { this.datasourceUpdates.push(item); } @@ -113,10 +117,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); - await this.model.sequelize.query( - INCREMENT_QUERY(schemaTable, key, amount), - tx && { transaction: tx } - ); + await this.model.sequelize.query(INCREMENT_QUERY(schemaTable, key, amount), tx && {transaction: tx}); } private async appendDynamicDatasources(items: DatasourceParams[], tx?: Transaction): Promise { @@ -124,10 +125,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); - await this.model.sequelize.query( - APPEND_DS_QUERY(schemaTable, items), - tx && { transaction: tx } - ); + await this.model.sequelize.query(APPEND_DS_QUERY(schemaTable, items), tx && {transaction: tx}); } private async handleSpecialKeys(tx?: Transaction): Promise { @@ -142,7 +140,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM **/ const val = this.setCache[key]; if (val !== undefined) { - await this.model.bulkCreate([{ key, value: val }], { transaction: tx, updateOnDuplicate: ['key', 'value'] }); + await this.model.bulkCreate([{key, value: val}], {transaction: tx, updateOnDuplicate: ['key', 'value']}); } else if (this.datasourceUpdates.length) { await this.appendDynamicDatasources(this.datasourceUpdates, tx); } @@ -169,7 +167,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM async runFlush(tx: Transaction, blockHeight?: number): Promise { const ops = Object.entries(this.setCache) .filter(([key]) => !specialKeys.includes(key as MetadataKey)) - .map(([key, value]) => ({ key, value } as Metadata)); + .map(([key, value]) => ({key, value} as Metadata)); const lastProcessedHeightIdx = ops.findIndex((k) => k.key === 'lastProcessedHeight'); if (blockHeight !== undefined && lastProcessedHeightIdx >= 0) { @@ -187,13 +185,14 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM updateOnDuplicate: ['key', 'value'], }), this.handleSpecialKeys(tx), - this.model.destroy({ where: { key: this.removeCache } }), + this.model.destroy({where: {key: this.removeCache}}), ]); } // This is current only use for migrate Poi // If concurrent change to cache, please add mutex if needed - async bulkRemove(keys: K[], tx: Transaction): Promise { + // eslint-disable-next-line @typescript-eslint/require-await + async bulkRemove(keys: K[], tx?: Transaction): Promise { this.removeCache.push(...keys); for (const key of keys) { delete this.setCache[key]; @@ -212,8 +211,8 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM newSetCache.lastProcessedHeight = this.setCache.lastProcessedHeight; this.flushableRecordCounter = 1; } - this.setCache = { ...newSetCache }; - this.getCache = { ...newSetCache }; + this.setCache = {...newSetCache}; + this.getCache = {...newSetCache}; this.datasourceUpdates = []; } } diff --git a/packages/node-core/src/indexer/storeCache/metadata/metadata.ts b/packages/node-core/src/indexer/storeCache/metadata/metadata.ts index a277d52707..4c465c97a8 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/metadata.ts +++ b/packages/node-core/src/indexer/storeCache/metadata/metadata.ts @@ -1,13 +1,12 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import assert from "assert"; -import { hasValue } from "@subql/node-core/utils"; -import { Op, Transaction } from '@subql/x-sequelize'; -import { DatasourceParams } from "../../dynamic-ds.service"; -import { Metadata, MetadataKeys, MetadataRepo } from "../../entities"; -import { APPEND_DS_QUERY, INCREMENT_QUERY } from "./utils"; - +import assert from 'assert'; +import {hasValue} from '@subql/node-core/utils'; +import {Op, Transaction} from '@subql/x-sequelize'; +import {DatasourceParams} from '../../dynamic-ds.service'; +import {Metadata, MetadataKeys, MetadataRepo} from '../../entities'; +import {APPEND_DS_QUERY, INCREMENT_QUERY} from './utils'; export type MetadataKey = keyof MetadataKeys; export const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; @@ -25,19 +24,13 @@ export interface IMetadata { bulkRemove(keys: K[], tx?: Transaction): Promise; } - - export class MetadataModel implements IMetadata { - constructor(readonly model: MetadataRepo) {} - async find(key: K, fallback?: MetadataKeys[K]): Promise { const record = await this.model.findByPk(key); - return hasValue(record) - ? record.toJSON().value as MetadataKeys[K] - : fallback; + return hasValue(record) ? (record.toJSON().value as MetadataKeys[K]) : fallback; } async findMany(keys: readonly K[]): Promise> { @@ -54,11 +47,11 @@ export class MetadataModel implements IMetadata { } async set(key: K, value: MetadataKeys[K], tx?: Transaction): Promise { - throw new Error("Method not implemented."); + return this.setBulk([{key, value}], tx); } async setBulk(metadata: Metadata[], tx?: Transaction): Promise { - throw new Error("Method not implemented."); + await this.model.bulkCreate(metadata, {transaction: tx}); } async setIncrement(key: IncrementalMetadataKey, amount = 1, tx?: Transaction): Promise { @@ -67,10 +60,7 @@ export class MetadataModel implements IMetadata { assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); assert(incrementKeys.includes(key), `Key ${key} is not incrementable`); - await this.model.sequelize.query( - INCREMENT_QUERY(schemaTable, key, amount), - tx && { transaction: tx } - ); + await this.model.sequelize.query(INCREMENT_QUERY(schemaTable, key, amount), tx && {transaction: tx}); } async setNewDynamicDatasource(item: DatasourceParams, tx?: Transaction): Promise { @@ -78,16 +68,13 @@ export class MetadataModel implements IMetadata { assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); - await this.model.sequelize.query( - APPEND_DS_QUERY(schemaTable, [item]), - tx && { transaction: tx }, - ); + await this.model.sequelize.query(APPEND_DS_QUERY(schemaTable, [item]), tx && {transaction: tx}); } async bulkRemove(keys: K[], tx: Transaction): Promise { await this.model.destroy({ - where: { key: { [Op.in]: keys } }, - transaction: tx + where: {key: {[Op.in]: keys}}, + transaction: tx, }); } } diff --git a/packages/node-core/src/indexer/storeCache/metadata/utils.ts b/packages/node-core/src/indexer/storeCache/metadata/utils.ts index 5b22ccb72a..e08b909279 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/utils.ts +++ b/packages/node-core/src/indexer/storeCache/metadata/utils.ts @@ -1,15 +1,16 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { DatasourceParams } from "../../dynamic-ds.service"; -import { MetadataKeys } from "../../entities"; +import {DatasourceParams} from '../../dynamic-ds.service'; +import {MetadataKeys} from '../../entities'; export type MetadataKey = keyof MetadataKeys; export const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; export type IncrementalMetadataKey = 'processedBlockCount' | 'schemaMigrationCount'; +export const METADATA_ENTITY_NAME = '_metadata'; -type SchemaTable = string | { tableName: string, schema: string, delimiter: string }; +type SchemaTable = string | {tableName: string; schema: string; delimiter: string}; export function INCREMENT_QUERY(schemaTable: SchemaTable, key: MetadataKey, amount = 1): string { return `INSERT INTO ${schemaTable} (key, value, "createdAt", "updatedAt") @@ -17,7 +18,7 @@ export function INCREMENT_QUERY(schemaTable: SchemaTable, key: MetadataKey, amou ON CONFLICT (key) DO UPDATE SET value = (COALESCE(${schemaTable}.value->>0)::int + '${amount}')::text::jsonb, "updatedAt" = CURRENT_TIMESTAMP - WHERE ${schemaTable}.key = '${key}';` + WHERE ${schemaTable}.key = '${key}';`; } export function APPEND_DS_QUERY(schemaTable: SchemaTable, items: DatasourceParams[]): string { diff --git a/packages/node-core/src/indexer/storeCache/cacheModel.spec.ts b/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts similarity index 88% rename from packages/node-core/src/indexer/storeCache/cacheModel.spec.ts rename to packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts index 66dfb101bf..6c37d9dd50 100644 --- a/packages/node-core/src/indexer/storeCache/cacheModel.spec.ts +++ b/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts @@ -3,7 +3,7 @@ import {delay} from '@subql/common'; import {Sequelize} from '@subql/x-sequelize'; -import {NodeConfig} from '../../configure'; +import {NodeConfig} from '../../../configure'; import {CachedModel} from './cacheModel'; jest.mock('@subql/x-sequelize', () => { @@ -106,7 +106,7 @@ describe('cacheModel', () => { // Set an initial model and flush it blockHeight = 1; - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -126,7 +126,7 @@ describe('cacheModel', () => { // updated height to 2 blockHeight = 2; - testModel.set( + await testModel.set( 'entity1_id_0x01', { ...entity1, @@ -144,7 +144,7 @@ describe('cacheModel', () => { await delay(0.2); const entity2 = await testModel.get('entity1_id_0x01'); - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -161,7 +161,7 @@ describe('cacheModel', () => { it('can call getByFields, with entities updated in the same block', async () => { blockHeight = 2; - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -187,7 +187,7 @@ describe('cacheModel', () => { }); it('cannot mutate data in the cache without calling methods', async () => { - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -210,13 +210,13 @@ describe('cacheModel', () => { /* getBy methods use set cache */ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const entity3 = (await testModel.getOneByField('field1', 2))!; + const [entity3] = (await testModel.getByFields([['field1', '=', 2]], {limit: 1}))!; expect(entity3?.field1).toEqual(2); // Mutate field directly entity3.field1 = -2; - const entity4 = await testModel.getOneByField('field1', 2); + const [entity4] = await testModel.getByFields([['field1', '=', 2]], {limit: 1}); expect(entity4?.field1).toEqual(2); }); }); @@ -233,7 +233,7 @@ describe('cacheModel', () => { it('when get data after flushed, it should exclude block range', async () => { const spyDbGet = jest.spyOn(testModel.model, 'findOne'); const sypOnApplyBlockRange = jest.spyOn(testModel as any, 'applyBlockRange'); - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -259,8 +259,8 @@ describe('cacheModel', () => { // Some edge cases for set get and remove describe('set, remove and get', () => { it('In different block, remove and set, should able to get', async () => { - testModel.remove('entity1_id_0x01', 4); - testModel.set( + await testModel.remove('entity1_id_0x01', 4); + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -272,9 +272,9 @@ describe('cacheModel', () => { expect(result?.field1).toBe(5); }); - it('In same block, remove then set, should able to get', () => { - testModel.remove('entity1_id_0x01', 1); - testModel.set( + it('In same block, remove then set, should able to get', async () => { + await testModel.remove('entity1_id_0x01', 1); + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -290,8 +290,8 @@ describe('cacheModel', () => { }); it('In different block, remove and set, then remove again, should get nothing', async () => { - testModel.remove('entity1_id_0x01', 4); - testModel.set( + await testModel.remove('entity1_id_0x01', 4); + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -299,7 +299,7 @@ describe('cacheModel', () => { }, 6 ); - testModel.remove('entity1_id_0x01', 8); + await testModel.remove('entity1_id_0x01', 8); const result = await testModel.get('entity1_id_0x01'); expect((testModel as any).removeCache.entity1_id_0x01).toBeDefined(); // should match with last removed @@ -313,8 +313,8 @@ describe('cacheModel', () => { }); it('In same block, remove and set, then remove again, should get nothing', async () => { - testModel.remove('entity1_id_0x01', 1); - testModel.set( + await testModel.remove('entity1_id_0x01', 1); + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -322,7 +322,7 @@ describe('cacheModel', () => { }, 1 ); - testModel.remove('entity1_id_0x01', 1); + await testModel.remove('entity1_id_0x01', 1); const result = await testModel.get('entity1_id_0x01'); expect((testModel as any).removeCache.entity1_id_0x01).toBeDefined(); @@ -333,8 +333,8 @@ describe('cacheModel', () => { expect(result).toBeUndefined(); }); - it('clean flushable records when applyBlockRange, if found set and removed happened in the same height', () => { - testModel.set( + it('clean flushable records when applyBlockRange, if found set and removed happened in the same height', async () => { + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -342,9 +342,9 @@ describe('cacheModel', () => { }, 1 ); - testModel.remove('entity1_id_0x01', 1); + await testModel.remove('entity1_id_0x01', 1); - testModel.set( + await testModel.set( 'entity1_id_0x02', { id: 'entity1_id_0x02', @@ -365,8 +365,8 @@ describe('cacheModel', () => { expect(records[0].id).toBe('entity1_id_0x02'); }); - it('clean flushable records when applyBlockRange, pass if set and remove in the different height', () => { - testModel.set( + it('clean flushable records when applyBlockRange, pass if set and remove in the different height', async () => { + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -374,9 +374,9 @@ describe('cacheModel', () => { }, 1 ); - testModel.remove('entity1_id_0x01', 2); + await testModel.remove('entity1_id_0x01', 2); - testModel.set( + await testModel.set( 'entity1_id_0x02', { id: 'entity1_id_0x02', @@ -394,7 +394,7 @@ describe('cacheModel', () => { }); it('getFromCache could filter out removed data', async () => { - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -402,9 +402,9 @@ describe('cacheModel', () => { }, 1 ); - testModel.remove('entity1_id_0x01', 1); + await testModel.remove('entity1_id_0x01', 1); - testModel.set( + await testModel.set( 'entity1_id_0x02', { id: 'entity1_id_0x02', @@ -413,7 +413,7 @@ describe('cacheModel', () => { 2 ); const spyFindAll = jest.spyOn(testModel.model, 'findAll'); - const result = await testModel.getByField('field1', 1, {offset: 0, limit: 50}); + const result = await testModel.getByFields([['field1', '=', 1]], {offset: 0, limit: 50}); expect(spyFindAll).toHaveBeenCalledTimes(1); expect(result).toStrictEqual([ @@ -422,7 +422,7 @@ describe('cacheModel', () => { }); it('getFromCache with removed and set again data', async () => { - testModel.set( + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -430,8 +430,8 @@ describe('cacheModel', () => { }, 1 ); - testModel.remove('entity1_id_0x01', 1); - testModel.set( + await testModel.remove('entity1_id_0x01', 1); + await testModel.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -440,12 +440,12 @@ describe('cacheModel', () => { 1 ); const spyFindAll = jest.spyOn(testModel.model, 'findAll'); - const result = await testModel.getByField('field1', 1, {offset: 0, limit: 50}); + const result = await testModel.getByFields([['field1', '=', 1]], {offset: 0, limit: 50}); expect(spyFindAll).toHaveBeenCalledTimes(1); expect(result).toStrictEqual([{id: 'entity1_id_0x01', field1: 1}]); // Should not include any previous recorded value - const result3 = await testModel.getByField('field1', 3, {offset: 0, limit: 50}); + const result3 = await testModel.getByFields([['field1', '=', 3]], {offset: 0, limit: 50}); // Expect only mocked expect(result3).toStrictEqual([]); }); diff --git a/packages/node-core/src/indexer/storeCache/cacheModel.test.ts b/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts similarity index 96% rename from packages/node-core/src/indexer/storeCache/cacheModel.test.ts rename to packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts index 7398d45839..fadf57e5e2 100644 --- a/packages/node-core/src/indexer/storeCache/cacheModel.test.ts +++ b/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts @@ -4,7 +4,7 @@ import {GraphQLModelsType} from '@subql/utils'; import {Sequelize, DataTypes, QueryTypes} from '@subql/x-sequelize'; import {cloneDeep, padStart} from 'lodash'; -import {DbOption, modelsTypeToModelAttributes, NodeConfig} from '../../'; +import {DbOption, modelsTypeToModelAttributes, NodeConfig} from '../../../'; import {CachedModel} from './cacheModel'; const option: DbOption = { @@ -74,7 +74,7 @@ describe('cacheMetadata integration', () => { // Pre-populate some data and flush it do the db let n = 0; while (n < 100) { - cacheModel.set( + await cacheModel.set( `entity1_id_0x${formatIdNumber(n)}`, { id: `entity1_id_0x${formatIdNumber(n)}`, @@ -90,7 +90,7 @@ describe('cacheMetadata integration', () => { // Updates to existing data let m = 20; while (m < 30) { - cacheModel.set( + await cacheModel.set( `entity1_id_0x${formatIdNumber(m)}`, { id: `entity1_id_0x${formatIdNumber(m)}`, @@ -104,7 +104,7 @@ describe('cacheMetadata integration', () => { // New data let o = 100; while (o < 130) { - cacheModel.set( + await cacheModel.set( `entity1_id_0x${formatIdNumber(o)}`, { id: `entity1_id_0x${formatIdNumber(o)}`, @@ -117,16 +117,17 @@ describe('cacheMetadata integration', () => { }); it('gets one item correctly', async () => { + const getOneBy = (field: 'id', value: string) => cacheModel.getByFields([[field, '=', value]], {limit: 1}); // Db value - const res0 = await cacheModel.getOneByField('id', 'entity1_id_0x001'); + const [res0] = await getOneBy('id', 'entity1_id_0x001'); expect(res0).toEqual({id: 'entity1_id_0x001', field1: 1}); // Cache value - const res1 = await cacheModel.getOneByField('id', 'entity1_id_0x020'); + const [res1] = await getOneBy('id', 'entity1_id_0x020'); expect(res1).toEqual({id: 'entity1_id_0x020', field1: 0}); // Cache value - const res2 = await cacheModel.getOneByField('id', 'entity1_id_0x021'); + const [res2] = await getOneBy('id', 'entity1_id_0x021'); expect(res2).toEqual({id: 'entity1_id_0x021', field1: 1}); }); @@ -436,7 +437,7 @@ describe('cacheModel integration', () => { // Update the value res1?.delegators.push({delegator: '0x03', amount: BigInt(9000000000000000000000n)}); - cacheModel.set(`0x01`, res1!, 2); + await cacheModel.set(`0x01`, res1!, 2); await flush(3); const res2 = await cacheModel.get('0x01'); console.log(JSON.stringify(res2)); @@ -486,7 +487,7 @@ describe('cacheModel integration', () => { amount: BigInt(6000000000000000000000n), nested: {testItem: 'test', amount: BigInt(6000000000000000000000n)}, }); - cacheModel.set(`0x01`, res1!, 4); + await cacheModel.set(`0x01`, res1!, 4); await flush(5); const [rows2] = await sequelize.query(`SELECT delegators FROM ${schema}."testModels" LIMIT 1;`, { type: QueryTypes.SELECT, @@ -526,7 +527,7 @@ describe('cacheModel integration', () => { ], randomNArray: undefined, }; - cacheModel.set(`0x02`, data0x02, 6); + await cacheModel.set(`0x02`, data0x02, 6); await flush(7); const res5 = ( await cacheModel.model.findOne({ diff --git a/packages/node-core/src/indexer/storeCache/cacheModel.ts b/packages/node-core/src/indexer/storeCache/model/cacheModel.ts similarity index 85% rename from packages/node-core/src/indexer/storeCache/cacheModel.ts rename to packages/node-core/src/indexer/storeCache/model/cacheModel.ts index c986ea5b48..683a9b853c 100644 --- a/packages/node-core/src/indexer/storeCache/cacheModel.ts +++ b/packages/node-core/src/indexer/storeCache/model/cacheModel.ts @@ -2,24 +2,16 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {FieldOperators, FieldsExpression, GetOptions} from '@subql/types-core'; +import {FieldsExpression, GetOptions} from '@subql/types-core'; import {CreationAttributes, Model, ModelStatic, Op, Sequelize, Transaction} from '@subql/x-sequelize'; -import {Fn} from '@subql/x-sequelize/types/utils'; import {flatten, uniq, cloneDeep, orderBy, unionBy} from 'lodash'; -import {NodeConfig} from '../../configure'; -import {Cacheable} from './cacheable'; -import {CsvStoreService} from './csvStore.service'; -import {SetValueModel} from './setValueModel'; -import { - ICachedModelControl, - RemoveValue, - SetData, - ICachedModel, - GetData, - FilteredHeightRecords, - SetValue, - Exporter, -} from './types'; +import {NodeConfig} from '../../../configure'; +import {Cacheable} from '../cacheable'; +import {CsvStoreService} from '../csvStore.service'; +import {SetValueModel} from '../setValueModel'; +import {ICachedModelControl, RemoveValue, SetData, GetData, FilteredHeightRecords, SetValue, Exporter} from '../types'; +import {BaseEntity, IModel} from './model'; +import {getFullOptions, operatorsMap} from './utils'; const getCacheOptions = { max: 500, // default value @@ -27,28 +19,9 @@ const getCacheOptions = { updateAgeOnGet: true, // we want to keep most used record in cache longer }; -const operatorsMap: Record = { - '=': Op.eq, - '!=': Op.ne, - in: Op.in, - '!in': Op.notIn, -}; - -const defaultOptions: Required> = { - offset: 0, - limit: 100, - orderBy: 'id', - orderDirection: 'ASC', -}; - -export class CachedModel< - T extends {id: string; __block_range?: (number | null)[] | Fn} = { - id: string; - __block_range?: (number | null)[] | Fn; - }, - > +export class CachedModel extends Cacheable - implements ICachedModel, ICachedModelControl + implements IModel, ICachedModelControl { // Null value indicates its not defined in the db private getCache: GetData; @@ -138,7 +111,7 @@ export class CachedModel< * There is also no way to flush data here, * flushing will only flush data before the current block so its still required to consider the setCache * */ - async getByFields(filters: FieldsExpression[], options: GetOptions = defaultOptions): Promise { + async getByFields(filters: FieldsExpression[], options?: GetOptions): Promise { filters.forEach(([field, operator]) => { assert( operatorsMap[operator], @@ -152,10 +125,7 @@ export class CachedModel< // If projects use inefficient store methods, thats on them. // Ensure we have all the options - const fullOptions: Required> = { - ...defaultOptions, - ...options, - }; + const fullOptions = getFullOptions(options); await this.mutex.waitForUnlock(); @@ -170,7 +140,7 @@ export class CachedModel< .map((value) => value.getLatest()?.data) .map((value) => cloneDeep(value)) as T[]; - const offsetCacheData = cacheData.slice(options.offset); + const offsetCacheData = cacheData.slice(fullOptions.offset); // Return early if cache covers all the data if (offsetCacheData.length > fullOptions.limit) { @@ -198,24 +168,25 @@ export class CachedModel< return combined; } - async getByField( - field: keyof T, - value: T[keyof T] | T[keyof T][], - options: GetOptions = defaultOptions - ): Promise { - return this.getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], options); - } - - async getOneByField(field: keyof T, value: T[keyof T]): Promise { - const [res] = await this.getByField(field, value, { - ...defaultOptions, - limit: 1, - }); - - return res; - } - - set(id: string, data: T, blockHeight: number): void { + // async getByField( + // field: keyof T, + // value: T[keyof T] | T[keyof T][], + // options: GetOptions = defaultOptions + // ): Promise { + // return this.getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], options); + // } + + // async getOneByField(field: keyof T, value: T[keyof T]): Promise { + // const [res] = await this.getByField(field, value, { + // ...defaultOptions, + // limit: 1, + // }); + + // return res; + // } + + // eslint-disable-next-line @typescript-eslint/require-await + async set(id: string, data: T, blockHeight: number): Promise { if (this.setCache[id] === undefined) { this.setCache[id] = new SetValueModel(); } @@ -231,23 +202,24 @@ export class CachedModel< this.flushableRecordCounter += 1; } - bulkCreate(data: T[], blockHeight: number): void { + async bulkCreate(data: T[], blockHeight: number): Promise { for (const entity of data) { - this.set(entity.id, entity, blockHeight); + await this.set(entity.id, entity, blockHeight); } } - bulkUpdate(data: T[], blockHeight: number, fields?: string[] | undefined): void { + async bulkUpdate(data: T[], blockHeight: number, fields?: string[]): Promise { //TODO, remove fields if (fields) { throw new Error(`Currently not supported: update by fields`); } for (const entity of data) { - this.set(entity.id, entity, blockHeight); + await this.set(entity.id, entity, blockHeight); } } - remove(id: string, blockHeight: number): void { + // eslint-disable-next-line @typescript-eslint/require-await + async remove(id: string, blockHeight: number): Promise { // we don't need to check whether id is already removed, // because it could be removed->create-> removed again, // the operationIndex should always be the latest operation @@ -266,8 +238,8 @@ export class CachedModel< } } - bulkRemove(ids: string[], blockHeight: number): void { - ids.map((id) => this.remove(id, blockHeight)); + async bulkRemove(ids: string[], blockHeight: number): Promise { + await Promise.all(ids.map((id) => this.remove(id, blockHeight))); } get isFlushable(): boolean { @@ -352,16 +324,13 @@ export class CachedModel< } private filterRemoveRecordByHeight(blockHeight: number, lessEqt: boolean): Record { - return Object.entries(this.removeCache).reduce( - (acc, [key, value]) => { - if (lessEqt ? value.removedAtBlock <= blockHeight : value.removedAtBlock > blockHeight) { - acc[key] = value; - } + return Object.entries(this.removeCache).reduce((acc, [key, value]) => { + if (lessEqt ? value.removedAtBlock <= blockHeight : value.removedAtBlock > blockHeight) { + acc[key] = value; + } - return acc; - }, - {} as Record - ); + return acc; + }, {} as Record); } private filterRecordsWithHeight(blockHeight: number): FilteredHeightRecords { diff --git a/packages/node-core/src/indexer/storeCache/model/index.ts b/packages/node-core/src/indexer/storeCache/model/index.ts new file mode 100644 index 0000000000..c32b7b677c --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/model/index.ts @@ -0,0 +1,5 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +export * from './cacheModel'; +export {IModel, BaseEntity} from './model'; diff --git a/packages/node-core/src/indexer/storeCache/model/model.ts b/packages/node-core/src/indexer/storeCache/model/model.ts new file mode 100644 index 0000000000..5d8b9d74d1 --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/model/model.ts @@ -0,0 +1,79 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import {FieldsExpression, GetOptions} from '@subql/types-core'; +import {Op, Model, ModelStatic, Transaction, CreationAttributes} from '@subql/x-sequelize'; +import {Fn} from '@subql/x-sequelize/types/utils'; +import {getFullOptions, operatorsMap} from './utils'; + +export type BaseEntity = {id: string; __block_range?: (number | null)[] | Fn}; + +export interface IModel { + get(id: string): Promise; + + getByFields(filters: FieldsExpression[], options: GetOptions): Promise; + + set: (id: string, data: T, blockHeight: number, tx?: Transaction) => Promise; + bulkCreate(data: T[], blockHeight: number, tx?: Transaction): Promise; + bulkUpdate(data: T[], blockHeight: number, fields?: string[], tx?: Transaction): Promise; + + bulkRemove(ids: string[], blockHeight: number, tx?: Transaction): Promise; +} + +export class PlainModel implements IModel { + constructor(readonly model: ModelStatic>, private readonly historical = true) {} + + async get(id: string): Promise { + const record = await this.model.findOne({ + // https://github.com/sequelize/sequelize/issues/15179 + where: {id} as any, + }); + + return record?.toJSON(); + } + + async getByFields(filters: FieldsExpression[], options: GetOptions): Promise { + const fullOptions = getFullOptions(options); + // Query DB with all params + const records = await this.model.findAll({ + where: { + [Op.and]: [...filters.map(([field, operator, value]) => ({[field]: {[operatorsMap[operator]]: value}}))] as any, // Types not working properly + }, + limit: fullOptions.limit, + offset: fullOptions.offset, + order: [[fullOptions.orderBy as string, fullOptions.orderDirection]], + }); + + return records.map((r) => r.toJSON()); + } + + async set(id: string, data: T, blockHeight: number, tx?: Transaction): Promise { + if (id !== data.id) { + throw new Error(`Id doesnt match with data`); + } + + await this.bulkCreate([data], blockHeight, tx); + } + + async bulkCreate(data: T[], blockHeight: number, tx?: Transaction): Promise { + if (!data.length) { + return; + } + await this.model.bulkCreate(data as CreationAttributes>[], { + transaction: tx, + updateOnDuplicate: Object.keys(data[0]) as unknown as (keyof T)[], + }); + } + + async bulkUpdate(data: T[], blockHeight: number, fields?: string[], tx?: Transaction): Promise { + //TODO, understand why this happens, its also on the store cache + if (fields) { + throw new Error(`Currently not supported: update by fields`); + } + await this.bulkCreate(data, blockHeight, tx); + } + + async bulkRemove(ids: string[], blockHeight: number, tx?: Transaction): Promise { + await this.model.destroy({where: {id: ids} as any, transaction: tx}); + } +} diff --git a/packages/node-core/src/indexer/storeCache/model/utils.ts b/packages/node-core/src/indexer/storeCache/model/utils.ts new file mode 100644 index 0000000000..9abb6b0ddf --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/model/utils.ts @@ -0,0 +1,26 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import {FieldOperators, GetOptions} from '@subql/types-core'; +import {Op} from '@subql/x-sequelize'; + +export const operatorsMap: Record = { + '=': Op.eq, + '!=': Op.ne, + in: Op.in, + '!in': Op.notIn, +}; + +const defaultOptions: Required> = { + offset: 0, + limit: 100, + orderBy: 'id', + orderDirection: 'ASC', +}; + +// Ensure we have all the options +export const getFullOptions = (options?: GetOptions): Required> => + ({ + ...(defaultOptions as GetOptions), + ...options, + } as Required>); diff --git a/packages/node-core/src/indexer/storeCache/cachePoi.spec.ts b/packages/node-core/src/indexer/storeCache/poi/cachePoi.spec.ts similarity index 99% rename from packages/node-core/src/indexer/storeCache/cachePoi.spec.ts rename to packages/node-core/src/indexer/storeCache/poi/cachePoi.spec.ts index 9c135c9dbb..f31ad5e3a7 100644 --- a/packages/node-core/src/indexer/storeCache/cachePoi.spec.ts +++ b/packages/node-core/src/indexer/storeCache/poi/cachePoi.spec.ts @@ -3,7 +3,7 @@ import {delay} from '@subql/common'; import {Op, Sequelize} from '@subql/x-sequelize'; -import {PoiRepo} from '../entities'; +import {PoiRepo} from '../../entities'; import {CachePoiModel} from './cachePoi'; const mockPoiRepo = (): PoiRepo => { diff --git a/packages/node-core/src/indexer/storeCache/cachePoi.ts b/packages/node-core/src/indexer/storeCache/poi/cachePoi.ts similarity index 86% rename from packages/node-core/src/indexer/storeCache/cachePoi.ts rename to packages/node-core/src/indexer/storeCache/poi/cachePoi.ts index c839cb4f89..c098563c48 100644 --- a/packages/node-core/src/indexer/storeCache/cachePoi.ts +++ b/packages/node-core/src/indexer/storeCache/poi/cachePoi.ts @@ -4,14 +4,14 @@ import {DEFAULT_FETCH_RANGE} from '@subql/common'; import {u8aToBuffer} from '@subql/utils'; import {Transaction} from '@subql/x-sequelize'; -import {getLogger} from '../../logger'; -import {PoiRepo, ProofOfIndex} from '../entities'; -import {PlainPoiModel, PoiInterface} from '../poi/poiModel'; -import {Cacheable} from './cacheable'; -import {ICachedModelControl} from './types'; +import {getLogger} from '../../../logger'; +import {PoiRepo, ProofOfIndex} from '../../entities'; +import {Cacheable} from '../cacheable'; +import {ICachedModelControl} from '../types'; +import {IPoi, PlainPoiModel} from './poi'; const logger = getLogger('PoiCache'); -export class CachePoiModel extends Cacheable implements ICachedModelControl, PoiInterface { +export class CachePoiModel extends Cacheable implements IPoi, ICachedModelControl { private setCache: Record = {}; flushableRecordCounter = 0; plainPoiModel: PlainPoiModel; @@ -21,7 +21,8 @@ export class CachePoiModel extends Cacheable implements ICachedModelControl, Poi this.plainPoiModel = new PlainPoiModel(model); } - bulkUpsert(proofs: ProofOfIndex[]): void { + // eslint-disable-next-line @typescript-eslint/require-await + async bulkUpsert(proofs: ProofOfIndex[]): Promise { for (const proof of proofs) { if (proof.chainBlockHash !== null) { proof.chainBlockHash = u8aToBuffer(proof.chainBlockHash); diff --git a/packages/node-core/src/indexer/storeCache/poi/index.ts b/packages/node-core/src/indexer/storeCache/poi/index.ts new file mode 100644 index 0000000000..8c876af2bd --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/poi/index.ts @@ -0,0 +1,5 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +export * from './cachePoi'; +export * from './poi'; diff --git a/packages/node-core/src/indexer/poi/poiModel.ts b/packages/node-core/src/indexer/storeCache/poi/poi.ts similarity index 90% rename from packages/node-core/src/indexer/poi/poiModel.ts rename to packages/node-core/src/indexer/storeCache/poi/poi.ts index d31f998510..5be2d37662 100644 --- a/packages/node-core/src/indexer/poi/poiModel.ts +++ b/packages/node-core/src/indexer/storeCache/poi/poi.ts @@ -4,12 +4,14 @@ import {DEFAULT_FETCH_RANGE} from '@subql/common'; import {u8aToBuffer} from '@subql/utils'; import {Op, Transaction} from '@subql/x-sequelize'; -import {BlockRangeDtoInterface} from '../../admin'; -import {PoiRepo, ProofOfIndex} from '../entities'; +import {BlockRangeDtoInterface} from '../../../admin'; +import {PoiRepo, ProofOfIndex} from '../../entities'; -export interface PoiInterface { - model: PoiRepo; - bulkUpsert(proofs: ProofOfIndex[], tx?: Transaction): Promise | void; +export const POI_ENTITY_NAME = '_poi'; + +export interface IPoi { + model: PoiRepo; // TODO remove + bulkUpsert(proofs: ProofOfIndex[], tx?: Transaction): Promise; /** * Gets the 100 blocks <= to the start height where there is an operation. * This can be used to determine the last blocks that had data to index. @@ -19,14 +21,14 @@ export interface PoiInterface { // When using cockroach db, poi id is store in bigint format, and sequelize toJSON() can not convert id correctly (to string) // This will ensure after toJSON Poi id converted to number -export function ensureProofOfIndexId(poi: ProofOfIndex): ProofOfIndex { +function ensureProofOfIndexId(poi: ProofOfIndex): ProofOfIndex { if (typeof poi?.id === 'string') { poi.id = Number(poi.id); } return poi; } -export class PlainPoiModel implements PoiInterface { +export class PlainPoiModel implements IPoi { constructor(readonly model: PoiRepo) {} async getFirst(): Promise { diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.spec.ts b/packages/node-core/src/indexer/storeCache/storeCache.service.spec.ts index 8b0cba5f7c..8454f53ffc 100644 --- a/packages/node-core/src/indexer/storeCache/storeCache.service.spec.ts +++ b/packages/node-core/src/indexer/storeCache/storeCache.service.spec.ts @@ -6,10 +6,13 @@ import {SchedulerRegistry} from '@nestjs/schedule'; import {Sequelize} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {delay} from '../../utils'; +import {BaseEntity} from './model'; import {StoreCacheService} from './storeCache.service'; const eventEmitter = new EventEmitter2(); +type TestEntity = BaseEntity & {field1: string}; + jest.mock('@subql/x-sequelize', () => { const mSequelize = { authenticate: jest.fn(), @@ -72,10 +75,10 @@ describe('Store Cache Service historical', () => { }); it('could set cache for multiple entities, also get from it', async () => { - const entity1Model = storeService.getModel('entity1'); - const entity2Model = storeService.getModel('entity2'); + const entity1Model = storeService.getModel('entity1'); + const entity2Model = storeService.getModel('entity2'); - entity1Model.set( + await entity1Model.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -83,7 +86,7 @@ describe('Store Cache Service historical', () => { }, 1 ); - entity2Model.set( + await entity2Model.set( 'entity2_id_0x02', { id: 'entity2_id_0x02', @@ -102,9 +105,9 @@ describe('Store Cache Service historical', () => { // TODO move this test to cacheModel it('set at different block height, will create historical records', async () => { - const appleModel = storeService.getModel('apple'); + const appleModel = storeService.getModel('apple'); - appleModel.set( + await appleModel.set( 'apple-01', { id: 'apple-01', @@ -113,10 +116,10 @@ describe('Store Cache Service historical', () => { 1 ); - const appleEntity_b1 = (await appleModel.get('apple-01')) as any; - expect(appleEntity_b1.field1).toBe('set apple at block 1'); + const appleEntity_b1 = await appleModel.get('apple-01'); + expect(appleEntity_b1!.field1).toBe('set apple at block 1'); // Add new record, should create historical records for same id entity - appleModel.set( + await appleModel.set( 'apple-01', { id: 'apple-01', @@ -152,11 +155,11 @@ describe('Store Cache flush with order', () => { storeService.init(false, true, {} as any, undefined); }); - it('when set/remove multiple model entities, operation index should added to record in sequential order', () => { - const entity1Model = storeService.getModel('entity1'); - const entity2Model = storeService.getModel('entity2'); + it('when set/remove multiple model entities, operation index should added to record in sequential order', async () => { + const entity1Model = storeService.getModel('entity1'); + const entity2Model = storeService.getModel('entity2'); - entity1Model.set( + await entity1Model.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -164,7 +167,7 @@ describe('Store Cache flush with order', () => { }, 1 ); - entity2Model.set( + await entity2Model.set( 'entity2_id_0x02', { id: 'entity2_id_0x02', @@ -172,7 +175,7 @@ describe('Store Cache flush with order', () => { }, 2 ); - entity1Model.remove('entity1_id_0x01', 3); + await entity1Model.bulkRemove(['entity1_id_0x01'], 3); const entity1 = (storeService as any).cachedModels.entity1; expect(entity1.removeCache.entity1_id_0x01.operationIndex).toBe(3); }); @@ -190,10 +193,10 @@ describe('Store Cache flush with non-historical', () => { }); it('Same Id with multiple operations, when flush it should always pick up the latest operation', async () => { - const entity1Model = storeService.getModel('entity1'); + const entity1Model = storeService.getModel('entity1'); //create Id 1 - entity1Model.set( + await entity1Model.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -202,11 +205,11 @@ describe('Store Cache flush with non-historical', () => { 1 ); // remove Id 1 and 2 - entity1Model.remove('entity1_id_0x02', 2); - entity1Model.remove('entity1_id_0x01', 3); + await entity1Model.bulkRemove(['entity1_id_0x02'], 2); + await entity1Model.bulkRemove(['entity1_id_0x01'], 3); // recreate id 1 again - entity1Model.set( + await entity1Model.set( 'entity1_id_0x01', { id: 'entity1_id_0x01', @@ -248,10 +251,10 @@ describe('Store cache upper threshold', () => { }); it('doesnt wait for flushing cache when threshold not met', async () => { - const entity1Model = storeService.getModel('entity1'); + const entity1Model = storeService.getModel('entity1'); for (let i = 0; i < 5; i++) { - entity1Model.set( + await entity1Model.set( `entity1_id_0x0${i}`, { id: `entity1_id_0x0${i}`, @@ -270,10 +273,10 @@ describe('Store cache upper threshold', () => { }); it('waits for flushing when threshold is met', async () => { - const entity1Model = storeService.getModel('entity1'); + const entity1Model = storeService.getModel('entity1'); for (let i = 0; i < 15; i++) { - entity1Model.set( + await entity1Model.set( `entity1_id_0x0${i}`, { id: `entity1_id_0x0${i}`, diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.ts b/packages/node-core/src/indexer/storeCache/storeCache.service.ts index 8807248faf..b8126b6648 100644 --- a/packages/node-core/src/indexer/storeCache/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeCache.service.ts @@ -2,35 +2,32 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import { Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { SchedulerRegistry } from '@nestjs/schedule'; -import { DatabaseError, Deferrable, ModelStatic, Sequelize, Transaction } from '@subql/x-sequelize'; -import { sum } from 'lodash'; -import { NodeConfig } from '../../configure'; -import { IndexerEvent } from '../../events'; -import { getLogger } from '../../logger'; -import { exitWithError } from '../../process'; -import { profiler } from '../../profiler'; -import { MetadataRepo, PoiRepo } from '../entities'; -import { BaseCacheService } from './baseCache.service'; -import { CachedModel } from './cacheModel'; -import { CachePoiModel } from './cachePoi'; -import { CsvStoreService } from './csvStore.service'; -import { CacheMetadataModel } from './metadata'; -import { Exporter, ICachedModel, ICachedModelControl } from './types'; +import {Injectable} from '@nestjs/common'; +import {EventEmitter2} from '@nestjs/event-emitter'; +import {SchedulerRegistry} from '@nestjs/schedule'; +import {DatabaseError, Deferrable, ModelStatic, Sequelize, Transaction} from '@subql/x-sequelize'; +import {sum} from 'lodash'; +import {NodeConfig} from '../../configure'; +import {IndexerEvent} from '../../events'; +import {getLogger} from '../../logger'; +import {exitWithError} from '../../process'; +import {profiler} from '../../profiler'; +import {MetadataRepo, PoiRepo} from '../entities'; +import {BaseCacheService} from './baseCache.service'; +import {CsvStoreService} from './csvStore.service'; +import {CacheMetadataModel} from './metadata'; +import {METADATA_ENTITY_NAME} from './metadata/utils'; +import {CachedModel} from './model'; +import {CachePoiModel, POI_ENTITY_NAME} from './poi'; +import {IStoreModelService} from './storeModel.service'; +import {Exporter, ICachedModelControl} from './types'; const logger = getLogger('StoreCacheService'); @Injectable() -export class StoreCacheService extends BaseCacheService { - private cachedModels: Record = {}; - private metadataRepo?: MetadataRepo; - private poiRepo?: PoiRepo; +export class StoreCacheService extends BaseCacheService implements IStoreModelService { private readonly storeCacheThreshold: number; private readonly cacheUpperLimit: number; - private _historical = true; - private _useCockroachDb?: boolean; private _storeOperationIndex = 0; private _lastFlushedOperationIndex = 0; private exports: Exporter[] = []; @@ -51,10 +48,7 @@ export class StoreCacheService extends BaseCacheService { } init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { - this._useCockroachDb = useCockroachDb; - this._historical = historical; - this.metadataRepo = meta; - this.poiRepo = poi; + super.init(historical, useCockroachDb, meta, poi); if (this.config.storeFlushInterval > 0) { this.schedulerRegistry.addInterval( @@ -80,25 +74,12 @@ export class StoreCacheService extends BaseCacheService { return this._storeOperationIndex; } - getModel(entity: string): ICachedModel { - if (entity === '_metadata') { - throw new Error('Please use getMetadataModel instead'); - } - if (entity === '_poi') { - throw new Error('Please use getPoiModel instead'); - } - if (!this.cachedModels[entity]) { - this.cachedModels[entity] = this.createModel(entity); - } - return this.cachedModels[entity] as unknown as ICachedModel; - } - - private createModel(entityName: string): CachedModel { + protected createModel(entityName: string): CachedModel { const model = this.sequelize.model(entityName); assert(model, `model ${entityName} not exists`); const cachedModel = new CachedModel( model, - this._historical, + this.historical, this.config, this.getNextStoreOperationIndex.bind(this) ); @@ -110,7 +91,7 @@ export class StoreCacheService extends BaseCacheService { return cachedModel; } - addExporter(cachedModel: CachedModel, exporterStore: CsvStoreService): void { + private addExporter(cachedModel: CachedModel, exporterStore: CsvStoreService): void { cachedModel.addExporterStore(exporterStore); this.exports.push(exporterStore); } @@ -119,35 +100,26 @@ export class StoreCacheService extends BaseCacheService { await Promise.all(this.exports.map((f) => f.shutdown())); } - updateModels({ modifiedModels, removedModels }: { modifiedModels: ModelStatic[]; removedModels: string[] }): void { - modifiedModels.forEach((m) => { - this.cachedModels[m.name] = this.createModel(m.name); - }); - removedModels.forEach((r) => delete this.cachedModels[r]); - } - get metadata(): CacheMetadataModel { - const entity = '_metadata'; - if (!this.cachedModels[entity]) { + if (!this.cachedModels[METADATA_ENTITY_NAME]) { if (!this.metadataRepo) { throw new Error('Metadata entity has not been set on store cache'); } - this.cachedModels[entity] = new CacheMetadataModel(this.metadataRepo); + this.cachedModels[METADATA_ENTITY_NAME] = new CacheMetadataModel(this.metadataRepo); } - return this.cachedModels[entity] as unknown as CacheMetadataModel; + return this.cachedModels[METADATA_ENTITY_NAME] as unknown as CacheMetadataModel; } get poi(): CachePoiModel | null { - const entity = '_poi'; - if (!this.cachedModels[entity]) { + if (!this.cachedModels[POI_ENTITY_NAME]) { if (!this.poiRepo) { return null; // throw new Error('Poi entity has not been set on store cache'); } - this.cachedModels[entity] = new CachePoiModel(this.poiRepo); + this.cachedModels[POI_ENTITY_NAME] = new CachePoiModel(this.poiRepo); } - return this.cachedModels[entity] as unknown as CachePoiModel; + return this.cachedModels[POI_ENTITY_NAME] as unknown as CachePoiModel; } private async flushRelationalModelsInOrder(updatableModels: ICachedModelControl[], tx: Transaction): Promise { @@ -168,14 +140,14 @@ export class StoreCacheService extends BaseCacheService { this.logger.debug('Flushing cache'); // With historical disabled we defer the constraints check so that it doesn't matter what order entities are modified const tx = await this.sequelize.transaction({ - deferrable: this._historical || this._useCockroachDb ? undefined : Deferrable.SET_DEFERRED(), + deferrable: this.historical || this.useCockroachDb ? undefined : Deferrable.SET_DEFERRED(), }); try { // Get the block height of all data we want to flush up to const blockHeight = await this.metadata.find('lastProcessedHeight'); // Get models that have data to flush const updatableModels = Object.values(this.cachedModels).filter((m) => m.isFlushable); - if (this._useCockroachDb) { + if (this.useCockroachDb) { // 1. Independent(no associations) models can flush simultaneously await Promise.all( updatableModels.filter((m) => !m.hasAssociations).map((model) => model.flush(tx, blockHeight)) @@ -224,4 +196,22 @@ export class StoreCacheService extends BaseCacheService { }); return numOfRecords >= this.storeCacheThreshold; } + + async applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise { + if (this.config.storeCacheAsync) { + // Flush all completed block data and don't wait + await this.flushAndWaitForCapacity(false)?.catch((e) => { + exitWithError(new Error(`Flushing cache failed`, {cause: e}), logger); + }); + } else { + // Flush all data from cache and wait + await this.flushCache(false); + } + + if (dataSourcesCompleted) { + const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; + await this.flushCache(false); + exitWithError(msg, logger, 0); + } + } } diff --git a/packages/node-core/src/indexer/storeCache/storeModel.service.ts b/packages/node-core/src/indexer/storeCache/storeModel.service.ts new file mode 100644 index 0000000000..1b199ec7f9 --- /dev/null +++ b/packages/node-core/src/indexer/storeCache/storeModel.service.ts @@ -0,0 +1,90 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import {Sequelize, ModelStatic} from '@subql/x-sequelize'; +import {NodeConfig} from '../../configure'; +import {getLogger} from '../../logger'; +import {exitWithError} from '../../process'; +import {StoreService} from '../store.service'; +import {BaseStoreModelService} from './baseStoreModel.service'; +import {CsvStoreService} from './csvStore.service'; +import {IMetadata} from './metadata'; +import {MetadataModel} from './metadata/metadata'; +import {METADATA_ENTITY_NAME} from './metadata/utils'; +import {BaseEntity, IModel} from './model'; +import {PlainModel} from './model/model'; +import {IPoi, PlainPoiModel, POI_ENTITY_NAME} from './poi'; + +export interface IStoreModelService { + poi: IPoi | null; + metadata: IMetadata; + + getModel(entity: string): IModel; + + // addExporter(entity: string, exporterStore: CsvStoreService): void; + + applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; + + updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; +} + +const logger = getLogger('PlainStoreModelService'); + +export class PlainStoreModelService extends BaseStoreModelService implements IStoreModelService { + constructor(private sequelize: Sequelize, private config: NodeConfig, private storeService: StoreService) { + super(); + } + + get metadata(): MetadataModel { + if (!this.cachedModels[METADATA_ENTITY_NAME]) { + if (!this.metadataRepo) { + throw new Error('Metadata entity has not been set'); + } + this.cachedModels[METADATA_ENTITY_NAME] = new MetadataModel(this.metadataRepo) as any; + } + + return this.cachedModels[METADATA_ENTITY_NAME] as unknown as MetadataModel; + } + + get poi(): PlainPoiModel | null { + if (!this.cachedModels[POI_ENTITY_NAME]) { + if (!this.poiRepo) { + return null; + // throw new Error('Poi entity has not been set on store cache'); + } + this.cachedModels[POI_ENTITY_NAME] = new PlainPoiModel(this.poiRepo) as any; + } + + return this.cachedModels[POI_ENTITY_NAME] as unknown as PlainPoiModel; + } + + protected createModel(entityName: string): IModel { + const model = this.sequelize.model(entityName); + + const plainModel = new PlainModel(model, this.historical); + + if (this.config.csvOutDir) { + const exporterStore = new CsvStoreService(entityName, this.config.csvOutDir); + this.addExporter(plainModel, exporterStore); + } + + return plainModel; + } + + private addExporter(model: PlainModel, exporterStore: CsvStoreService): void { + throw new Error('Not implemented'); + } + + async applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise { + const tx = this.storeService.transaction; + if (!tx) { + exitWithError(new Error('Transaction not found'), logger, 1); + } + await tx.commit(); + + if (dataSourcesCompleted) { + const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; + exitWithError(msg, logger, 0); + } + } +} diff --git a/packages/node-core/src/indexer/storeCache/types.ts b/packages/node-core/src/indexer/storeCache/types.ts index 360e0aed73..8c27ff19a4 100644 --- a/packages/node-core/src/indexer/storeCache/types.ts +++ b/packages/node-core/src/indexer/storeCache/types.ts @@ -1,25 +1,12 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {FieldsExpression, GetOptions} from '@subql/types-core'; import {Transaction} from '@subql/x-sequelize'; import {LRUCache} from 'lru-cache'; import {SetValueModel} from './setValueModel'; export type HistoricalModel = {__block_range: any}; -export interface ICachedModel { - get: (id: string) => Promise; - // limit always defined from store - getByField: (field: keyof T, value: T[keyof T] | T[keyof T][], options?: GetOptions) => Promise; - getByFields: (filter: FieldsExpression[], options?: GetOptions) => Promise; - getOneByField: (field: keyof T, value: T[keyof T]) => Promise; - set: (id: string, data: T, blockHeight: number) => void; - bulkCreate: (data: T[], blockHeight: number) => void; - bulkUpdate: (data: T[], blockHeight: number, fields?: string[]) => void; - remove: (id: string, blockHeight: number) => void; - bulkRemove: (ids: string[], blockHeight: number) => void; -} export interface ICachedModelControl { isFlushable: boolean; hasAssociations?: boolean; diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts index 68907abd00..c8848a67f3 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts @@ -151,7 +151,7 @@ describe('UnfinalizedBlocksService', () => { // After this the call stack is something like: // indexerManager -> blockDispatcher -> project -> project -> reindex -> blockDispatcher.resetUnfinalizedBlocks - unfinalizedBlocksService.resetUnfinalizedBlocks(); + await unfinalizedBlocksService.resetUnfinalizedBlocks(); expect((unfinalizedBlocksService as any).unfinalizedBlocks).toEqual([]); }); @@ -251,7 +251,7 @@ describe('UnfinalizedBlocksService', () => { storeCache.init(true, false, {} as any, undefined); - storeCache.metadata.set( + await storeCache.metadata.set( METADATA_UNFINALIZED_BLOCKS_KEY, JSON.stringify([ {blockHeight: 90, blockHash: '0xabcd'}, @@ -259,7 +259,7 @@ describe('UnfinalizedBlocksService', () => { {blockHeight: 92, blockHash: '0xabc92'}, ]) ); - storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, 90); + await storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, 90); const unfinalizedBlocksService2 = new UnfinalizedBlocksService({unfinalizedBlocks: false} as any, storeCache); const reindex = jest.fn().mockReturnValue(Promise.resolve()); diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts index e6dca92ccf..f15d0466d7 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; +import {Transaction} from '@subql/x-sequelize'; import {isEqual, last} from 'lodash'; import {NodeConfig} from '../configure'; import {Header, IBlock} from '../indexer/types'; @@ -27,8 +28,8 @@ export interface IUnfinalizedBlocksService extends IUnfinalizedBlocksServiceU init(reindex: (targetHeight: number) => Promise): Promise; processUnfinalizedBlocks(block: IBlock | undefined): Promise; processUnfinalizedBlockHeader(header: Header | undefined): Promise; - resetUnfinalizedBlocks(): void; - resetLastFinalizedVerifiedHeight(): void; + resetUnfinalizedBlocks(tx?: Transaction): Promise; + resetLastFinalizedVerifiedHeight(tx?: Transaction): Promise; getMetadataUnfinalizedBlocks(): Promise; } @@ -69,10 +70,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo return this._finalizedHeader; } - constructor( - protected readonly nodeConfig: NodeConfig, - protected readonly storeCache: StoreCacheService - ) {} + constructor(protected readonly nodeConfig: NodeConfig, protected readonly storeCache: StoreCacheService) {} async init(reindex: (targetHeight: number) => Promise): Promise { logger.info(`Unfinalized blocks is ${this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'}`); @@ -94,8 +92,8 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo logger.info(`Successful rewind to block ${rewindHeight}!`); return rewindHeight; } else { - this.resetUnfinalizedBlocks(); - this.resetLastFinalizedVerifiedHeight(); + await this.resetUnfinalizedBlocks(); + await this.resetLastFinalizedVerifiedHeight(); } } } @@ -106,14 +104,14 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo async processUnfinalizedBlockHeader(header?: Header): Promise { if (header) { - this.registerUnfinalizedBlock(header); + await this.registerUnfinalizedBlock(header); } const forkedHeader = await this.hasForked(); if (!forkedHeader) { // Remove blocks that are now confirmed finalized - this.deleteFinalizedBlock(); + await this.deleteFinalizedBlock(); } else { // Get the last unfinalized block that is now finalized return this.getLastCorrectFinalizedBlock(forkedHeader); @@ -133,7 +131,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo this.finalizedHeader = header; } - private registerUnfinalizedBlock(header: Header): void { + private async registerUnfinalizedBlock(header: Header): Promise { if (header.blockHeight <= this.finalizedBlockNumber) return; // Ensure order @@ -146,14 +144,14 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo } this.unfinalizedBlocks.push(header); - this.saveUnfinalizedBlocks(this.unfinalizedBlocks); + await this.saveUnfinalizedBlocks(this.unfinalizedBlocks); } - private deleteFinalizedBlock(): void { + private async deleteFinalizedBlock(): Promise { if (this.lastCheckedBlockHeight !== undefined && this.lastCheckedBlockHeight < this.finalizedBlockNumber) { this.removeFinalized(this.finalizedBlockNumber); - this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber); - this.saveUnfinalizedBlocks(this.unfinalizedBlocks); + await this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber); + await this.saveUnfinalizedBlocks(this.unfinalizedBlocks); } this.lastCheckedBlockHeight = this.finalizedBlockNumber; } @@ -282,21 +280,21 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo throw new Error('Unable to find a POI block with matching block hash'); } - private saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): void { + private async saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): Promise { return this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, JSON.stringify(unfinalizedBlocks)); } - private saveLastFinalizedVerifiedHeight(height: number): void { + private async saveLastFinalizedVerifiedHeight(height: number): Promise { return this.storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, height); } - resetUnfinalizedBlocks(): void { - this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]'); + async resetUnfinalizedBlocks(tx?: Transaction): Promise { + await this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]', tx); this.unfinalizedBlocks = []; } - resetLastFinalizedVerifiedHeight(): void { - return this.storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, null as any); + async resetLastFinalizedVerifiedHeight(tx?: Transaction): Promise { + return this.storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, null as any, tx); } //string should be jsonb object diff --git a/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts b/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts index 59a0b46556..adfd3262de 100644 --- a/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts +++ b/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts @@ -31,10 +31,12 @@ export class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksServ init(reindex: (targetHeight: number) => Promise): Promise { throw new Error('This method should not be called from a worker'); } - resetUnfinalizedBlocks(): void { + // eslint-disable-next-line @typescript-eslint/require-await + async resetUnfinalizedBlocks(): Promise { throw new Error('This method should not be called from a worker'); } - resetLastFinalizedVerifiedHeight(): void { + // eslint-disable-next-line @typescript-eslint/require-await + async resetLastFinalizedVerifiedHeight(): Promise { throw new Error('This method should not be called from a worker'); } diff --git a/packages/node-core/src/meta/health.service.ts b/packages/node-core/src/meta/health.service.ts index e23cf542be..6bbcd1d49a 100644 --- a/packages/node-core/src/meta/health.service.ts +++ b/packages/node-core/src/meta/health.service.ts @@ -39,7 +39,7 @@ export class HealthService { } if (healthy !== this.indexerHealthy) { - await this.storeService.storeCache.metadata.model.upsert({key: 'indexerHealthy', value: healthy}); + await this.storeService.storeCache.metadata.set('indexerHealthy', healthy); this.indexerHealthy = healthy; } } diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index a93fb2c855..7c626853ff 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -6,7 +6,7 @@ import {Inject, Injectable} from '@nestjs/common'; import {BaseDataSource} from '@subql/types-core'; import {Sequelize} from '@subql/x-sequelize'; import {NodeConfig, ProjectUpgradeService} from '../configure'; -import {CacheMetadataModel, IUnfinalizedBlocksService, StoreService, ISubqueryProject, PoiService} from '../indexer'; +import {IUnfinalizedBlocksService, StoreService, ISubqueryProject, PoiService, IMetadata} from '../indexer'; import {DynamicDsService} from '../indexer/dynamic-ds.service'; import {getLogger} from '../logger'; import {exitWithError, monitorWrite} from '../process'; @@ -17,7 +17,7 @@ const logger = getLogger('Reindex'); @Injectable() export class ReindexService

{ - private _metadataRepo?: CacheMetadataModel; + private _metadataRepo?: IMetadata; private _lastProcessedHeight?: number; constructor( @@ -32,7 +32,7 @@ export class ReindexService

) {} - private get metadataRepo(): CacheMetadataModel { + private get metadataRepo(): IMetadata { assert(this._metadataRepo, 'BaseReindexService has not been init'); return this._metadataRepo; } diff --git a/packages/node-core/src/utils/reindex.ts b/packages/node-core/src/utils/reindex.ts index 384c8bf2a0..d66019a0d2 100644 --- a/packages/node-core/src/utils/reindex.ts +++ b/packages/node-core/src/utils/reindex.ts @@ -74,9 +74,9 @@ export async function reindex( await Promise.all([ storeService.rewind(targetBlockHeight, transaction), - unfinalizedBlockService.resetUnfinalizedBlocks(), // TODO: may not needed for nonfinalized chains - unfinalizedBlockService.resetLastFinalizedVerifiedHeight(), // TODO: may not needed for nonfinalized chains - dynamicDsService.resetDynamicDatasource(targetBlockHeight), + unfinalizedBlockService.resetUnfinalizedBlocks(transaction), // TODO: may not needed for nonfinalized chains + unfinalizedBlockService.resetLastFinalizedVerifiedHeight(transaction), // TODO: may not needed for nonfinalized chains + dynamicDsService.resetDynamicDatasource(targetBlockHeight, transaction), poiService?.rewind(targetBlockHeight, transaction), ]); // Flush metadata changes from above Promise.all From 838ae83e04cfae22562a9a66f9a5dff1393cbf6f Mon Sep 17 00:00:00 2001 From: Tate Date: Sun, 20 Oct 2024 10:13:24 +0000 Subject: [PATCH 03/39] Fix remaining build errors and update interfaces to abstract between flushing cache/managing db transaction --- .../src/configure/ProjectUpgrade.service.ts | 27 ++++++++--------- .../SchemaMigration.service.test.ts | 8 +---- .../SchemaMigration.service.ts | 7 ++--- .../blockDispatcher/base-block-dispatcher.ts | 2 +- .../node-core/src/indexer/poi/poi.service.ts | 16 +++++++--- .../node-core/src/indexer/project.service.ts | 8 ++--- .../node-core/src/indexer/store.service.ts | 29 +++++++++---------- packages/node-core/src/indexer/store/store.ts | 24 +++++++-------- .../indexer/storeCache/baseCache.service.ts | 6 ++-- .../indexer/storeCache/metadata/metadata.ts | 2 ++ .../indexer/storeCache/storeCache.service.ts | 8 ++--- .../indexer/storeCache/storeModel.service.ts | 13 ++++++++- packages/node-core/src/indexer/test.runner.ts | 4 +-- packages/node-core/src/meta/health.service.ts | 7 +++-- .../src/subcommands/reindex.service.ts | 4 +-- packages/node-core/src/utils/reindex.ts | 9 +++--- 16 files changed, 95 insertions(+), 79 deletions(-) diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.ts b/packages/node-core/src/configure/ProjectUpgrade.service.ts index 571d185c00..2e6e50e680 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.ts @@ -121,13 +121,20 @@ export class ProjectUpgradeService

; private migrationService?: SchemaMigrationService; - private constructor(private _projects: BlockHeightMap

, currentHeight: number, private _isRewindable = true) { + private constructor( + private _projects: BlockHeightMap

, + currentHeight: number, + private _isRewindable = true + ) { logger.info( `Projects: ${JSON.stringify( - [..._projects.getAll().entries()].reduce((acc, curr) => { - acc[curr[0]] = curr[1].id; - return acc; - }, {} as Record), + [..._projects.getAll().entries()].reduce( + (acc, curr) => { + acc[curr[0]] = curr[1].id; + return acc; + }, + {} as Record + ), undefined, 2 )}` @@ -150,16 +157,10 @@ export class ProjectUpgradeService

Promise, private dbSchema: string, private config: NodeConfig, private dbType: SUPPORT_DB = SUPPORT_DB.postgres @@ -115,7 +114,7 @@ export class SchemaMigrationService { const sortedAddedModels = alignModelOrder(sortedSchemaModels, addedModels); const sortedModifiedModels = alignModelOrder(sortedSchemaModels, modifiedModels); - await this.storeService.storeCache._flushCache(true); + await this.storeService.storeModel.flushData?.(true); const migrationAction = await Migration.create( this.sequelize, this.storeService, @@ -184,10 +183,10 @@ export class SchemaMigrationService { const modelChanges = await migrationAction.run(transaction); // Update any relevant application state so the right models are used - this.storeService.storeCache.updateModels(modelChanges); + this.storeService.storeModel.updateModels(modelChanges); await this.storeService.updateModels(this.dbSchema, getAllEntitiesRelations(nextSchema)); - await this.flushCache(); + await this.storeService.storeModel.flushData?.(true); } catch (e: any) { logger.error(e, 'Failed to execute Schema Migration'); throw e; diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index 02f8caaec7..68aa695b8a 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -218,7 +218,7 @@ export abstract class BaseBlockDispatcher implements IB await this.storeModelService.applyPendingChanges(height, !this.projectService.hasDataSourcesAfterHeight(height)); - // if (this.storeModelService instanceof StoreCacheService) { + // if (this.storeModelService instanceof StorCeacheService) { // if (this.nodeConfig.storeCacheAsync) { // // Flush all completed block data and don't wait // await this.storeModelService.flushAndWaitForCapacity(false)?.catch((e) => { diff --git a/packages/node-core/src/indexer/poi/poi.service.ts b/packages/node-core/src/indexer/poi/poi.service.ts index db932f2906..c99dccbde8 100644 --- a/packages/node-core/src/indexer/poi/poi.service.ts +++ b/packages/node-core/src/indexer/poi/poi.service.ts @@ -22,9 +22,12 @@ const logger = getLogger('PoiService'); @Injectable() export class PoiService implements OnApplicationShutdown { private isShutdown = false; - private _poiRepo?: CachePoiModel; + private _poiRepo?: IPoi; - constructor(protected readonly nodeConfig: NodeConfig, private storeCache: IStoreModelService) {} + constructor( + protected readonly nodeConfig: NodeConfig, + private storeCache: IStoreModelService + ) {} onApplicationShutdown(): void { this.isShutdown = true; @@ -40,7 +43,7 @@ export class PoiService implements OnApplicationShutdown { }; } - get poiRepo(): CachePoiModel { + get poiRepo(): IPoi { if (!this._poiRepo) { throw new Error(`No poi repo inited`); } @@ -48,7 +51,12 @@ export class PoiService implements OnApplicationShutdown { } get plainPoiRepo(): PlainPoiModel { - return this.poiRepo.plainPoiModel; + if (this.poiRepo instanceof CachePoiModel) { + return this.poiRepo.plainPoiModel; + } else if (this.poiRepo instanceof PlainPoiModel) { + return this.poiRepo; + } + throw new Error(`No plainPoiRepo repo inited`); } /** diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index 36853fcdeb..04f11c58b0 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -106,7 +106,7 @@ export abstract class BaseProjectService< await this.storeService.initCoreTables(this._schema); await this.ensureMetadata(); // DynamicDsService is dependent on metadata so we need to ensure it exists first - await this.dynamicDsService.init(this.storeService.storeCache.metadata); + await this.dynamicDsService.init(this.storeService.storeModel.metadata); /** * WARNING: The order of the following steps is very important. @@ -146,7 +146,7 @@ export abstract class BaseProjectService< } // Flush any pending operations to set up DB - await this.storeService.storeCache.flushCache(true); + await this.storeService.storeModel.flushData?.(true); } else { assert(startHeight, 'ProjectService must be initalized with a start height in workers'); this.projectUpgradeService.initWorker(startHeight, this.handleProjectChange.bind(this)); @@ -195,7 +195,7 @@ export abstract class BaseProjectService< } private async ensureMetadata(): Promise { - const metadata = this.storeService.storeCache.metadata; + const metadata = this.storeService.storeModel.metadata; this.eventEmitter.emit(IndexerEvent.NetworkMetadata, this.apiService.networkMeta); @@ -269,7 +269,7 @@ export abstract class BaseProjectService< } protected async getLastProcessedHeight(): Promise { - return this.storeService.storeCache.metadata.find('lastProcessedHeight'); + return this.storeService.storeModel.metadata.find('lastProcessedHeight'); } private async nextProcessHeight(): Promise { diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 37b568cce2..8d5b4986be 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -79,7 +79,7 @@ export class StoreService { constructor( private sequelize: Sequelize, private config: NodeConfig, - readonly storeCache: IStoreModelService, + readonly storeModel: IStoreModelService, @Inject('ISubqueryProject') private subqueryProject: ISubqueryProject ) {} @@ -125,7 +125,7 @@ export class StoreService { this._lastTimeDbSizeChecked = Date.now(); return getDbSizeAndUpdateMetadata(this.sequelize, this.schema); } else { - return this.storeCache.metadata.find('dbSize').then((cachedDbSize) => { + return this.storeModel.metadata.find('dbSize').then((cachedDbSize) => { if (cachedDbSize !== undefined) { return cachedDbSize; } else { @@ -177,9 +177,9 @@ export class StoreService { } logger.info(`Historical state is ${this.historical ? 'enabled' : 'disabled'}`); - this.storeCache.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); + this.storeModel.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); - this._metadataModel = this.storeCache.metadata; + this._metadataModel = this.storeModel.metadata; await this.initHotSchemaReloadQueries(schema); @@ -199,13 +199,7 @@ export class StoreService { On SyncSchema, if no schema migration is introduced, it would consider current schema to be null, and go all db operations again every start up is a migration */ - const schemaMigrationService = new SchemaMigrationService( - this.sequelize, - this, - this.storeCache._flushCache.bind(this.storeCache), - schema, - this.config - ); + const schemaMigrationService = new SchemaMigrationService(this.sequelize, this, schema, this.config); await schemaMigrationService.run(null, this.subqueryProject.schema, tx); @@ -330,10 +324,13 @@ export class StoreService { {type: QueryTypes.SELECT} ); - const store = res.reduce(function (total, current) { - total[current.key] = current.value; - return total; - }, {} as {[key: string]: string | boolean}); + const store = res.reduce( + function (total, current) { + total[current.key] = current.value; + return total; + }, + {} as {[key: string]: string | boolean} + ); const useHistorical = store.historicalStateEnabled === undefined ? !disableHistorical : (store.historicalStateEnabled as boolean); @@ -476,7 +473,7 @@ group by } getStore(): Store { - return new Store(this.config, this.storeCache, this); + return new Store(this.config, this.storeModel, this); } } diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index 173f6c99a6..5bc7f7d94b 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -27,12 +27,12 @@ type Context = { export class Store implements IStore { /* These need to explicily be private using JS style private properties in order to not leak these in the sandbox */ #config: NodeConfig; - #storeCache: IStoreModelService; + #storeModel: IStoreModelService; #context: Context; - constructor(config: NodeConfig, storeCache: IStoreModelService, context: Context) { + constructor(config: NodeConfig, storeModel: IStoreModelService, context: Context) { this.#config = config; - this.#storeCache = storeCache; + this.#storeModel = storeModel; this.#context = context; } @@ -52,7 +52,7 @@ export class Store implements IStore { async get(entity: string, id: string): Promise { try { - const raw = await this.#storeCache.getModel(entity).get(id); + const raw = await this.#storeModel.getModel(entity).get(id); monitorWrite(`-- [Store][get] Entity ${entity} ID ${id}, data: ${handledStringify(raw)}`); return EntityClass.create(entity, raw, this); } catch (e) { @@ -72,7 +72,7 @@ export class Store implements IStore { this.#queryLimitCheck('getByField', entity, options); - const raw = await this.#storeCache + const raw = await this.#storeModel .getModel(entity) .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], options); monitorWrite(`-- [Store][getByField] Entity ${entity}, data: ${handledStringify(raw)}`); @@ -98,7 +98,7 @@ export class Store implements IStore { this.#queryLimitCheck('getByFields', entity, options); - const raw = await this.#storeCache.getModel(entity).getByFields(filter, options); + const raw = await this.#storeModel.getModel(entity).getByFields(filter, options); monitorWrite(`-- [Store][getByFields] Entity ${entity}, data: ${handledStringify(raw)}`); return raw.map((v) => EntityClass.create(entity, v, this)) as T[]; } catch (e) { @@ -110,7 +110,7 @@ export class Store implements IStore { try { const indexed = this.#context.isIndexedHistorical(entity, field as string); assert(indexed, `to query by field ${String(field)}, a unique index must be created on model ${entity}`); - const [raw] = await this.#storeCache + const [raw] = await this.#storeModel .getModel(entity) .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], {limit: 1}); monitorWrite(`-- [Store][getOneByField] Entity ${entity}, data: ${handledStringify(raw)}`); @@ -122,7 +122,7 @@ export class Store implements IStore { async set(entity: string, _id: string, data: Entity): Promise { try { - await this.#storeCache.getModel(entity).set(_id, data, this.#context.blockHeight, this.#context.transaction); + await this.#storeModel.getModel(entity).set(_id, data, this.#context.blockHeight, this.#context.transaction); monitorWrite( `-- [Store][set] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` ); @@ -134,7 +134,7 @@ export class Store implements IStore { async bulkCreate(entity: string, data: Entity[]): Promise { try { - await this.#storeCache.getModel(entity).bulkCreate(data, this.#context.blockHeight, this.#context.transaction); + await this.#storeModel.getModel(entity).bulkCreate(data, this.#context.blockHeight, this.#context.transaction); for (const item of data) { this.#context.operationStack?.put(OperationType.Set, entity, item); } @@ -148,7 +148,7 @@ export class Store implements IStore { async bulkUpdate(entity: string, data: Entity[], fields?: string[]): Promise { try { - await this.#storeCache + await this.#storeModel .getModel(entity) .bulkUpdate(data, this.#context.blockHeight, fields, this.#context.transaction); for (const item of data) { @@ -164,7 +164,7 @@ export class Store implements IStore { async remove(entity: string, id: string): Promise { try { - await this.#storeCache.getModel(entity).bulkRemove([id], this.#context.blockHeight, this.#context.transaction); + await this.#storeModel.getModel(entity).bulkRemove([id], this.#context.blockHeight, this.#context.transaction); this.#context.operationStack?.put(OperationType.Remove, entity, id); monitorWrite(`-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, id: ${id}`); } catch (e) { @@ -174,7 +174,7 @@ export class Store implements IStore { async bulkRemove(entity: string, ids: string[]): Promise { try { - await this.#storeCache.getModel(entity).bulkRemove(ids, this.#context.blockHeight, this.#context.transaction); + await this.#storeModel.getModel(entity).bulkRemove(ids, this.#context.blockHeight, this.#context.transaction); for (const id of ids) { this.#context.operationStack?.put(OperationType.Remove, entity, id); diff --git a/packages/node-core/src/indexer/storeCache/baseCache.service.ts b/packages/node-core/src/indexer/storeCache/baseCache.service.ts index b0c25cd142..512d9d5398 100644 --- a/packages/node-core/src/indexer/storeCache/baseCache.service.ts +++ b/packages/node-core/src/indexer/storeCache/baseCache.service.ts @@ -30,7 +30,7 @@ export abstract class BaseCacheService } @profiler() - async flushCache(forceFlush?: boolean): Promise { + async flushData(forceFlush?: boolean): Promise { const flushCacheGuarded = async (forceFlush?: boolean): Promise => { // When we force flush, this will ensure not interrupt current block flushing, // Force flush will continue after last block flush tx committed. @@ -55,12 +55,12 @@ export abstract class BaseCacheService return this.queuedFlush; } - async resetCache(): Promise { + async resetData(): Promise { await this._resetCache(); } async beforeApplicationShutdown(): Promise { - await timeout(this.flushCache(true), 60, 'Before shutdown flush cache timeout'); + await timeout(this.flushData(true), 60, 'Before shutdown flush cache timeout'); this.logger.info(`Force flush cache successful!`); await this.flushExportStores(); this.logger.info(`Force flush exports successful!`); diff --git a/packages/node-core/src/indexer/storeCache/metadata/metadata.ts b/packages/node-core/src/indexer/storeCache/metadata/metadata.ts index 4c465c97a8..7ab4aded5a 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/metadata.ts +++ b/packages/node-core/src/indexer/storeCache/metadata/metadata.ts @@ -22,6 +22,8 @@ export interface IMetadata { setNewDynamicDatasource(item: DatasourceParams, tx?: Transaction): Promise; bulkRemove(keys: K[], tx?: Transaction): Promise; + + flush?(tx: Transaction, blockHeight: number): Promise; } export class MetadataModel implements IMetadata { diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.ts b/packages/node-core/src/indexer/storeCache/storeCache.service.ts index b8126b6648..895c204995 100644 --- a/packages/node-core/src/indexer/storeCache/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeCache.service.ts @@ -54,7 +54,7 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelSe this.schedulerRegistry.addInterval( 'storeFlushInterval', setInterval(() => { - this.flushCache(true).catch((e) => logger.warn(`storeFlushInterval failed ${e.message}`)); + this.flushData(true).catch((e) => logger.warn(`storeFlushInterval failed ${e.message}`)); }, this.config.storeFlushInterval * 1000) ); } @@ -177,7 +177,7 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelSe async flushAndWaitForCapacity(forceFlush?: boolean): Promise { const flushableRecords = this.flushableRecords; - const pendingFlush = this.flushCache(forceFlush); + const pendingFlush = this.flushData(forceFlush); if (flushableRecords >= this.cacheUpperLimit) { await pendingFlush; @@ -205,12 +205,12 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelSe }); } else { // Flush all data from cache and wait - await this.flushCache(false); + await this.flushData(false); } if (dataSourcesCompleted) { const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; - await this.flushCache(false); + await this.flushData(false); exitWithError(msg, logger, 0); } } diff --git a/packages/node-core/src/indexer/storeCache/storeModel.service.ts b/packages/node-core/src/indexer/storeCache/storeModel.service.ts index 1b199ec7f9..c1ac01b9d9 100644 --- a/packages/node-core/src/indexer/storeCache/storeModel.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeModel.service.ts @@ -5,6 +5,7 @@ import {Sequelize, ModelStatic} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {getLogger} from '../../logger'; import {exitWithError} from '../../process'; +import {MetadataRepo, PoiRepo} from '../entities'; import {StoreService} from '../store.service'; import {BaseStoreModelService} from './baseStoreModel.service'; import {CsvStoreService} from './csvStore.service'; @@ -19,6 +20,8 @@ export interface IStoreModelService { poi: IPoi | null; metadata: IMetadata; + init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void; + getModel(entity: string): IModel; // addExporter(entity: string, exporterStore: CsvStoreService): void; @@ -26,12 +29,20 @@ export interface IStoreModelService { applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; + + resetData?(): Promise; + + flushData?(forceFlush?: boolean): Promise; } const logger = getLogger('PlainStoreModelService'); export class PlainStoreModelService extends BaseStoreModelService implements IStoreModelService { - constructor(private sequelize: Sequelize, private config: NodeConfig, private storeService: StoreService) { + constructor( + private sequelize: Sequelize, + private config: NodeConfig, + private storeService: StoreService + ) { super(); } diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 0d37aa6b09..07be28a951 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -68,7 +68,7 @@ export class TestRunner { try { await indexBlock(block, test.handler, this.indexerManager, this.apiService); - await this.storeService.storeCache.flushCache(true); + await this.storeService.storeModel.flushData?.(true); } catch (e: any) { logger.warn(`Test: ${test.name} field due to runtime error`, e); this.failedTestSummary = { @@ -132,7 +132,7 @@ export class TestRunner { } } - await this.storeService.storeCache.flushCache(true); + await this.storeService.storeModel.flushData?.(true); logger.info( `Test: ${test.name} completed with ${chalk.green(`${this.passedTests} passed`)} and ${chalk.red( `${this.failedTests} failed` diff --git a/packages/node-core/src/meta/health.service.ts b/packages/node-core/src/meta/health.service.ts index 6bbcd1d49a..1124131099 100644 --- a/packages/node-core/src/meta/health.service.ts +++ b/packages/node-core/src/meta/health.service.ts @@ -22,7 +22,10 @@ export class HealthService { private healthTimeout: number; private indexerHealthy?: boolean; - constructor(protected nodeConfig: NodeConfig, private storeService: StoreService) { + constructor( + protected nodeConfig: NodeConfig, + private storeService: StoreService + ) { this.healthTimeout = Math.max(DEFAULT_TIMEOUT, this.nodeConfig.timeout * 1000); this.blockTime = Math.max(DEFAULT_BLOCK_TIME, this.nodeConfig.blockTime); } @@ -39,7 +42,7 @@ export class HealthService { } if (healthy !== this.indexerHealthy) { - await this.storeService.storeCache.metadata.set('indexerHealthy', healthy); + await this.storeService.storeModel.metadata.set('indexerHealthy', healthy); this.indexerHealthy = healthy; } } diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index 7c626853ff..3ff07cdd70 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -51,7 +51,7 @@ export class ReindexService

Date: Mon, 21 Oct 2024 10:37:51 +0000 Subject: [PATCH 04/39] flag `cache-disable` and Modifying inheritance relationships --- .../node-core/src/configure/NodeConfig.ts | 5 ++++ .../configure/ProjectUpgrade.service.spec.ts | 3 ++- packages/node-core/src/indexer/core.module.ts | 25 ++++++++++++++++--- .../node-core/src/indexer/store.service.ts | 5 ++-- .../storeCache/baseStoreModel.service.ts | 19 ++++++++++++-- .../indexer/storeCache/storeModel.service.ts | 6 ----- packages/node-core/src/yargs.ts | 5 ++++ 7 files changed, 54 insertions(+), 14 deletions(-) diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts index 8429a8796f..1485553daa 100644 --- a/packages/node-core/src/configure/NodeConfig.ts +++ b/packages/node-core/src/configure/NodeConfig.ts @@ -56,6 +56,7 @@ export interface IConfig { readonly csvOutDir?: string; readonly monitorOutDir: string; readonly monitorFileSize?: number; + readonly cacheDisable?: boolean; } export type MinConfig = Partial> & Pick; @@ -328,6 +329,10 @@ export class NodeConfig implements IConfig { return this._config.monitorFileSize ?? this._config.proofOfIndex ? defaultMonitorFileSize : 0; } + get cacheDisable(): boolean { + return this._config.cacheDisable || false; + } + merge(config: Partial): this { assign(this._config, config); return this; diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts b/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts index d1cda85ead..38da91aab3 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts @@ -4,6 +4,7 @@ import {SchedulerRegistry} from '@nestjs/schedule'; import {Sequelize} from '@subql/x-sequelize'; import {CacheMetadataModel, ISubqueryProject, StoreCacheService, StoreService} from '../indexer'; +import {BaseStoreModelService} from '../indexer/storeCache/baseStoreModel.service'; import {IStoreModelService} from '../indexer/storeCache/storeModel.service'; import {NodeConfig} from './NodeConfig'; import {IProjectUpgradeService, ProjectUpgradeService, upgradableSubqueryProject} from './ProjectUpgrade.service'; @@ -290,7 +291,7 @@ describe('Project Upgrades', () => { describe('Upgradable subquery project', () => { let upgradeService: ProjectUpgradeService; let project: ISubqueryProject & IProjectUpgradeService; - let storeCache: IStoreModelService; + let storeCache: BaseStoreModelService; beforeEach(async () => { storeCache = new StoreCacheService({} as any, {} as any, {} as any, new SchedulerRegistry()); diff --git a/packages/node-core/src/indexer/core.module.ts b/packages/node-core/src/indexer/core.module.ts index fe26a601d9..384618c849 100644 --- a/packages/node-core/src/indexer/core.module.ts +++ b/packages/node-core/src/indexer/core.module.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: GPL-3.0 import {Module} from '@nestjs/common'; +import {EventEmitter2} from '@nestjs/event-emitter'; +import {SchedulerRegistry} from '@nestjs/schedule'; +import {Sequelize} from '@subql/x-sequelize'; import {AdminController, AdminListener} from '../admin/admin.controller'; +import {NodeConfig} from '../configure'; import {IndexingBenchmarkService, PoiBenchmarkService} from './benchmark.service'; import {ConnectionPoolService} from './connectionPool.service'; import {ConnectionPoolStateManager} from './connectionPoolState.manager'; @@ -11,7 +15,8 @@ import {MonitorService} from './monitor.service'; import {PoiService, PoiSyncService} from './poi'; import {SandboxService} from './sandbox.service'; import {StoreService} from './store.service'; -import {StoreCacheService} from './storeCache'; +import {PlainStoreModelService, StoreCacheService} from './storeCache'; +import {BaseStoreModelService} from './storeCache/baseStoreModel.service'; @Module({ providers: [ @@ -25,7 +30,21 @@ import {StoreCacheService} from './storeCache'; PoiService, PoiSyncService, StoreService, - StoreCacheService, + { + provide: BaseStoreModelService, + useFactory: ( + storeService: StoreService, + nodeConfig: NodeConfig, + eventEmitter: EventEmitter2, + schedulerRegistry: SchedulerRegistry, + sequelize: Sequelize + ): BaseStoreModelService => { + return nodeConfig.cacheDisable + ? new PlainStoreModelService(sequelize, nodeConfig, storeService) + : new StoreCacheService(sequelize, nodeConfig, eventEmitter, schedulerRegistry); + }, + inject: [Sequelize, NodeConfig, EventEmitter2, SchedulerRegistry, StoreService], + }, AdminListener, ], controllers: [AdminController], @@ -37,7 +56,7 @@ import {StoreCacheService} from './storeCache'; PoiService, PoiSyncService, StoreService, - StoreCacheService, + BaseStoreModelService, InMemoryCacheService, ], }) diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 8d5b4986be..2d1dcf7e49 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -38,7 +38,8 @@ import {exitWithError} from '../process'; import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils'; import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities'; import {Store} from './store'; -import {IMetadata, IStoreModelService} from './storeCache'; +import {ICachedModelControl, IMetadata, IModel, IStoreModelService} from './storeCache'; +import {BaseStoreModelService} from './storeCache/baseStoreModel.service'; import {StoreOperations} from './StoreOperations'; import {ISubqueryProject} from './types'; @@ -79,7 +80,7 @@ export class StoreService { constructor( private sequelize: Sequelize, private config: NodeConfig, - readonly storeModel: IStoreModelService, + readonly storeModel: BaseStoreModelService>, @Inject('ISubqueryProject') private subqueryProject: ISubqueryProject ) {} diff --git a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts index e28957b7f1..bde57dbe4d 100644 --- a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts +++ b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts @@ -1,13 +1,17 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import {getLogger} from '@subql/node-core/logger'; import {ModelStatic} from '@subql/x-sequelize'; import {MetadataRepo, PoiRepo} from '../entities'; +import {IMetadata} from './metadata'; import {METADATA_ENTITY_NAME} from './metadata/utils'; import {BaseEntity, IModel} from './model'; -import {POI_ENTITY_NAME} from './poi'; +import {IPoi, POI_ENTITY_NAME} from './poi'; +import {IStoreModelService} from './storeModel.service'; -export abstract class BaseStoreModelService> { +const logger = getLogger('BaseStoreModelService'); +export abstract class BaseStoreModelService> implements IStoreModelService { protected historical = true; protected poiRepo?: PoiRepo; protected metadataRepo?: MetadataRepo; @@ -16,6 +20,10 @@ export abstract class BaseStoreModelService> { protected abstract createModel(entity: string): M; + abstract poi: IPoi | null; + abstract metadata: IMetadata; + abstract applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; + init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { this.historical = historical; this.metadataRepo = meta; @@ -42,4 +50,11 @@ export abstract class BaseStoreModelService> { }); removedModels.forEach((r) => delete this.cachedModels[r]); } + + resetData() { + logger.info('No need to resetData'); + } + flushData(forceFlush?: boolean) { + logger.info('No need to flushData'); + } } diff --git a/packages/node-core/src/indexer/storeCache/storeModel.service.ts b/packages/node-core/src/indexer/storeCache/storeModel.service.ts index c1ac01b9d9..28059f6b36 100644 --- a/packages/node-core/src/indexer/storeCache/storeModel.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeModel.service.ts @@ -20,8 +20,6 @@ export interface IStoreModelService { poi: IPoi | null; metadata: IMetadata; - init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void; - getModel(entity: string): IModel; // addExporter(entity: string, exporterStore: CsvStoreService): void; @@ -29,10 +27,6 @@ export interface IStoreModelService { applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; - - resetData?(): Promise; - - flushData?(forceFlush?: boolean): Promise; } const logger = getLogger('PlainStoreModelService'); diff --git a/packages/node-core/src/yargs.ts b/packages/node-core/src/yargs.ts index 646f372c98..98cb141097 100644 --- a/packages/node-core/src/yargs.ts +++ b/packages/node-core/src/yargs.ts @@ -256,6 +256,11 @@ export function yargsBuilder< describe: 'monitor file size limit in MB ', type: 'number', }, + 'cache-disable': { + describe: 'cache disable', + type: 'boolean', + default: false, + }, }) .hide('root'), // root is hidden because its for internal use handler: () => { From 10a4a57c89d7b9e90730989b036160cd10cb75ab Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 22 Oct 2024 09:17:30 +0000 Subject: [PATCH 05/39] rename storeStore to modelProvider --- .../node-core/src/configure/ProjectUpgrade.service.ts | 11 +++++++---- .../db/migration-service/SchemaMigration.service.ts | 6 +++--- packages/node-core/src/indexer/project.service.ts | 8 ++++---- packages/node-core/src/indexer/store.service.ts | 10 +++++----- .../src/indexer/storeCache/baseStoreModel.service.ts | 7 +++++-- packages/node-core/src/indexer/test.runner.ts | 4 ++-- packages/node-core/src/meta/health.service.ts | 2 +- packages/node-core/src/subcommands/reindex.service.ts | 4 ++-- packages/node-core/src/utils/reindex.ts | 8 ++++---- 9 files changed, 33 insertions(+), 27 deletions(-) diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.ts b/packages/node-core/src/configure/ProjectUpgrade.service.ts index 2e6e50e680..2978afc7e0 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.ts @@ -114,7 +114,7 @@ export class ProjectUpgradeService

(sortedSchemaModels, addedModels); const sortedModifiedModels = alignModelOrder(sortedSchemaModels, modifiedModels); - await this.storeService.storeModel.flushData?.(true); + await this.storeService.modelProvider.flushData(true); const migrationAction = await Migration.create( this.sequelize, this.storeService, @@ -183,10 +183,10 @@ export class SchemaMigrationService { const modelChanges = await migrationAction.run(transaction); // Update any relevant application state so the right models are used - this.storeService.storeModel.updateModels(modelChanges); + this.storeService.modelProvider.updateModels(modelChanges); await this.storeService.updateModels(this.dbSchema, getAllEntitiesRelations(nextSchema)); - await this.storeService.storeModel.flushData?.(true); + await this.storeService.modelProvider.flushData(true); } catch (e: any) { logger.error(e, 'Failed to execute Schema Migration'); throw e; diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index 04f11c58b0..1643e51ee4 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -106,7 +106,7 @@ export abstract class BaseProjectService< await this.storeService.initCoreTables(this._schema); await this.ensureMetadata(); // DynamicDsService is dependent on metadata so we need to ensure it exists first - await this.dynamicDsService.init(this.storeService.storeModel.metadata); + await this.dynamicDsService.init(this.storeService.modelProvider.metadata); /** * WARNING: The order of the following steps is very important. @@ -146,7 +146,7 @@ export abstract class BaseProjectService< } // Flush any pending operations to set up DB - await this.storeService.storeModel.flushData?.(true); + await this.storeService.modelProvider.flushData(true); } else { assert(startHeight, 'ProjectService must be initalized with a start height in workers'); this.projectUpgradeService.initWorker(startHeight, this.handleProjectChange.bind(this)); @@ -195,7 +195,7 @@ export abstract class BaseProjectService< } private async ensureMetadata(): Promise { - const metadata = this.storeService.storeModel.metadata; + const metadata = this.storeService.modelProvider.metadata; this.eventEmitter.emit(IndexerEvent.NetworkMetadata, this.apiService.networkMeta); @@ -269,7 +269,7 @@ export abstract class BaseProjectService< } protected async getLastProcessedHeight(): Promise { - return this.storeService.storeModel.metadata.find('lastProcessedHeight'); + return this.storeService.modelProvider.metadata.find('lastProcessedHeight'); } private async nextProcessHeight(): Promise { diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 2d1dcf7e49..e0c8cf7b33 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -80,7 +80,7 @@ export class StoreService { constructor( private sequelize: Sequelize, private config: NodeConfig, - readonly storeModel: BaseStoreModelService>, + readonly modelProvider: BaseStoreModelService>, @Inject('ISubqueryProject') private subqueryProject: ISubqueryProject ) {} @@ -126,7 +126,7 @@ export class StoreService { this._lastTimeDbSizeChecked = Date.now(); return getDbSizeAndUpdateMetadata(this.sequelize, this.schema); } else { - return this.storeModel.metadata.find('dbSize').then((cachedDbSize) => { + return this.modelProvider.metadata.find('dbSize').then((cachedDbSize) => { if (cachedDbSize !== undefined) { return cachedDbSize; } else { @@ -178,9 +178,9 @@ export class StoreService { } logger.info(`Historical state is ${this.historical ? 'enabled' : 'disabled'}`); - this.storeModel.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); + this.modelProvider.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); - this._metadataModel = this.storeModel.metadata; + this._metadataModel = this.modelProvider.metadata; await this.initHotSchemaReloadQueries(schema); @@ -474,7 +474,7 @@ group by } getStore(): Store { - return new Store(this.config, this.storeModel, this); + return new Store(this.config, this.modelProvider, this); } } diff --git a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts index bde57dbe4d..0854451eaa 100644 --- a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts +++ b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts @@ -51,10 +51,13 @@ export abstract class BaseStoreModelService> implements IStoreMo removedModels.forEach((r) => delete this.cachedModels[r]); } - resetData() { + // eslint-disable-next-line @typescript-eslint/require-await + async resetData() { logger.info('No need to resetData'); } - flushData(forceFlush?: boolean) { + + // eslint-disable-next-line @typescript-eslint/require-await + async flushData(forceFlush?: boolean) { logger.info('No need to flushData'); } } diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 07be28a951..4b1e16b7d2 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -68,7 +68,7 @@ export class TestRunner { try { await indexBlock(block, test.handler, this.indexerManager, this.apiService); - await this.storeService.storeModel.flushData?.(true); + await this.storeService.modelProvider.flushData(true); } catch (e: any) { logger.warn(`Test: ${test.name} field due to runtime error`, e); this.failedTestSummary = { @@ -132,7 +132,7 @@ export class TestRunner { } } - await this.storeService.storeModel.flushData?.(true); + await this.storeService.modelProvider.flushData(true); logger.info( `Test: ${test.name} completed with ${chalk.green(`${this.passedTests} passed`)} and ${chalk.red( `${this.failedTests} failed` diff --git a/packages/node-core/src/meta/health.service.ts b/packages/node-core/src/meta/health.service.ts index 1124131099..8d8b3b5438 100644 --- a/packages/node-core/src/meta/health.service.ts +++ b/packages/node-core/src/meta/health.service.ts @@ -42,7 +42,7 @@ export class HealthService { } if (healthy !== this.indexerHealthy) { - await this.storeService.storeModel.metadata.set('indexerHealthy', healthy); + await this.storeService.modelProvider.metadata.set('indexerHealthy', healthy); this.indexerHealthy = healthy; } } diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index 3ff07cdd70..9cd51b7c15 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -51,7 +51,7 @@ export class ReindexService

Date: Tue, 22 Oct 2024 09:44:16 +0000 Subject: [PATCH 06/39] rename #storeModel to #modelProvider --- packages/node-core/src/indexer/store/store.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index 5bc7f7d94b..74f1c796db 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -27,12 +27,12 @@ type Context = { export class Store implements IStore { /* These need to explicily be private using JS style private properties in order to not leak these in the sandbox */ #config: NodeConfig; - #storeModel: IStoreModelService; + #modelProvider: IStoreModelService; #context: Context; - constructor(config: NodeConfig, storeModel: IStoreModelService, context: Context) { + constructor(config: NodeConfig, modelProvider: IStoreModelService, context: Context) { this.#config = config; - this.#storeModel = storeModel; + this.#modelProvider = modelProvider; this.#context = context; } @@ -52,7 +52,7 @@ export class Store implements IStore { async get(entity: string, id: string): Promise { try { - const raw = await this.#storeModel.getModel(entity).get(id); + const raw = await this.#modelProvider.getModel(entity).get(id); monitorWrite(`-- [Store][get] Entity ${entity} ID ${id}, data: ${handledStringify(raw)}`); return EntityClass.create(entity, raw, this); } catch (e) { @@ -72,7 +72,7 @@ export class Store implements IStore { this.#queryLimitCheck('getByField', entity, options); - const raw = await this.#storeModel + const raw = await this.#modelProvider .getModel(entity) .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], options); monitorWrite(`-- [Store][getByField] Entity ${entity}, data: ${handledStringify(raw)}`); @@ -98,7 +98,7 @@ export class Store implements IStore { this.#queryLimitCheck('getByFields', entity, options); - const raw = await this.#storeModel.getModel(entity).getByFields(filter, options); + const raw = await this.#modelProvider.getModel(entity).getByFields(filter, options); monitorWrite(`-- [Store][getByFields] Entity ${entity}, data: ${handledStringify(raw)}`); return raw.map((v) => EntityClass.create(entity, v, this)) as T[]; } catch (e) { @@ -110,7 +110,7 @@ export class Store implements IStore { try { const indexed = this.#context.isIndexedHistorical(entity, field as string); assert(indexed, `to query by field ${String(field)}, a unique index must be created on model ${entity}`); - const [raw] = await this.#storeModel + const [raw] = await this.#modelProvider .getModel(entity) .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], {limit: 1}); monitorWrite(`-- [Store][getOneByField] Entity ${entity}, data: ${handledStringify(raw)}`); @@ -122,7 +122,7 @@ export class Store implements IStore { async set(entity: string, _id: string, data: Entity): Promise { try { - await this.#storeModel.getModel(entity).set(_id, data, this.#context.blockHeight, this.#context.transaction); + await this.#modelProvider.getModel(entity).set(_id, data, this.#context.blockHeight, this.#context.transaction); monitorWrite( `-- [Store][set] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` ); @@ -134,7 +134,7 @@ export class Store implements IStore { async bulkCreate(entity: string, data: Entity[]): Promise { try { - await this.#storeModel.getModel(entity).bulkCreate(data, this.#context.blockHeight, this.#context.transaction); + await this.#modelProvider.getModel(entity).bulkCreate(data, this.#context.blockHeight, this.#context.transaction); for (const item of data) { this.#context.operationStack?.put(OperationType.Set, entity, item); } @@ -148,7 +148,7 @@ export class Store implements IStore { async bulkUpdate(entity: string, data: Entity[], fields?: string[]): Promise { try { - await this.#storeModel + await this.#modelProvider .getModel(entity) .bulkUpdate(data, this.#context.blockHeight, fields, this.#context.transaction); for (const item of data) { @@ -164,7 +164,7 @@ export class Store implements IStore { async remove(entity: string, id: string): Promise { try { - await this.#storeModel.getModel(entity).bulkRemove([id], this.#context.blockHeight, this.#context.transaction); + await this.#modelProvider.getModel(entity).bulkRemove([id], this.#context.blockHeight, this.#context.transaction); this.#context.operationStack?.put(OperationType.Remove, entity, id); monitorWrite(`-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, id: ${id}`); } catch (e) { @@ -174,7 +174,7 @@ export class Store implements IStore { async bulkRemove(entity: string, ids: string[]): Promise { try { - await this.#storeModel.getModel(entity).bulkRemove(ids, this.#context.blockHeight, this.#context.transaction); + await this.#modelProvider.getModel(entity).bulkRemove(ids, this.#context.blockHeight, this.#context.transaction); for (const id of ids) { this.#context.operationStack?.put(OperationType.Remove, entity, id); From 33ad32c666644dfefaeaf8bc275ee373fddb2083 Mon Sep 17 00:00:00 2001 From: Tate Date: Fri, 25 Oct 2024 08:28:54 +0000 Subject: [PATCH 07/39] update cacheProvider flushData --- .../configure/ProjectUpgrade.service.spec.ts | 6 ++-- .../src/configure/ProjectUpgrade.service.ts | 11 ++------ .../SchemaMigration.service.ts | 12 +++++--- .../blockDispatcher/base-block-dispatcher.ts | 4 +-- .../worker-block-dispatcher.ts | 4 +-- .../node-core/src/indexer/poi/poi.service.ts | 4 +-- .../node-core/src/indexer/project.service.ts | 5 +++- .../node-core/src/indexer/store.service.ts | 9 +++--- packages/node-core/src/indexer/store/store.ts | 6 ++-- .../storeCache/baseStoreModel.service.ts | 20 ++----------- .../indexer/storeCache/storeCache.service.ts | 13 +++++++-- .../indexer/storeCache/storeModel.service.ts | 28 ++++++------------- .../node-core/src/indexer/storeCache/types.ts | 22 ++++++++++++++- packages/node-core/src/indexer/test.runner.ts | 9 ++++-- .../src/subcommands/reindex.service.ts | 14 ++++++++-- packages/node-core/src/utils/reindex.ts | 19 ++++++++++--- 16 files changed, 106 insertions(+), 80 deletions(-) diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts b/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts index 38da91aab3..6921b6cf39 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.spec.ts @@ -3,9 +3,7 @@ import {SchedulerRegistry} from '@nestjs/schedule'; import {Sequelize} from '@subql/x-sequelize'; -import {CacheMetadataModel, ISubqueryProject, StoreCacheService, StoreService} from '../indexer'; -import {BaseStoreModelService} from '../indexer/storeCache/baseStoreModel.service'; -import {IStoreModelService} from '../indexer/storeCache/storeModel.service'; +import {CacheMetadataModel, IStoreModelProvider, ISubqueryProject, StoreCacheService, StoreService} from '../indexer'; import {NodeConfig} from './NodeConfig'; import {IProjectUpgradeService, ProjectUpgradeService, upgradableSubqueryProject} from './ProjectUpgrade.service'; @@ -291,7 +289,7 @@ describe('Project Upgrades', () => { describe('Upgradable subquery project', () => { let upgradeService: ProjectUpgradeService; let project: ISubqueryProject & IProjectUpgradeService; - let storeCache: BaseStoreModelService; + let storeCache: IStoreModelProvider; beforeEach(async () => { storeCache = new StoreCacheService({} as any, {} as any, {} as any, new SchedulerRegistry()); diff --git a/packages/node-core/src/configure/ProjectUpgrade.service.ts b/packages/node-core/src/configure/ProjectUpgrade.service.ts index 2978afc7e0..09f8d8a535 100644 --- a/packages/node-core/src/configure/ProjectUpgrade.service.ts +++ b/packages/node-core/src/configure/ProjectUpgrade.service.ts @@ -7,14 +7,7 @@ import {ParentProject} from '@subql/types-core'; import {Sequelize, Transaction} from '@subql/x-sequelize'; import {findLast, last, parseInt} from 'lodash'; import {SchemaMigrationService} from '../db'; -import { - CacheMetadataModel, - IMetadata, - IStoreModelService, - ISubqueryProject, - StoreCacheService, - StoreService, -} from '../indexer'; +import {IMetadata, IStoreModelProvider, ISubqueryProject, StoreService} from '../indexer'; import {getLogger} from '../logger'; import {exitWithError, monitorWrite} from '../process'; import {getStartHeight, mainThreadOnly} from '../utils'; @@ -114,7 +107,7 @@ export class ProjectUpgradeService

(sortedSchemaModels, addedModels); const sortedModifiedModels = alignModelOrder(sortedSchemaModels, modifiedModels); - await this.storeService.modelProvider.flushData(true); + if (isCachePolicy(this.storeService.modelProvider)) { + await this.storeService.modelProvider.flushData(true); + } const migrationAction = await Migration.create( this.sequelize, this.storeService, @@ -186,7 +188,9 @@ export class SchemaMigrationService { this.storeService.modelProvider.updateModels(modelChanges); await this.storeService.updateModels(this.dbSchema, getAllEntitiesRelations(nextSchema)); - await this.storeService.modelProvider.flushData(true); + if (isCachePolicy(this.storeService.modelProvider)) { + await this.storeService.modelProvider.flushData(true); + } } catch (e: any) { logger.error(e, 'Failed to execute Schema Migration'); throw e; diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index 68aa695b8a..376edb77eb 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -15,7 +15,7 @@ import {MonitorServiceInterface} from '../monitor.service'; import {PoiBlock, PoiSyncService} from '../poi'; import {SmartBatchService} from '../smartBatch.service'; import {StoreService} from '../store.service'; -import {IStoreModelService} from '../storeCache'; +import {IStoreModelProvider} from '../storeCache'; import {IPoi} from '../storeCache/poi'; import {IBlock, IProjectService, ISubqueryProject} from '../types'; @@ -64,7 +64,7 @@ export abstract class BaseBlockDispatcher implements IB private projectUpgradeService: IProjectUpgradeService, protected queue: Q, protected storeService: StoreService, - private storeModelService: IStoreModelService, + private storeModelService: IStoreModelProvider, private poiSyncService: PoiSyncService, protected monitorService?: MonitorServiceInterface ) { diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index a271d1e8c2..df72c56c8f 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -15,7 +15,7 @@ import {monitorWrite} from '../../process'; import {AutoQueue, isTaskFlushedError} from '../../utils'; import {MonitorServiceInterface} from '../monitor.service'; import {StoreService} from '../store.service'; -import {IStoreModelService, StoreCacheService} from '../storeCache'; +import {IStoreModelProvider, StoreCacheService} from '../storeCache'; import {ISubqueryProject, IProjectService} from '../types'; import {isBlockUnavailableError} from '../worker/utils'; import {BaseBlockDispatcher} from './base-block-dispatcher'; @@ -58,7 +58,7 @@ export abstract class WorkerBlockDispatcher projectService: IProjectService, projectUpgradeService: IProjectUpgradeService, storeService: StoreService, - storeModelService: IStoreModelService, + storeModelService: IStoreModelProvider, poiSyncService: PoiSyncService, project: ISubqueryProject, private createIndexerWorker: () => Promise, diff --git a/packages/node-core/src/indexer/poi/poi.service.ts b/packages/node-core/src/indexer/poi/poi.service.ts index c99dccbde8..5739580ba1 100644 --- a/packages/node-core/src/indexer/poi/poi.service.ts +++ b/packages/node-core/src/indexer/poi/poi.service.ts @@ -9,7 +9,7 @@ import {sqlIterator} from '../../db'; import {getLogger} from '../../logger'; import {PoiRepo} from '../entities'; import {ProofOfIndex, ProofOfIndexHuman, SyncedProofOfIndex} from '../entities/Poi.entity'; -import {IStoreModelService} from '../storeCache'; +import {IStoreModelProvider} from '../storeCache'; import {IPoi, CachePoiModel, PlainPoiModel} from '../storeCache/poi'; const logger = getLogger('PoiService'); @@ -26,7 +26,7 @@ export class PoiService implements OnApplicationShutdown { constructor( protected readonly nodeConfig: NodeConfig, - private storeCache: IStoreModelService + private storeCache: IStoreModelProvider ) {} onApplicationShutdown(): void { diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index 1643e51ee4..bb25413bde 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -19,6 +19,7 @@ import {MetadataKeys} from './entities'; import {PoiSyncService} from './poi'; import {PoiService} from './poi/poi.service'; import {StoreService} from './store.service'; +import {isCachePolicy} from './storeCache'; import {ISubqueryProject, IProjectService} from './types'; import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; @@ -146,7 +147,9 @@ export abstract class BaseProjectService< } // Flush any pending operations to set up DB - await this.storeService.modelProvider.flushData(true); + if (isCachePolicy(this.storeService.modelProvider)) { + await this.storeService.modelProvider.flushData(true); + } } else { assert(startHeight, 'ProjectService must be initalized with a start height in workers'); this.projectUpgradeService.initWorker(startHeight, this.handleProjectChange.bind(this)); diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index e0c8cf7b33..c31804ea94 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -38,8 +38,7 @@ import {exitWithError} from '../process'; import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils'; import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities'; import {Store} from './store'; -import {ICachedModelControl, IMetadata, IModel, IStoreModelService} from './storeCache'; -import {BaseStoreModelService} from './storeCache/baseStoreModel.service'; +import {IMetadata, isCachePolicy, IStoreModelProvider} from './storeCache'; import {StoreOperations} from './StoreOperations'; import {ISubqueryProject} from './types'; @@ -80,7 +79,7 @@ export class StoreService { constructor( private sequelize: Sequelize, private config: NodeConfig, - readonly modelProvider: BaseStoreModelService>, + readonly modelProvider: IStoreModelProvider, @Inject('ISubqueryProject') private subqueryProject: ISubqueryProject ) {} @@ -178,7 +177,9 @@ export class StoreService { } logger.info(`Historical state is ${this.historical ? 'enabled' : 'disabled'}`); - this.modelProvider.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); + if (isCachePolicy(this.modelProvider)) { + this.modelProvider.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); + } this._metadataModel = this.modelProvider.metadata; diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index 74f1c796db..a83ae841bf 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -8,7 +8,7 @@ import {NodeConfig} from '../../configure'; import {getLogger} from '../../logger'; import {monitorWrite} from '../../process'; import {handledStringify} from '../../utils'; -import {IStoreModelService} from '../storeCache'; +import {IStoreModelProvider} from '../storeCache'; import {StoreOperations} from '../StoreOperations'; import {OperationType} from '../types'; import {EntityClass} from './entity'; @@ -27,10 +27,10 @@ type Context = { export class Store implements IStore { /* These need to explicily be private using JS style private properties in order to not leak these in the sandbox */ #config: NodeConfig; - #modelProvider: IStoreModelService; + #modelProvider: IStoreModelProvider; #context: Context; - constructor(config: NodeConfig, modelProvider: IStoreModelService, context: Context) { + constructor(config: NodeConfig, modelProvider: IStoreModelProvider, context: Context) { this.#config = config; this.#modelProvider = modelProvider; this.#context = context; diff --git a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts index 0854451eaa..60eeda1add 100644 --- a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts +++ b/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts @@ -4,14 +4,12 @@ import {getLogger} from '@subql/node-core/logger'; import {ModelStatic} from '@subql/x-sequelize'; import {MetadataRepo, PoiRepo} from '../entities'; -import {IMetadata} from './metadata'; import {METADATA_ENTITY_NAME} from './metadata/utils'; import {BaseEntity, IModel} from './model'; -import {IPoi, POI_ENTITY_NAME} from './poi'; -import {IStoreModelService} from './storeModel.service'; +import {POI_ENTITY_NAME} from './poi'; const logger = getLogger('BaseStoreModelService'); -export abstract class BaseStoreModelService> implements IStoreModelService { +export abstract class BaseStoreModelService> { protected historical = true; protected poiRepo?: PoiRepo; protected metadataRepo?: MetadataRepo; @@ -20,10 +18,6 @@ export abstract class BaseStoreModelService> implements IStoreMo protected abstract createModel(entity: string): M; - abstract poi: IPoi | null; - abstract metadata: IMetadata; - abstract applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; - init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { this.historical = historical; this.metadataRepo = meta; @@ -50,14 +44,4 @@ export abstract class BaseStoreModelService> implements IStoreMo }); removedModels.forEach((r) => delete this.cachedModels[r]); } - - // eslint-disable-next-line @typescript-eslint/require-await - async resetData() { - logger.info('No need to resetData'); - } - - // eslint-disable-next-line @typescript-eslint/require-await - async flushData(forceFlush?: boolean) { - logger.info('No need to flushData'); - } } diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.ts b/packages/node-core/src/indexer/storeCache/storeCache.service.ts index 895c204995..fcfc5c8f56 100644 --- a/packages/node-core/src/indexer/storeCache/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeCache.service.ts @@ -19,13 +19,16 @@ import {CacheMetadataModel} from './metadata'; import {METADATA_ENTITY_NAME} from './metadata/utils'; import {CachedModel} from './model'; import {CachePoiModel, POI_ENTITY_NAME} from './poi'; -import {IStoreModelService} from './storeModel.service'; -import {Exporter, ICachedModelControl} from './types'; +import {Exporter, ICachedModelControl, IStoreModelProvider, FlushPolicy} from './types'; const logger = getLogger('StoreCacheService'); +export function isCachePolicy(modelProvider: IStoreModelProvider): modelProvider is StoreCacheService { + return modelProvider.flushPolicy === FlushPolicy.Cache; +} + @Injectable() -export class StoreCacheService extends BaseCacheService implements IStoreModelService { +export class StoreCacheService extends BaseCacheService implements IStoreModelProvider { private readonly storeCacheThreshold: number; private readonly cacheUpperLimit: number; private _storeOperationIndex = 0; @@ -47,6 +50,10 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelSe } } + get flushPolicy() { + return FlushPolicy.Cache; + } + init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { super.init(historical, useCockroachDb, meta, poi); diff --git a/packages/node-core/src/indexer/storeCache/storeModel.service.ts b/packages/node-core/src/indexer/storeCache/storeModel.service.ts index 28059f6b36..c7290bd19f 100644 --- a/packages/node-core/src/indexer/storeCache/storeModel.service.ts +++ b/packages/node-core/src/indexer/storeCache/storeModel.service.ts @@ -1,37 +1,23 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Sequelize, ModelStatic} from '@subql/x-sequelize'; +import {Sequelize} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {getLogger} from '../../logger'; import {exitWithError} from '../../process'; -import {MetadataRepo, PoiRepo} from '../entities'; import {StoreService} from '../store.service'; import {BaseStoreModelService} from './baseStoreModel.service'; import {CsvStoreService} from './csvStore.service'; -import {IMetadata} from './metadata'; import {MetadataModel} from './metadata/metadata'; import {METADATA_ENTITY_NAME} from './metadata/utils'; -import {BaseEntity, IModel} from './model'; +import {IModel} from './model'; import {PlainModel} from './model/model'; -import {IPoi, PlainPoiModel, POI_ENTITY_NAME} from './poi'; - -export interface IStoreModelService { - poi: IPoi | null; - metadata: IMetadata; - - getModel(entity: string): IModel; - - // addExporter(entity: string, exporterStore: CsvStoreService): void; - - applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; - - updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; -} +import {PlainPoiModel, POI_ENTITY_NAME} from './poi'; +import {IStoreModelProvider, FlushPolicy} from './types'; const logger = getLogger('PlainStoreModelService'); -export class PlainStoreModelService extends BaseStoreModelService implements IStoreModelService { +export class PlainStoreModelService extends BaseStoreModelService implements IStoreModelProvider { constructor( private sequelize: Sequelize, private config: NodeConfig, @@ -40,6 +26,10 @@ export class PlainStoreModelService extends BaseStoreModelService implements ISt super(); } + get flushPolicy() { + return FlushPolicy.RealTime; + } + get metadata(): MetadataModel { if (!this.cachedModels[METADATA_ENTITY_NAME]) { if (!this.metadataRepo) { diff --git a/packages/node-core/src/indexer/storeCache/types.ts b/packages/node-core/src/indexer/storeCache/types.ts index 8c27ff19a4..c47b1f81cf 100644 --- a/packages/node-core/src/indexer/storeCache/types.ts +++ b/packages/node-core/src/indexer/storeCache/types.ts @@ -1,11 +1,31 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Transaction} from '@subql/x-sequelize'; +import {ENUM, ModelStatic, Transaction} from '@subql/x-sequelize'; import {LRUCache} from 'lru-cache'; +import {IMetadata} from './metadata'; +import {BaseEntity, IModel} from './model'; +import {IPoi} from './poi'; import {SetValueModel} from './setValueModel'; export type HistoricalModel = {__block_range: any}; +export enum FlushPolicy { + RealTime = 'realTime', + Cache = 'cached', +} +export interface IStoreModelProvider { + poi: IPoi | null; + metadata: IMetadata; + flushPolicy: FlushPolicy; + + getModel(entity: string): IModel; + + // addExporter(entity: string, exporterStore: CsvStoreService): void; + + applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; + + updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; +} export interface ICachedModelControl { isFlushable: boolean; diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 4b1e16b7d2..4cd68fce5f 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -11,6 +11,7 @@ import {NodeConfig} from '../configure/NodeConfig'; import {getLogger} from '../logger'; import {TestSandbox} from './sandbox'; import {StoreService} from './store.service'; +import {isCachePolicy} from './storeCache'; import {IBlock, IIndexerManager} from './types'; const logger = getLogger('test-runner'); @@ -68,7 +69,9 @@ export class TestRunner { try { await indexBlock(block, test.handler, this.indexerManager, this.apiService); - await this.storeService.modelProvider.flushData(true); + if (isCachePolicy(this.storeService.modelProvider)) { + await this.storeService.modelProvider.flushData(true); + } } catch (e: any) { logger.warn(`Test: ${test.name} field due to runtime error`, e); this.failedTestSummary = { @@ -132,7 +135,9 @@ export class TestRunner { } } - await this.storeService.modelProvider.flushData(true); + if (isCachePolicy(this.storeService.modelProvider)) { + await this.storeService.modelProvider.flushData(true); + } logger.info( `Test: ${test.name} completed with ${chalk.green(`${this.passedTests} passed`)} and ${chalk.red( `${this.failedTests} failed` diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index 9cd51b7c15..a3e2d3e203 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -6,7 +6,14 @@ import {Inject, Injectable} from '@nestjs/common'; import {BaseDataSource} from '@subql/types-core'; import {Sequelize} from '@subql/x-sequelize'; import {NodeConfig, ProjectUpgradeService} from '../configure'; -import {IUnfinalizedBlocksService, StoreService, ISubqueryProject, PoiService, IMetadata} from '../indexer'; +import { + IUnfinalizedBlocksService, + StoreService, + ISubqueryProject, + PoiService, + IMetadata, + isCachePolicy, +} from '../indexer'; import {DynamicDsService} from '../indexer/dynamic-ds.service'; import {getLogger} from '../logger'; import {exitWithError, monitorWrite} from '../process'; @@ -119,7 +126,10 @@ export class ReindexService

Date: Thu, 31 Oct 2024 09:38:16 +0000 Subject: [PATCH 08/39] remane storeModelProvider: IStoreModelProvider --- .../blockDispatcher/base-block-dispatcher.ts | 12 +++++------ .../blockDispatcher/block-dispatcher.ts | 6 +++--- .../worker-block-dispatcher.ts | 6 +++--- packages/node-core/src/indexer/core.module.ts | 9 ++++---- .../node-core/src/indexer/fetch.service.ts | 6 +++--- .../src/indexer/unfinalizedBlocks.service.ts | 21 +++++++++++-------- 6 files changed, 31 insertions(+), 29 deletions(-) diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index 376edb77eb..8371c0f4c7 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -64,7 +64,7 @@ export abstract class BaseBlockDispatcher implements IB private projectUpgradeService: IProjectUpgradeService, protected queue: Q, protected storeService: StoreService, - private storeModelService: IStoreModelProvider, + private storeModelProvider: IStoreModelProvider, private poiSyncService: PoiSyncService, protected monitorService?: MonitorServiceInterface ) { @@ -76,7 +76,7 @@ export abstract class BaseBlockDispatcher implements IB async init(onDynamicDsCreated: (height: number) => void): Promise { this._onDynamicDsCreated = onDynamicDsCreated; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.setProcessedBlockCount((await this.storeModelService.metadata.find('processedBlockCount', 0))!); + this.setProcessedBlockCount((await this.storeModelProvider.metadata.find('processedBlockCount', 0))!); } get queueSize(): number { @@ -216,7 +216,7 @@ export abstract class BaseBlockDispatcher implements IB this.setLatestProcessedHeight(height); } - await this.storeModelService.applyPendingChanges(height, !this.projectService.hasDataSourcesAfterHeight(height)); + await this.storeModelProvider.applyPendingChanges(height, !this.projectService.hasDataSourcesAfterHeight(height)); // if (this.storeModelService instanceof StorCeacheService) { // if (this.nodeConfig.storeCacheAsync) { @@ -292,7 +292,7 @@ export abstract class BaseBlockDispatcher implements IB const poiBlock = PoiBlock.create(height, blockHash, operationHash, this.project.id); // This is the first creation of POI await this.poi.bulkUpsert([poiBlock], tx); - await this.storeModelService.metadata.setBulk([{key: 'lastCreatedPoiHeight', value: height}], tx); + await this.storeModelProvider.metadata.setBulk([{key: 'lastCreatedPoiHeight', value: height}], tx); this.eventEmitter.emit(PoiEvent.PoiTarget, { height, timestamp: Date.now(), @@ -301,7 +301,7 @@ export abstract class BaseBlockDispatcher implements IB @mainThreadOnly() private async updateStoreMetadata(height: number, updateProcessed = true, tx?: Transaction): Promise { - const meta = this.storeModelService.metadata; + const meta = this.storeModelProvider.metadata; // Update store metadata await meta.setBulk( [ @@ -317,7 +317,7 @@ export abstract class BaseBlockDispatcher implements IB } private get poi(): IPoi { - const poi = this.storeModelService.poi; + const poi = this.storeModelProvider.poi; if (!poi) { throw new Error('Poi service expected poi repo but it was not found'); } diff --git a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts index 83071a864e..e549c5ab76 100644 --- a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts @@ -14,7 +14,7 @@ import {exitWithError, monitorWrite} from '../../process'; import {profilerWrap} from '../../profiler'; import {Queue, AutoQueue, delay, memoryLock, waitForBatchSize, isTaskFlushedError} from '../../utils'; import {StoreService} from '../store.service'; -import {StoreCacheService} from '../storeCache'; +import {IStoreModelProvider} from '../storeCache'; import {IProjectService, ISubqueryProject} from '../types'; import {BaseBlockDispatcher, ProcessBlockResponse} from './base-block-dispatcher'; @@ -45,7 +45,7 @@ export abstract class BlockDispatcher projectService: IProjectService, projectUpgradeService: IProjectUpgradeService, storeService: StoreService, - storeCacheService: StoreCacheService, + storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, project: ISubqueryProject, fetchBlocksBatches: BatchBlockFetcher @@ -58,7 +58,7 @@ export abstract class BlockDispatcher projectUpgradeService, new Queue(nodeConfig.batchSize * 3), storeService, - storeCacheService, + storeModelProvider, poiSyncService ); this.processQueue = new AutoQueue(nodeConfig.batchSize * 3, 1, nodeConfig.timeout, 'Process'); diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index df72c56c8f..88d3c60530 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -15,7 +15,7 @@ import {monitorWrite} from '../../process'; import {AutoQueue, isTaskFlushedError} from '../../utils'; import {MonitorServiceInterface} from '../monitor.service'; import {StoreService} from '../store.service'; -import {IStoreModelProvider, StoreCacheService} from '../storeCache'; +import {IStoreModelProvider} from '../storeCache'; import {ISubqueryProject, IProjectService} from '../types'; import {isBlockUnavailableError} from '../worker/utils'; import {BaseBlockDispatcher} from './base-block-dispatcher'; @@ -58,7 +58,7 @@ export abstract class WorkerBlockDispatcher projectService: IProjectService, projectUpgradeService: IProjectUpgradeService, storeService: StoreService, - storeModelService: IStoreModelProvider, + storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, project: ISubqueryProject, private createIndexerWorker: () => Promise, @@ -72,7 +72,7 @@ export abstract class WorkerBlockDispatcher projectUpgradeService, initAutoQueue(nodeConfig.workers, nodeConfig.batchSize, nodeConfig.timeout, 'Worker'), storeService, - storeModelService, + storeModelProvider, poiSyncService, monitorService ); diff --git a/packages/node-core/src/indexer/core.module.ts b/packages/node-core/src/indexer/core.module.ts index 384618c849..1a54c83348 100644 --- a/packages/node-core/src/indexer/core.module.ts +++ b/packages/node-core/src/indexer/core.module.ts @@ -15,8 +15,7 @@ import {MonitorService} from './monitor.service'; import {PoiService, PoiSyncService} from './poi'; import {SandboxService} from './sandbox.service'; import {StoreService} from './store.service'; -import {PlainStoreModelService, StoreCacheService} from './storeCache'; -import {BaseStoreModelService} from './storeCache/baseStoreModel.service'; +import {IStoreModelProvider, PlainStoreModelService, StoreCacheService} from './storeCache'; @Module({ providers: [ @@ -31,14 +30,14 @@ import {BaseStoreModelService} from './storeCache/baseStoreModel.service'; PoiSyncService, StoreService, { - provide: BaseStoreModelService, + provide: 'IStoreModelProvider', useFactory: ( storeService: StoreService, nodeConfig: NodeConfig, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, sequelize: Sequelize - ): BaseStoreModelService => { + ): IStoreModelProvider => { return nodeConfig.cacheDisable ? new PlainStoreModelService(sequelize, nodeConfig, storeService) : new StoreCacheService(sequelize, nodeConfig, eventEmitter, schedulerRegistry); @@ -56,7 +55,7 @@ import {BaseStoreModelService} from './storeCache/baseStoreModel.service'; PoiService, PoiSyncService, StoreService, - BaseStoreModelService, + 'IStoreModelProvider', InMemoryCacheService, ], }) diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index 5e80595589..d2dd0ba0d8 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -15,7 +15,7 @@ import {IBlockDispatcher} from './blockDispatcher'; import {mergeNumAndBlocksToNums} from './dictionary'; import {DictionaryService} from './dictionary/dictionary.service'; import {mergeNumAndBlocks} from './dictionary/utils'; -import {StoreCacheService} from './storeCache'; +import {IStoreModelProvider} from './storeCache'; import {BypassBlocks, Header, IBlock, IProjectService} from './types'; import {IUnfinalizedBlocksServiceUtil} from './unfinalizedBlocks.service'; @@ -51,7 +51,7 @@ export abstract class BaseFetchService implements IUnfinalizedBlo return this._finalizedHeader; } - constructor(protected readonly nodeConfig: NodeConfig, protected readonly storeCache: StoreCacheService) {} + constructor( + protected readonly nodeConfig: NodeConfig, + protected readonly storeModelProvider: IStoreModelProvider + ) {} async init(reindex: (targetHeight: number) => Promise): Promise { logger.info(`Unfinalized blocks is ${this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'}`); @@ -240,7 +243,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // Finds the last POI that had a correct block hash, this is used with the Eth sdk protected async findFinalizedUsingPOI(header: Header): Promise

{ - const poiModel = this.storeCache.poi; + const poiModel = this.storeModelProvider.poi; if (!poiModel) { throw new Error(POI_NOT_ENABLED_ERROR_MESSAGE); } @@ -281,25 +284,25 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo } private async saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): Promise { - return this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, JSON.stringify(unfinalizedBlocks)); + return this.storeModelProvider.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, JSON.stringify(unfinalizedBlocks)); } private async saveLastFinalizedVerifiedHeight(height: number): Promise { - return this.storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, height); + return this.storeModelProvider.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, height); } async resetUnfinalizedBlocks(tx?: Transaction): Promise { - await this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]', tx); + await this.storeModelProvider.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]', tx); this.unfinalizedBlocks = []; } async resetLastFinalizedVerifiedHeight(tx?: Transaction): Promise { - return this.storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, null as any, tx); + return this.storeModelProvider.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, null as any, tx); } //string should be jsonb object async getMetadataUnfinalizedBlocks(): Promise { - const val = await this.storeCache.metadata.find(METADATA_UNFINALIZED_BLOCKS_KEY); + const val = await this.storeModelProvider.metadata.find(METADATA_UNFINALIZED_BLOCKS_KEY); if (val) { return JSON.parse(val) as UnfinalizedBlocks; } @@ -307,6 +310,6 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo } async getLastFinalizedVerifiedHeight(): Promise { - return this.storeCache.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY); + return this.storeModelProvider.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY); } } From 0d515615af2e1de65c10e22db928d61fc7f06606 Mon Sep 17 00:00:00 2001 From: Tate Date: Fri, 1 Nov 2024 03:39:35 +0000 Subject: [PATCH 09/39] Rename the directory file to storeModelProvider; rename MetadataModel to MetadataEntity --- .../blockDispatcher/base-block-dispatcher.ts | 4 +- .../blockDispatcher/block-dispatcher.ts | 2 +- .../worker-block-dispatcher.spec.ts | 2 +- .../worker-block-dispatcher.ts | 2 +- packages/node-core/src/indexer/core.module.ts | 2 +- .../src/indexer/dynamic-ds.service.spec.ts | 2 +- .../src/indexer/dynamic-ds.service.ts | 2 +- .../src/indexer/entities/Metadata.entity.ts | 4 +- .../node-core/src/indexer/fetch.service.ts | 2 +- packages/node-core/src/indexer/index.ts | 2 +- .../src/indexer/poi/poi.service.spec.ts | 2 +- .../node-core/src/indexer/poi/poi.service.ts | 4 +- .../src/indexer/poi/poiSync.service.spec.ts | 2 +- .../src/indexer/poi/poiSync.service.ts | 2 +- .../node-core/src/indexer/project.service.ts | 2 +- .../node-core/src/indexer/store.service.ts | 2 +- packages/node-core/src/indexer/store/store.ts | 2 +- .../baseCache.service.ts | 0 .../baseStoreModel.service.ts | 0 .../cacheable.ts | 0 .../csvStore.service.spec.ts | 0 .../csvStore.service.ts | 5 +- .../index.ts | 0 .../metadata/cacheMetadata.spec.ts | 0 .../metadata/cacheMetadata.test.ts | 46 +++++++++---------- .../metadata/cacheMetadata.ts | 2 +- .../metadata/index.ts | 2 +- .../metadata/metadata.ts | 0 .../metadata/utils.ts | 0 .../model/cacheModel.spec.ts | 0 .../model/cacheModel.test.ts | 2 +- .../model/cacheModel.ts | 0 .../model/index.ts | 0 .../model/model.ts | 5 +- .../model/utils.ts | 2 +- .../poi/cachePoi.spec.ts | 0 .../poi/cachePoi.ts | 0 .../poi/index.ts | 0 .../poi/poi.ts | 0 .../setValueModel.spec.ts | 0 .../setValueModel.ts | 0 .../storeCache.service.spec.ts | 0 .../storeCache.service.ts | 0 .../storeModel.service.ts | 0 .../types.ts | 0 packages/node-core/src/indexer/test.runner.ts | 2 +- .../indexer/unfinalizedBlocks.service.spec.ts | 2 +- .../src/indexer/unfinalizedBlocks.service.ts | 2 +- 48 files changed, 58 insertions(+), 52 deletions(-) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/baseCache.service.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/baseStoreModel.service.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/cacheable.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/csvStore.service.spec.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/csvStore.service.ts (96%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/index.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/metadata/cacheMetadata.spec.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/metadata/cacheMetadata.test.ts (65%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/metadata/cacheMetadata.ts (99%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/metadata/index.ts (76%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/metadata/metadata.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/metadata/utils.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/model/cacheModel.spec.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/model/cacheModel.test.ts (99%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/model/cacheModel.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/model/index.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/model/model.ts (96%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/model/utils.ts (95%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/poi/cachePoi.spec.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/poi/cachePoi.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/poi/index.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/poi/poi.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/setValueModel.spec.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/setValueModel.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/storeCache.service.spec.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/storeCache.service.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/storeModel.service.ts (100%) rename packages/node-core/src/indexer/{storeCache => storeModelProvider}/types.ts (100%) diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index 8371c0f4c7..b78a71584f 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -15,8 +15,8 @@ import {MonitorServiceInterface} from '../monitor.service'; import {PoiBlock, PoiSyncService} from '../poi'; import {SmartBatchService} from '../smartBatch.service'; import {StoreService} from '../store.service'; -import {IStoreModelProvider} from '../storeCache'; -import {IPoi} from '../storeCache/poi'; +import {IStoreModelProvider} from '../storeModelProvider'; +import {IPoi} from '../storeModelProvider/poi'; import {IBlock, IProjectService, ISubqueryProject} from '../types'; const logger = getLogger('BaseBlockDispatcherService'); diff --git a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts index e549c5ab76..01ca467238 100644 --- a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts @@ -14,7 +14,7 @@ import {exitWithError, monitorWrite} from '../../process'; import {profilerWrap} from '../../profiler'; import {Queue, AutoQueue, delay, memoryLock, waitForBatchSize, isTaskFlushedError} from '../../utils'; import {StoreService} from '../store.service'; -import {IStoreModelProvider} from '../storeCache'; +import {IStoreModelProvider} from '../storeModelProvider'; import {IProjectService, ISubqueryProject} from '../types'; import {BaseBlockDispatcher, ProcessBlockResponse} from './base-block-dispatcher'; diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts index a1ce276196..cc8e5da01e 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts @@ -5,7 +5,7 @@ import {EventEmitter2} from '@nestjs/event-emitter'; import {IProjectUpgradeService, NodeConfig} from '../../configure'; import {PoiSyncService} from '../poi'; import {StoreService} from '../store.service'; -import {StoreCacheService} from '../storeCache'; +import {StoreCacheService} from '../storeModelProvider'; import {IProjectService, ISubqueryProject} from '../types'; import {WorkerBlockDispatcher} from './worker-block-dispatcher'; diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index 88d3c60530..8b105e1272 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -15,7 +15,7 @@ import {monitorWrite} from '../../process'; import {AutoQueue, isTaskFlushedError} from '../../utils'; import {MonitorServiceInterface} from '../monitor.service'; import {StoreService} from '../store.service'; -import {IStoreModelProvider} from '../storeCache'; +import {IStoreModelProvider} from '../storeModelProvider'; import {ISubqueryProject, IProjectService} from '../types'; import {isBlockUnavailableError} from '../worker/utils'; import {BaseBlockDispatcher} from './base-block-dispatcher'; diff --git a/packages/node-core/src/indexer/core.module.ts b/packages/node-core/src/indexer/core.module.ts index 1a54c83348..53637925d5 100644 --- a/packages/node-core/src/indexer/core.module.ts +++ b/packages/node-core/src/indexer/core.module.ts @@ -15,7 +15,7 @@ import {MonitorService} from './monitor.service'; import {PoiService, PoiSyncService} from './poi'; import {SandboxService} from './sandbox.service'; import {StoreService} from './store.service'; -import {IStoreModelProvider, PlainStoreModelService, StoreCacheService} from './storeCache'; +import {IStoreModelProvider, PlainStoreModelService, StoreCacheService} from './storeModelProvider'; @Module({ providers: [ diff --git a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts index fc3b0807bb..6c7ca7be7d 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import {DatasourceParams, DynamicDsService} from './dynamic-ds.service'; -import {CacheMetadataModel} from './storeCache'; +import {CacheMetadataModel} from './storeModelProvider'; import {ISubqueryProject} from './types'; class TestDynamicDsService extends DynamicDsService { diff --git a/packages/node-core/src/indexer/dynamic-ds.service.ts b/packages/node-core/src/indexer/dynamic-ds.service.ts index 5e214ebd49..9dae8d6245 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.ts @@ -5,7 +5,7 @@ import {Transaction} from '@subql/x-sequelize'; import {cloneDeep} from 'lodash'; import {getLogger} from '../logger'; import {exitWithError} from '../process'; -import {IMetadata} from './storeCache'; +import {IMetadata} from './storeModelProvider'; import {ISubqueryProject} from './types'; const logger = getLogger('dynamic-ds'); diff --git a/packages/node-core/src/indexer/entities/Metadata.entity.ts b/packages/node-core/src/indexer/entities/Metadata.entity.ts index e02224dee5..b3d7056442 100644 --- a/packages/node-core/src/indexer/entities/Metadata.entity.ts +++ b/packages/node-core/src/indexer/entities/Metadata.entity.ts @@ -39,10 +39,10 @@ export interface Metadata { value: MetadataKeys[keyof MetadataKeys]; } -export interface MetadataModel extends Model, Metadata {} +interface MetadataEntity extends Model, Metadata {} export type MetadataRepo = typeof Model & { - new (values?: unknown, options?: BuildOptions): MetadataModel; + new (values?: unknown, options?: BuildOptions): MetadataEntity; }; async function checkSchemaMetadata(sequelize: Sequelize, schema: string, chainId: string): Promise { diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index d2dd0ba0d8..82eefc5127 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -15,7 +15,7 @@ import {IBlockDispatcher} from './blockDispatcher'; import {mergeNumAndBlocksToNums} from './dictionary'; import {DictionaryService} from './dictionary/dictionary.service'; import {mergeNumAndBlocks} from './dictionary/utils'; -import {IStoreModelProvider} from './storeCache'; +import {IStoreModelProvider} from './storeModelProvider'; import {BypassBlocks, Header, IBlock, IProjectService} from './types'; import {IUnfinalizedBlocksServiceUtil} from './unfinalizedBlocks.service'; diff --git a/packages/node-core/src/indexer/index.ts b/packages/node-core/src/indexer/index.ts index a72711e53e..39791422b7 100644 --- a/packages/node-core/src/indexer/index.ts +++ b/packages/node-core/src/indexer/index.ts @@ -9,7 +9,7 @@ export * from './types'; export * from './StoreOperations'; export * from './store.service'; export * from './inMemoryCache.service'; -export * from './storeCache'; +export * from './storeModelProvider'; export * from './worker'; export * from './dictionary'; export * from './sandbox'; diff --git a/packages/node-core/src/indexer/poi/poi.service.spec.ts b/packages/node-core/src/indexer/poi/poi.service.spec.ts index 09b5ac54c9..8fc787f900 100644 --- a/packages/node-core/src/indexer/poi/poi.service.spec.ts +++ b/packages/node-core/src/indexer/poi/poi.service.spec.ts @@ -8,7 +8,7 @@ import {delay} from '@subql/common'; import {Sequelize, Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {ProofOfIndex} from '../entities/Poi.entity'; -import {StoreCacheService} from '../storeCache'; +import {StoreCacheService} from '../storeModelProvider'; import {PoiService} from './poi.service'; jest.mock('@subql/x-sequelize', () => { diff --git a/packages/node-core/src/indexer/poi/poi.service.ts b/packages/node-core/src/indexer/poi/poi.service.ts index 5739580ba1..69acd96f87 100644 --- a/packages/node-core/src/indexer/poi/poi.service.ts +++ b/packages/node-core/src/indexer/poi/poi.service.ts @@ -9,8 +9,8 @@ import {sqlIterator} from '../../db'; import {getLogger} from '../../logger'; import {PoiRepo} from '../entities'; import {ProofOfIndex, ProofOfIndexHuman, SyncedProofOfIndex} from '../entities/Poi.entity'; -import {IStoreModelProvider} from '../storeCache'; -import {IPoi, CachePoiModel, PlainPoiModel} from '../storeCache/poi'; +import {IStoreModelProvider} from '../storeModelProvider'; +import {IPoi, CachePoiModel, PlainPoiModel} from '../storeModelProvider/poi'; const logger = getLogger('PoiService'); diff --git a/packages/node-core/src/indexer/poi/poiSync.service.spec.ts b/packages/node-core/src/indexer/poi/poiSync.service.spec.ts index d154b22777..fa30f84b18 100644 --- a/packages/node-core/src/indexer/poi/poiSync.service.spec.ts +++ b/packages/node-core/src/indexer/poi/poiSync.service.spec.ts @@ -8,7 +8,7 @@ import {range} from 'lodash'; import {NodeConfig} from '../../configure'; import {MetadataFactory, PoiFactory, ProofOfIndex} from '../../indexer'; import {Queue} from '../../utils'; -import {PlainPoiModel} from '../storeCache/poi'; +import {PlainPoiModel} from '../storeModelProvider/poi'; import {ISubqueryProject} from '../types'; import {PoiSyncService} from './poiSync.service'; diff --git a/packages/node-core/src/indexer/poi/poiSync.service.ts b/packages/node-core/src/indexer/poi/poiSync.service.ts index fb4e3451d3..93254b036d 100644 --- a/packages/node-core/src/indexer/poi/poiSync.service.ts +++ b/packages/node-core/src/indexer/poi/poiSync.service.ts @@ -14,7 +14,7 @@ import {exitWithError} from '../../process'; import {hasValue, Queue} from '../../utils'; import {Metadata, MetadataFactory, MetadataRepo} from '../entities'; import {PoiFactory, ProofOfIndex, SyncedProofOfIndex} from '../entities/Poi.entity'; -import {PlainPoiModel} from '../storeCache/poi'; +import {PlainPoiModel} from '../storeModelProvider/poi'; import {ISubqueryProject} from '../types'; import {PoiBlock} from './PoiBlock'; diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index 09048f666a..a632ae9edb 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -19,7 +19,7 @@ import {MetadataKeys} from './entities'; import {PoiSyncService} from './poi'; import {PoiService} from './poi/poi.service'; import {StoreService} from './store.service'; -import {isCachePolicy} from './storeCache'; +import {isCachePolicy} from './storeModelProvider'; import {ISubqueryProject, IProjectService, BypassBlocks} from './types'; import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index c31804ea94..e406d57959 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -38,7 +38,7 @@ import {exitWithError} from '../process'; import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils'; import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities'; import {Store} from './store'; -import {IMetadata, isCachePolicy, IStoreModelProvider} from './storeCache'; +import {IMetadata, isCachePolicy, IStoreModelProvider} from './storeModelProvider'; import {StoreOperations} from './StoreOperations'; import {ISubqueryProject} from './types'; diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index 87a4ca78eb..e157eed9d8 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -7,7 +7,7 @@ import {Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {monitorWrite} from '../../process'; import {handledStringify} from '../../utils'; -import {IStoreModelProvider} from '../storeCache'; +import {IStoreModelProvider} from '../storeModelProvider'; import {StoreOperations} from '../StoreOperations'; import {OperationType} from '../types'; import {EntityClass} from './entity'; diff --git a/packages/node-core/src/indexer/storeCache/baseCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/baseCache.service.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/baseCache.service.ts rename to packages/node-core/src/indexer/storeModelProvider/baseCache.service.ts diff --git a/packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/baseStoreModel.service.ts rename to packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts diff --git a/packages/node-core/src/indexer/storeCache/cacheable.ts b/packages/node-core/src/indexer/storeModelProvider/cacheable.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/cacheable.ts rename to packages/node-core/src/indexer/storeModelProvider/cacheable.ts diff --git a/packages/node-core/src/indexer/storeCache/csvStore.service.spec.ts b/packages/node-core/src/indexer/storeModelProvider/csvStore.service.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/csvStore.service.spec.ts rename to packages/node-core/src/indexer/storeModelProvider/csvStore.service.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/csvStore.service.ts b/packages/node-core/src/indexer/storeModelProvider/csvStore.service.ts similarity index 96% rename from packages/node-core/src/indexer/storeCache/csvStore.service.ts rename to packages/node-core/src/indexer/storeModelProvider/csvStore.service.ts index 116310ac55..7b7ddadca8 100644 --- a/packages/node-core/src/indexer/storeCache/csvStore.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/csvStore.service.ts @@ -14,7 +14,10 @@ export class CsvStoreService implements Exporter { private stringifyStream: Stringifier; private readonly writeStream: fs.WriteStream; - constructor(private modelName: string, private outputPath: string) { + constructor( + private modelName: string, + private outputPath: string + ) { this.writeStream = fs.createWriteStream(this.getCsvFilePath(), {flags: 'a'}); this.stringifyStream = stringify({header: !this.fileExist}).on('error', (err) => { diff --git a/packages/node-core/src/indexer/storeCache/index.ts b/packages/node-core/src/indexer/storeModelProvider/index.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/index.ts rename to packages/node-core/src/indexer/storeModelProvider/index.ts diff --git a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.spec.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.spec.ts rename to packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.test.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts similarity index 65% rename from packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.test.ts rename to packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts index 3255e07bbf..73594431de 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.test.ts +++ b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts @@ -1,10 +1,10 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Sequelize } from '@subql/x-sequelize'; -import { MetadataFactory, MetadataKeys, MetadataRepo } from '../../'; -import { DbOption } from '../../../'; -import { CacheMetadataModel } from './cacheMetadata'; +import {Sequelize} from '@subql/x-sequelize'; +import {MetadataFactory, MetadataKeys, MetadataRepo} from '../..'; +import {DbOption} from '../../..'; +import {CacheMetadataModel} from './cacheMetadata'; const option: DbOption = { host: process.env.DB_HOST ?? '127.0.0.1', @@ -49,7 +49,7 @@ describe('cacheMetadata integration', () => { }; afterAll(async () => { - await sequelize.dropSchema(schema, { logging: false }); + await sequelize.dropSchema(schema, {logging: false}); await sequelize.close(); }); @@ -70,18 +70,18 @@ describe('cacheMetadata integration', () => { describe('dynamicDatasources', () => { beforeEach(async () => { // Ensure value exits so we can update it - await metaDataRepo.bulkCreate([{ key: 'dynamicDatasources', value: [] }], { updateOnDuplicate: ['key', 'value'] }); + await metaDataRepo.bulkCreate([{key: 'dynamicDatasources', value: []}], {updateOnDuplicate: ['key', 'value']}); }); it('Appends dynamicDatasources correctly', async () => { - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'bar', startBlock: 2 }); - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'baz', startBlock: 3 }); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'bar', startBlock: 2}); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'baz', startBlock: 3}); const expected = [ - { templateName: 'foo', startBlock: 1 }, - { templateName: 'bar', startBlock: 2 }, - { templateName: 'baz', startBlock: 3 }, + {templateName: 'foo', startBlock: 1}, + {templateName: 'bar', startBlock: 2}, + {templateName: 'baz', startBlock: 3}, ]; await flush(); @@ -94,40 +94,40 @@ describe('cacheMetadata integration', () => { }); it('Allows overriding all dynamicDatasources', async () => { - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); - cacheMetadataModel.set('dynamicDatasources', [{ templateName: 'bar', startBlock: 2 }]); + cacheMetadataModel.set('dynamicDatasources', [{templateName: 'bar', startBlock: 2}]); await flush(); const v = await queryMeta('dynamicDatasources'); - expect(v).toEqual([{ templateName: 'bar', startBlock: 2 }]); + expect(v).toEqual([{templateName: 'bar', startBlock: 2}]); }); it('Caches the dynamicDatasources correctly after using set', async () => { - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); await flush(); const cacheV = await cacheMetadataModel.find('dynamicDatasources'); - expect(cacheV).toEqual([{ templateName: 'foo', startBlock: 1 }]); + expect(cacheV).toEqual([{templateName: 'foo', startBlock: 1}]); - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'bar', startBlock: 2 }); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'bar', startBlock: 2}); // await flush(); const cacheV2 = await cacheMetadataModel.find('dynamicDatasources'); expect(cacheV2).toEqual([ - { templateName: 'foo', startBlock: 1 }, - { templateName: 'bar', startBlock: 2 }, + {templateName: 'foo', startBlock: 1}, + {templateName: 'bar', startBlock: 2}, ]); }); it('Uses the correct cache values when using new and set', async () => { - cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 }); + cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1}); - cacheMetadataModel.set('dynamicDatasources', [{ templateName: 'bar', startBlock: 2 }]); + cacheMetadataModel.set('dynamicDatasources', [{templateName: 'bar', startBlock: 2}]); const cacheV = await cacheMetadataModel.find('dynamicDatasources'); - expect(cacheV).toEqual([{ templateName: 'bar', startBlock: 2 }]); + expect(cacheV).toEqual([{templateName: 'bar', startBlock: 2}]); }); }); }); diff --git a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts similarity index 99% rename from packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts rename to packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts index fcac1df9eb..d070a131c1 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/cacheMetadata.ts +++ b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts @@ -167,7 +167,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM async runFlush(tx: Transaction, blockHeight?: number): Promise { const ops = Object.entries(this.setCache) .filter(([key]) => !specialKeys.includes(key as MetadataKey)) - .map(([key, value]) => ({key, value} as Metadata)); + .map(([key, value]) => ({key, value}) as Metadata); const lastProcessedHeightIdx = ops.findIndex((k) => k.key === 'lastProcessedHeight'); if (blockHeight !== undefined && lastProcessedHeightIdx >= 0) { diff --git a/packages/node-core/src/indexer/storeCache/metadata/index.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/index.ts similarity index 76% rename from packages/node-core/src/indexer/storeCache/metadata/index.ts rename to packages/node-core/src/indexer/storeModelProvider/metadata/index.ts index 961566f4d0..94df86710c 100644 --- a/packages/node-core/src/indexer/storeCache/metadata/index.ts +++ b/packages/node-core/src/indexer/storeModelProvider/metadata/index.ts @@ -2,4 +2,4 @@ // SPDX-License-Identifier: GPL-3.0 export * from './cacheMetadata'; -export { IMetadata } from './metadata'; +export {IMetadata} from './metadata'; diff --git a/packages/node-core/src/indexer/storeCache/metadata/metadata.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/metadata.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/metadata/metadata.ts rename to packages/node-core/src/indexer/storeModelProvider/metadata/metadata.ts diff --git a/packages/node-core/src/indexer/storeCache/metadata/utils.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/utils.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/metadata/utils.ts rename to packages/node-core/src/indexer/storeModelProvider/metadata/utils.ts diff --git a/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts rename to packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.test.ts similarity index 99% rename from packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts rename to packages/node-core/src/indexer/storeModelProvider/model/cacheModel.test.ts index eff3b0b774..a7163ecfe5 100644 --- a/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.test.ts @@ -4,7 +4,7 @@ import {GraphQLModelsType} from '@subql/utils'; import {Sequelize, DataTypes, QueryTypes} from '@subql/x-sequelize'; import {cloneDeep, padStart} from 'lodash'; -import {DbOption, modelsTypeToModelAttributes, NodeConfig} from '../../../'; +import {DbOption, modelsTypeToModelAttributes, NodeConfig} from '../../..'; import {CachedModel} from './cacheModel'; const option: DbOption = { diff --git a/packages/node-core/src/indexer/storeCache/model/cacheModel.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/model/cacheModel.ts rename to packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts diff --git a/packages/node-core/src/indexer/storeCache/model/index.ts b/packages/node-core/src/indexer/storeModelProvider/model/index.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/model/index.ts rename to packages/node-core/src/indexer/storeModelProvider/model/index.ts diff --git a/packages/node-core/src/indexer/storeCache/model/model.ts b/packages/node-core/src/indexer/storeModelProvider/model/model.ts similarity index 96% rename from packages/node-core/src/indexer/storeCache/model/model.ts rename to packages/node-core/src/indexer/storeModelProvider/model/model.ts index 5d8b9d74d1..822424b796 100644 --- a/packages/node-core/src/indexer/storeCache/model/model.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/model.ts @@ -21,7 +21,10 @@ export interface IModel { } export class PlainModel implements IModel { - constructor(readonly model: ModelStatic>, private readonly historical = true) {} + constructor( + readonly model: ModelStatic>, + private readonly historical = true + ) {} async get(id: string): Promise { const record = await this.model.findOne({ diff --git a/packages/node-core/src/indexer/storeCache/model/utils.ts b/packages/node-core/src/indexer/storeModelProvider/model/utils.ts similarity index 95% rename from packages/node-core/src/indexer/storeCache/model/utils.ts rename to packages/node-core/src/indexer/storeModelProvider/model/utils.ts index 9abb6b0ddf..5c1aa9c564 100644 --- a/packages/node-core/src/indexer/storeCache/model/utils.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/utils.ts @@ -23,4 +23,4 @@ export const getFullOptions = (options?: GetOptions): Required), ...options, - } as Required>); + }) as Required>; diff --git a/packages/node-core/src/indexer/storeCache/poi/cachePoi.spec.ts b/packages/node-core/src/indexer/storeModelProvider/poi/cachePoi.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/poi/cachePoi.spec.ts rename to packages/node-core/src/indexer/storeModelProvider/poi/cachePoi.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/poi/cachePoi.ts b/packages/node-core/src/indexer/storeModelProvider/poi/cachePoi.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/poi/cachePoi.ts rename to packages/node-core/src/indexer/storeModelProvider/poi/cachePoi.ts diff --git a/packages/node-core/src/indexer/storeCache/poi/index.ts b/packages/node-core/src/indexer/storeModelProvider/poi/index.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/poi/index.ts rename to packages/node-core/src/indexer/storeModelProvider/poi/index.ts diff --git a/packages/node-core/src/indexer/storeCache/poi/poi.ts b/packages/node-core/src/indexer/storeModelProvider/poi/poi.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/poi/poi.ts rename to packages/node-core/src/indexer/storeModelProvider/poi/poi.ts diff --git a/packages/node-core/src/indexer/storeCache/setValueModel.spec.ts b/packages/node-core/src/indexer/storeModelProvider/setValueModel.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/setValueModel.spec.ts rename to packages/node-core/src/indexer/storeModelProvider/setValueModel.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/setValueModel.ts b/packages/node-core/src/indexer/storeModelProvider/setValueModel.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/setValueModel.ts rename to packages/node-core/src/indexer/storeModelProvider/setValueModel.ts diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.spec.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.spec.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/storeCache.service.spec.ts rename to packages/node-core/src/indexer/storeModelProvider/storeCache.service.spec.ts diff --git a/packages/node-core/src/indexer/storeCache/storeCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/storeCache.service.ts rename to packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts diff --git a/packages/node-core/src/indexer/storeCache/storeModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/storeModel.service.ts rename to packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts diff --git a/packages/node-core/src/indexer/storeCache/types.ts b/packages/node-core/src/indexer/storeModelProvider/types.ts similarity index 100% rename from packages/node-core/src/indexer/storeCache/types.ts rename to packages/node-core/src/indexer/storeModelProvider/types.ts diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 16eff11b22..1ac0492ccf 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -11,7 +11,7 @@ import {NodeConfig} from '../configure/NodeConfig'; import {getLogger} from '../logger'; import {TestSandbox} from './sandbox'; import {StoreService} from './store.service'; -import {isCachePolicy} from './storeCache'; +import {isCachePolicy} from './storeModelProvider'; import {IBlock, IIndexerManager} from './types'; const logger = getLogger('test-runner'); diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts index c8848a67f3..a8fc670fe4 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts @@ -5,7 +5,7 @@ import {EventEmitter2} from '@nestjs/event-emitter'; import {SchedulerRegistry} from '@nestjs/schedule'; import {Header, IBlock} from '../indexer'; -import {StoreCacheService, CacheMetadataModel} from './storeCache'; +import {StoreCacheService, CacheMetadataModel} from './storeModelProvider'; import { METADATA_LAST_FINALIZED_PROCESSED_KEY, METADATA_UNFINALIZED_BLOCKS_KEY, diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts index 3356cdfab3..51d20f3ecc 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts @@ -11,7 +11,7 @@ import {exitWithError} from '../process'; import {mainThreadOnly} from '../utils'; import {ProofOfIndex} from './entities'; import {PoiBlock} from './poi'; -import {IStoreModelProvider} from './storeCache'; +import {IStoreModelProvider} from './storeModelProvider'; const logger = getLogger('UnfinalizedBlocks'); From b428e6ec939d4651573a5c3395524400149a3d81 Mon Sep 17 00:00:00 2001 From: Tate Date: Fri, 1 Nov 2024 06:10:29 +0000 Subject: [PATCH 10/39] node package update provider StoreCacheService to IStoreModelProvider --- .../indexer/blockDispatcher/block-dispatcher.service.ts | 6 +++--- .../blockDispatcher/worker-block-dispatcher.service.ts | 6 +++--- packages/node/src/indexer/fetch.module.ts | 7 ++++--- packages/node/src/indexer/fetch.service.ts | 6 +++--- packages/node/src/indexer/unfinalizedBlocks.service.ts | 6 +++--- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index d5e74ad5dd..1c93b8eb61 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -6,7 +6,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - StoreCacheService, StoreService, IProjectService, BlockDispatcher, @@ -14,6 +13,7 @@ import { IProjectUpgradeService, PoiSyncService, IBlock, + IStoreModelProvider, } from '@subql/node-core'; import { SubstrateDatasource } from '@subql/types'; import { SubqueryProject } from '../../configure/SubqueryProject'; @@ -42,7 +42,7 @@ export class BlockDispatcherService @Inject('IProjectUpgradeService') projectUpgradeService: IProjectUpgradeService, storeService: StoreService, - storeCacheService: StoreCacheService, + storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, ) { @@ -52,7 +52,7 @@ export class BlockDispatcherService projectService, projectUpgradeService, storeService, - storeCacheService, + storeModelProvider, poiSyncService, project, async ( diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 200f84754c..107c6bbf15 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -8,7 +8,6 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, StoreService, - StoreCacheService, IProjectService, WorkerBlockDispatcher, ConnectionPoolStateManager, @@ -17,6 +16,7 @@ import { InMemoryCacheService, createIndexerWorker as createIndexerWorkerCore, MonitorServiceInterface, + IStoreModelProvider, } from '@subql/node-core'; import { SubstrateBlock, SubstrateDatasource } from '@subql/types'; import { SubqueryProject } from '../../configure/SubqueryProject'; @@ -51,7 +51,7 @@ export class WorkerBlockDispatcherService projectUpgadeService: IProjectUpgradeService, cacheService: InMemoryCacheService, storeService: StoreService, - storeCacheService: StoreCacheService, + storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, dynamicDsService: DynamicDsService, @@ -65,7 +65,7 @@ export class WorkerBlockDispatcherService projectService, projectUpgadeService, storeService, - storeCacheService, + storeModelProvider, poiSyncService, project, () => diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 218a01919f..1284aec87a 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -13,6 +13,7 @@ import { InMemoryCacheService, MonitorService, CoreModule, + IStoreModelProvider, } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; @@ -46,7 +47,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; indexerManager: IndexerManager, cacheService: InMemoryCacheService, storeService: StoreService, - storeCacheService: StoreCacheService, + storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, project: SubqueryProject, dynamicDsService: DynamicDsService, @@ -62,7 +63,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; projectUpgradeService, cacheService, storeService, - storeCacheService, + storeModelProvider, poiSyncService, project, dynamicDsService, @@ -78,7 +79,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; projectService, projectUpgradeService, storeService, - storeCacheService, + storeModelProvider, poiSyncService, project, ), diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index d3927e5c8b..61c3fe617c 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -12,7 +12,7 @@ import { BaseFetchService, getModulos, Header, - StoreCacheService, + IStoreModelProvider, } from '@subql/node-core'; import { SubstrateDatasource, SubstrateBlock } from '@subql/types'; import { calcInterval, substrateHeaderToHeader } from '../utils/substrate'; @@ -43,7 +43,7 @@ export class FetchService extends BaseFetchService< eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, private runtimeService: RuntimeService, - storeCacheService: StoreCacheService, + storeModelProvider: IStoreModelProvider, ) { super( nodeConfig, @@ -53,7 +53,7 @@ export class FetchService extends BaseFetchService< eventEmitter, schedulerRegistry, unfinalizedBlocksService, - storeCacheService, + storeModelProvider, ); } diff --git a/packages/node/src/indexer/unfinalizedBlocks.service.ts b/packages/node/src/indexer/unfinalizedBlocks.service.ts index 196564212a..5ca3b84289 100644 --- a/packages/node/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/unfinalizedBlocks.service.ts @@ -5,9 +5,9 @@ import { Injectable } from '@nestjs/common'; import { BaseUnfinalizedBlocksService, Header, + IStoreModelProvider, mainThreadOnly, NodeConfig, - StoreCacheService, } from '@subql/node-core'; import { substrateHeaderToHeader } from '../utils/substrate'; import { ApiService } from './api.service'; @@ -20,9 +20,9 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService< constructor( private readonly apiService: ApiService, nodeConfig: NodeConfig, - storeCache: StoreCacheService, + storeModelProvider: IStoreModelProvider, ) { - super(nodeConfig, storeCache); + super(nodeConfig, storeModelProvider); } @mainThreadOnly() From 200361ba7fb3f42e18db6c7fbded7d7fb86d6efd Mon Sep 17 00:00:00 2001 From: Tate Date: Sun, 3 Nov 2024 16:30:41 +0800 Subject: [PATCH 11/39] =?UTF-8?q?fix=20=C2=A0storeService=20dependency=20l?= =?UTF-8?q?oop=20and=20repeat=20db=20transaction=20creation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../blockDispatcher/base-block-dispatcher.ts | 8 ++++++-- packages/node-core/src/indexer/core.module.ts | 5 ++--- .../node-core/src/indexer/poi/poi.service.ts | 18 +++++++++--------- .../node-core/src/indexer/store.service.ts | 14 ++++++++------ packages/node-core/src/indexer/store/store.ts | 3 ++- .../storeModelProvider/storeModel.service.ts | 10 +++++----- .../src/indexer/storeModelProvider/types.ts | 2 +- packages/node-core/src/utils/index.ts | 2 +- .../block-dispatcher.service.ts | 2 +- .../worker-block-dispatcher.service.ts | 2 +- packages/node/src/indexer/fetch.module.ts | 2 +- packages/node/src/indexer/fetch.service.ts | 2 +- .../src/indexer/unfinalizedBlocks.service.ts | 4 ++-- 13 files changed, 40 insertions(+), 34 deletions(-) diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index b78a71584f..c6437f133c 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -158,7 +158,7 @@ export abstract class BaseBlockDispatcher implements IB @mainThreadOnly() protected async preProcessBlock(height: number): Promise { monitorCreateBlockStart(height); - this.storeService.setBlockHeight(height); + await this.storeService.setBlockHeight(height); await this.projectUpgradeService.setCurrentHeight(height); @@ -216,7 +216,11 @@ export abstract class BaseBlockDispatcher implements IB this.setLatestProcessedHeight(height); } - await this.storeModelProvider.applyPendingChanges(height, !this.projectService.hasDataSourcesAfterHeight(height)); + await this.storeModelProvider.applyPendingChanges( + height, + !this.projectService.hasDataSourcesAfterHeight(height), + this.storeService.transaction + ); // if (this.storeModelService instanceof StorCeacheService) { // if (this.nodeConfig.storeCacheAsync) { diff --git a/packages/node-core/src/indexer/core.module.ts b/packages/node-core/src/indexer/core.module.ts index 53637925d5..a9aefc9b30 100644 --- a/packages/node-core/src/indexer/core.module.ts +++ b/packages/node-core/src/indexer/core.module.ts @@ -32,17 +32,16 @@ import {IStoreModelProvider, PlainStoreModelService, StoreCacheService} from './ { provide: 'IStoreModelProvider', useFactory: ( - storeService: StoreService, nodeConfig: NodeConfig, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, sequelize: Sequelize ): IStoreModelProvider => { return nodeConfig.cacheDisable - ? new PlainStoreModelService(sequelize, nodeConfig, storeService) + ? new PlainStoreModelService(sequelize, nodeConfig) : new StoreCacheService(sequelize, nodeConfig, eventEmitter, schedulerRegistry); }, - inject: [Sequelize, NodeConfig, EventEmitter2, SchedulerRegistry, StoreService], + inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize], }, AdminListener, ], diff --git a/packages/node-core/src/indexer/poi/poi.service.ts b/packages/node-core/src/indexer/poi/poi.service.ts index 69acd96f87..67bfcd7e15 100644 --- a/packages/node-core/src/indexer/poi/poi.service.ts +++ b/packages/node-core/src/indexer/poi/poi.service.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Injectable, OnApplicationShutdown} from '@nestjs/common'; +import {Inject, Injectable, OnApplicationShutdown} from '@nestjs/common'; import {u8aToHex} from '@subql/utils'; import {Op, QueryTypes, Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; @@ -26,7 +26,7 @@ export class PoiService implements OnApplicationShutdown { constructor( protected readonly nodeConfig: NodeConfig, - private storeCache: IStoreModelProvider + @Inject('IStoreModelProvider') private storeModelProvider: IStoreModelProvider ) {} onApplicationShutdown(): void { @@ -65,11 +65,11 @@ export class PoiService implements OnApplicationShutdown { * @param schema */ async init(schema: string): Promise { - this._poiRepo = this.storeCache.poi ?? undefined; + this._poiRepo = this.storeModelProvider.poi ?? undefined; if (!this._poiRepo) { return; } - const latestSyncedPoiHeight = await this.storeCache.metadata.find('latestSyncedPoiHeight'); + const latestSyncedPoiHeight = await this.storeModelProvider.metadata.find('latestSyncedPoiHeight'); if (latestSyncedPoiHeight === undefined) { await this.migratePoi(schema); } @@ -98,7 +98,7 @@ export class PoiService implements OnApplicationShutdown { }); // Drop previous keys in metadata - await this.storeCache.metadata.bulkRemove(['blockOffset', 'latestPoiWithMmr', 'lastPoiHeight']); + await this.storeModelProvider.metadata.bulkRemove(['blockOffset', 'latestPoiWithMmr', 'lastPoiHeight']); const queries: string[] = []; @@ -170,7 +170,7 @@ export class PoiService implements OnApplicationShutdown { async rewind(targetBlockHeight: number, transaction: Transaction): Promise { await batchDeletePoi(this.poiRepo.model, transaction, targetBlockHeight); - const lastSyncedPoiHeight = await this.storeCache.metadata.find('latestSyncedPoiHeight'); + const lastSyncedPoiHeight = await this.storeModelProvider.metadata.find('latestSyncedPoiHeight'); if (lastSyncedPoiHeight !== undefined && lastSyncedPoiHeight > targetBlockHeight) { const genesisPoi = await this.poiRepo.model.findOne({ @@ -180,12 +180,12 @@ export class PoiService implements OnApplicationShutdown { // This indicates reindex height is less than genesis poi height // And genesis poi has been remove from `batchDeletePoi` if (!genesisPoi) { - await this.storeCache.metadata.bulkRemove(['latestSyncedPoiHeight'], transaction); + await this.storeModelProvider.metadata.bulkRemove(['latestSyncedPoiHeight'], transaction); } else { - await this.storeCache.metadata.set('latestSyncedPoiHeight', targetBlockHeight, transaction); + await this.storeModelProvider.metadata.set('latestSyncedPoiHeight', targetBlockHeight, transaction); } } - await this.storeCache.metadata.bulkRemove(['lastCreatedPoiHeight'], transaction); + await this.storeModelProvider.metadata.bulkRemove(['lastCreatedPoiHeight'], transaction); } } diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index e406d57959..47f2168624 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -79,7 +79,7 @@ export class StoreService { constructor( private sequelize: Sequelize, private config: NodeConfig, - readonly modelProvider: IStoreModelProvider, + @Inject('IStoreModelProvider') readonly modelProvider: IStoreModelProvider, @Inject('ISubqueryProject') private subqueryProject: ISubqueryProject ) {} @@ -358,11 +358,13 @@ export class StoreService { async setBlockHeight(blockHeight: number): Promise { this._blockHeight = blockHeight; - // TODO do we need to set hooks for block height? - this.#transaction = await this.sequelize.transaction({ - deferrable: this._historical || this.dbType === SUPPORT_DB.cockRoach ? undefined : Deferrable.SET_DEFERRED(), - }); - this.#transaction.afterCommit(() => (this.#transaction = undefined)); + if (!this.#transaction) { + // TODO do we need to set hooks for block height? + this.#transaction = await this.sequelize.transaction({ + deferrable: this._historical || this.dbType === SUPPORT_DB.cockRoach ? undefined : Deferrable.SET_DEFERRED(), + }); + this.#transaction.afterCommit(() => (this.#transaction = undefined)); + } if (this.config.proofOfIndex) { this.operationStack = new StoreOperations(this.modelsRelations.models); diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index e157eed9d8..bd518b7599 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; +import {Inject} from '@nestjs/common'; import {Store as IStore, Entity, FieldsExpression, GetOptions} from '@subql/types-core'; import {Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; @@ -27,7 +28,7 @@ export class Store implements IStore { #modelProvider: IStoreModelProvider; #context: Context; - constructor(config: NodeConfig, modelProvider: IStoreModelProvider, context: Context) { + constructor(config: NodeConfig, @Inject('IStoreModelProvider') modelProvider: IStoreModelProvider, context: Context) { this.#config = config; this.#modelProvider = modelProvider; this.#context = context; diff --git a/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts index c7290bd19f..908baa5253 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts @@ -1,7 +1,8 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Sequelize} from '@subql/x-sequelize'; +import {Injectable} from '@nestjs/common'; +import {Sequelize, Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {getLogger} from '../../logger'; import {exitWithError} from '../../process'; @@ -17,11 +18,11 @@ import {IStoreModelProvider, FlushPolicy} from './types'; const logger = getLogger('PlainStoreModelService'); +@Injectable() export class PlainStoreModelService extends BaseStoreModelService implements IStoreModelProvider { constructor( private sequelize: Sequelize, - private config: NodeConfig, - private storeService: StoreService + private config: NodeConfig ) { super(); } @@ -70,8 +71,7 @@ export class PlainStoreModelService extends BaseStoreModelService implements ISt throw new Error('Not implemented'); } - async applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise { - const tx = this.storeService.transaction; + async applyPendingChanges(height: number, dataSourcesCompleted: boolean, tx: Transaction): Promise { if (!tx) { exitWithError(new Error('Transaction not found'), logger, 1); } diff --git a/packages/node-core/src/indexer/storeModelProvider/types.ts b/packages/node-core/src/indexer/storeModelProvider/types.ts index c47b1f81cf..8a6ac410b8 100644 --- a/packages/node-core/src/indexer/storeModelProvider/types.ts +++ b/packages/node-core/src/indexer/storeModelProvider/types.ts @@ -22,7 +22,7 @@ export interface IStoreModelProvider { // addExporter(entity: string, exporterStore: CsvStoreService): void; - applyPendingChanges(height: number, dataSourcesCompleted: boolean): Promise; + applyPendingChanges(height: number, dataSourcesCompleted: boolean, tx?: Transaction): Promise; updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; } diff --git a/packages/node-core/src/utils/index.ts b/packages/node-core/src/utils/index.ts index 76748a97b6..53fa95d549 100644 --- a/packages/node-core/src/utils/index.ts +++ b/packages/node-core/src/utils/index.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +export * from './decorators'; export * from './object'; export * from './promise'; export * from './graphql'; @@ -11,7 +12,6 @@ export * from './fetchHelpers'; export * from './blockSizeBuffer'; export * from './configure'; export * from './reindex'; -export * from './decorators'; export * from './blocks'; export * from './blockHeightMap'; export * from './string'; diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index 1c93b8eb61..91d4178cf8 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -42,7 +42,7 @@ export class BlockDispatcherService @Inject('IProjectUpgradeService') projectUpgradeService: IProjectUpgradeService, storeService: StoreService, - storeModelProvider: IStoreModelProvider, + @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, ) { diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 107c6bbf15..bf9be4346f 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -51,7 +51,7 @@ export class WorkerBlockDispatcherService projectUpgadeService: IProjectUpgradeService, cacheService: InMemoryCacheService, storeService: StoreService, - storeModelProvider: IStoreModelProvider, + @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, dynamicDsService: DynamicDsService, diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 1284aec87a..acf65ad23b 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -92,7 +92,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; IndexerManager, InMemoryCacheService, StoreService, - StoreCacheService, + 'IStoreModelProvider', PoiSyncService, 'ISubqueryProject', DynamicDsService, diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 61c3fe617c..9f2eacd163 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -43,7 +43,7 @@ export class FetchService extends BaseFetchService< eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, private runtimeService: RuntimeService, - storeModelProvider: IStoreModelProvider, + @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, ) { super( nodeConfig, diff --git a/packages/node/src/indexer/unfinalizedBlocks.service.ts b/packages/node/src/indexer/unfinalizedBlocks.service.ts index 5ca3b84289..6aff82c68a 100644 --- a/packages/node/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/unfinalizedBlocks.service.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { BaseUnfinalizedBlocksService, Header, @@ -20,7 +20,7 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService< constructor( private readonly apiService: ApiService, nodeConfig: NodeConfig, - storeModelProvider: IStoreModelProvider, + @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, ) { super(nodeConfig, storeModelProvider); } From c53eaf3b6c9b0549a5fc4b29ecf0bb19a3fe76d9 Mon Sep 17 00:00:00 2001 From: Tate Date: Mon, 4 Nov 2024 02:42:25 +0000 Subject: [PATCH 12/39] remove isCachePolicy and update enable-cache flag --- .../node-core/src/configure/NodeConfig.ts | 4 +-- .../SchemaMigration.service.ts | 6 ++-- .../blockDispatcher/base-block-dispatcher.ts | 31 ------------------- .../node-core/src/indexer/project.service.ts | 4 +-- .../node-core/src/indexer/store.service.ts | 4 +-- packages/node-core/src/indexer/store/store.ts | 2 +- .../storeModelProvider/storeCache.service.ts | 8 ----- .../storeModelProvider/storeModel.service.ts | 4 --- .../src/indexer/storeModelProvider/types.ts | 1 - packages/node-core/src/indexer/test.runner.ts | 6 ++-- .../src/subcommands/reindex.service.ts | 4 +-- packages/node-core/src/utils/reindex.ts | 6 ++-- packages/node-core/src/yargs.ts | 6 ++-- 13 files changed, 21 insertions(+), 65 deletions(-) diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts index 1485553daa..8ba6a39339 100644 --- a/packages/node-core/src/configure/NodeConfig.ts +++ b/packages/node-core/src/configure/NodeConfig.ts @@ -56,7 +56,7 @@ export interface IConfig { readonly csvOutDir?: string; readonly monitorOutDir: string; readonly monitorFileSize?: number; - readonly cacheDisable?: boolean; + readonly enableCache?: boolean; } export type MinConfig = Partial> & Pick; @@ -330,7 +330,7 @@ export class NodeConfig implements IConfig { } get cacheDisable(): boolean { - return this._config.cacheDisable || false; + return this._config.enableCache ?? true; } merge(config: Partial): this { diff --git a/packages/node-core/src/db/migration-service/SchemaMigration.service.ts b/packages/node-core/src/db/migration-service/SchemaMigration.service.ts index cafbba30cf..0bc2911d6d 100644 --- a/packages/node-core/src/db/migration-service/SchemaMigration.service.ts +++ b/packages/node-core/src/db/migration-service/SchemaMigration.service.ts @@ -6,7 +6,7 @@ import {getAllEntitiesRelations, GraphQLModelsType, GraphQLRelationsType} from ' import {Sequelize, Transaction} from '@subql/x-sequelize'; import {GraphQLSchema} from 'graphql'; import {NodeConfig} from '../../configure'; -import {isCachePolicy, StoreService} from '../../indexer'; +import {StoreCacheService, StoreService} from '../../indexer'; import {getLogger} from '../../logger'; import {sortModels} from '../sync-helper'; import {Migration} from './migration'; @@ -114,7 +114,7 @@ export class SchemaMigrationService { const sortedAddedModels = alignModelOrder(sortedSchemaModels, addedModels); const sortedModifiedModels = alignModelOrder(sortedSchemaModels, modifiedModels); - if (isCachePolicy(this.storeService.modelProvider)) { + if (this.storeService.modelProvider instanceof StoreCacheService) { await this.storeService.modelProvider.flushData(true); } const migrationAction = await Migration.create( @@ -188,7 +188,7 @@ export class SchemaMigrationService { this.storeService.modelProvider.updateModels(modelChanges); await this.storeService.updateModels(this.dbSchema, getAllEntitiesRelations(nextSchema)); - if (isCachePolicy(this.storeService.modelProvider)) { + if (this.storeService.modelProvider instanceof StoreCacheService) { await this.storeService.modelProvider.flushData(true); } } catch (e: any) { diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index c6437f133c..f3d98557da 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -221,37 +221,6 @@ export abstract class BaseBlockDispatcher implements IB !this.projectService.hasDataSourcesAfterHeight(height), this.storeService.transaction ); - - // if (this.storeModelService instanceof StorCeacheService) { - // if (this.nodeConfig.storeCacheAsync) { - // // Flush all completed block data and don't wait - // await this.storeModelService.flushAndWaitForCapacity(false)?.catch((e) => { - // exitWithError(new Error(`Flushing cache failed`, { cause: e }), logger); - // }); - // } else { - // // Flush all data from cache and wait - // await this.storeModelService.flushCache(false); - // } - - // if (!this.projectService.hasDataSourcesAfterHeight(height)) { - // const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; - // await this.storeModelService.flushCache(false); - // exitWithError(msg, logger, 0); - // } - // } else if (this.storeModelService instanceof PlainStoreModelService) { - // const tx = this.storeService.transaction; - // if (!tx) { - // exitWithError(new Error('Transaction not found'), logger, 1); - // } - // await tx.commit(); - - // if (!this.projectService.hasDataSourcesAfterHeight(height)) { - // const msg = `All data sources have been processed up to block number ${height}. Exiting gracefully...`; - // exitWithError(msg, logger, 0); - // } - // } else { - // exitWithError(new Error('Unknown store model service'), logger, 1); - // } } @OnEvent(AdminEvent.rewindTarget) diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index a632ae9edb..4705c05db2 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -19,7 +19,7 @@ import {MetadataKeys} from './entities'; import {PoiSyncService} from './poi'; import {PoiService} from './poi/poi.service'; import {StoreService} from './store.service'; -import {isCachePolicy} from './storeModelProvider'; +import {StoreCacheService} from './storeModelProvider'; import {ISubqueryProject, IProjectService, BypassBlocks} from './types'; import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; @@ -150,7 +150,7 @@ export abstract class BaseProjectService< } // Flush any pending operations to set up DB - if (isCachePolicy(this.storeService.modelProvider)) { + if (this.storeService.modelProvider instanceof StoreCacheService) { await this.storeService.modelProvider.flushData(true); } } else { diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 47f2168624..b9a9341146 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -38,7 +38,7 @@ import {exitWithError} from '../process'; import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils'; import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities'; import {Store} from './store'; -import {IMetadata, isCachePolicy, IStoreModelProvider} from './storeModelProvider'; +import {IMetadata, IStoreModelProvider, StoreCacheService} from './storeModelProvider'; import {StoreOperations} from './StoreOperations'; import {ISubqueryProject} from './types'; @@ -177,7 +177,7 @@ export class StoreService { } logger.info(`Historical state is ${this.historical ? 'enabled' : 'disabled'}`); - if (isCachePolicy(this.modelProvider)) { + if (this.modelProvider instanceof StoreCacheService) { this.modelProvider.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); } diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index bd518b7599..20030b2ce9 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -28,7 +28,7 @@ export class Store implements IStore { #modelProvider: IStoreModelProvider; #context: Context; - constructor(config: NodeConfig, @Inject('IStoreModelProvider') modelProvider: IStoreModelProvider, context: Context) { + constructor(config: NodeConfig, modelProvider: IStoreModelProvider, context: Context) { this.#config = config; this.#modelProvider = modelProvider; this.#context = context; diff --git a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts index fcfc5c8f56..ed23835878 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts @@ -23,10 +23,6 @@ import {Exporter, ICachedModelControl, IStoreModelProvider, FlushPolicy} from '. const logger = getLogger('StoreCacheService'); -export function isCachePolicy(modelProvider: IStoreModelProvider): modelProvider is StoreCacheService { - return modelProvider.flushPolicy === FlushPolicy.Cache; -} - @Injectable() export class StoreCacheService extends BaseCacheService implements IStoreModelProvider { private readonly storeCacheThreshold: number; @@ -50,10 +46,6 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr } } - get flushPolicy() { - return FlushPolicy.Cache; - } - init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { super.init(historical, useCockroachDb, meta, poi); diff --git a/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts index 908baa5253..9d8e787097 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts @@ -27,10 +27,6 @@ export class PlainStoreModelService extends BaseStoreModelService implements ISt super(); } - get flushPolicy() { - return FlushPolicy.RealTime; - } - get metadata(): MetadataModel { if (!this.cachedModels[METADATA_ENTITY_NAME]) { if (!this.metadataRepo) { diff --git a/packages/node-core/src/indexer/storeModelProvider/types.ts b/packages/node-core/src/indexer/storeModelProvider/types.ts index 8a6ac410b8..3b8d9b8636 100644 --- a/packages/node-core/src/indexer/storeModelProvider/types.ts +++ b/packages/node-core/src/indexer/storeModelProvider/types.ts @@ -16,7 +16,6 @@ export enum FlushPolicy { export interface IStoreModelProvider { poi: IPoi | null; metadata: IMetadata; - flushPolicy: FlushPolicy; getModel(entity: string): IModel; diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 1ac0492ccf..776f679777 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -11,7 +11,7 @@ import {NodeConfig} from '../configure/NodeConfig'; import {getLogger} from '../logger'; import {TestSandbox} from './sandbox'; import {StoreService} from './store.service'; -import {isCachePolicy} from './storeModelProvider'; +import {StoreCacheService} from './storeModelProvider'; import {IBlock, IIndexerManager} from './types'; const logger = getLogger('test-runner'); @@ -71,7 +71,7 @@ export class TestRunner { try { await indexBlock(block, test.handler, this.indexerManager, this.apiService); - if (isCachePolicy(this.storeService.modelProvider)) { + if (this.storeService.modelProvider instanceof StoreCacheService) { await this.storeService.modelProvider.flushData(true); } } catch (e: any) { @@ -139,7 +139,7 @@ export class TestRunner { } } - if (isCachePolicy(this.storeService.modelProvider)) { + if (this.storeService.modelProvider instanceof StoreCacheService) { await this.storeService.modelProvider.flushData(true); } logger.info( diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index a3e2d3e203..e3c256cafd 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -12,7 +12,7 @@ import { ISubqueryProject, PoiService, IMetadata, - isCachePolicy, + StoreCacheService, } from '../indexer'; import {DynamicDsService} from '../indexer/dynamic-ds.service'; import {getLogger} from '../logger'; @@ -127,7 +127,7 @@ export class ReindexService

Date: Tue, 19 Nov 2024 03:27:24 +0000 Subject: [PATCH 31/39] db store csv export --- .../storeModelProvider/baseCache.service.ts | 4 +--- .../baseStoreModel.service.ts | 10 +++++++++- .../indexer/storeModelProvider/model/model.ts | 18 +++++++++++++++++- .../storeModelProvider/storeCache.service.ts | 5 ----- .../storeModelProvider/storeModel.service.ts | 3 ++- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/packages/node-core/src/indexer/storeModelProvider/baseCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/baseCache.service.ts index 512d9d5398..29d7c69aad 100644 --- a/packages/node-core/src/indexer/storeModelProvider/baseCache.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/baseCache.service.ts @@ -22,7 +22,6 @@ export abstract class BaseCacheService abstract _resetCache(): Promise | void; abstract isFlushable(): boolean; abstract get flushableRecords(): number; - abstract flushExportStores(): Promise; protected constructor(loggerName: string) { super(); @@ -62,7 +61,6 @@ export abstract class BaseCacheService async beforeApplicationShutdown(): Promise { await timeout(this.flushData(true), 60, 'Before shutdown flush cache timeout'); this.logger.info(`Force flush cache successful!`); - await this.flushExportStores(); - this.logger.info(`Force flush exports successful!`); + await super.beforeApplicationShutdown(); } } diff --git a/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts index 60eeda1add..524662894d 100644 --- a/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts @@ -1,20 +1,23 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import {BeforeApplicationShutdown} from '@nestjs/common'; import {getLogger} from '@subql/node-core/logger'; import {ModelStatic} from '@subql/x-sequelize'; import {MetadataRepo, PoiRepo} from '../entities'; import {METADATA_ENTITY_NAME} from './metadata/utils'; import {BaseEntity, IModel} from './model'; import {POI_ENTITY_NAME} from './poi'; +import {Exporter} from './types'; const logger = getLogger('BaseStoreModelService'); -export abstract class BaseStoreModelService> { +export abstract class BaseStoreModelService> implements BeforeApplicationShutdown { protected historical = true; protected poiRepo?: PoiRepo; protected metadataRepo?: MetadataRepo; protected cachedModels: Record = {}; protected useCockroachDb?: boolean; + protected exports: Exporter[] = []; protected abstract createModel(entity: string): M; @@ -44,4 +47,9 @@ export abstract class BaseStoreModelService> { }); removedModels.forEach((r) => delete this.cachedModels[r]); } + + async beforeApplicationShutdown(): Promise { + await Promise.all(this.exports.map((f) => f.shutdown())); + logger.info(`Force flush exports successful!`); + } } diff --git a/packages/node-core/src/indexer/storeModelProvider/model/model.ts b/packages/node-core/src/indexer/storeModelProvider/model/model.ts index 3689e05f5c..5075157a3b 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/model.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/model.ts @@ -4,7 +4,9 @@ import {FieldsExpression, GetOptions} from '@subql/types-core'; import {Op, Model, ModelStatic, Transaction, CreationAttributes, Sequelize} from '@subql/x-sequelize'; import {Fn} from '@subql/x-sequelize/types/utils'; -import _, {cloneDeep} from 'lodash'; +import _ from 'lodash'; +import {CsvStoreService} from '../csvStore.service'; +import {Exporter} from '../types'; import {getFullOptions, operatorsMap} from './utils'; export type BaseEntity = {id: string; __block_range?: (number | null)[] | Fn}; @@ -23,6 +25,8 @@ export interface IModel { // All operations must be carried out within a transaction. export class PlainModel implements IModel { + private exporters: Exporter[] = []; + constructor( readonly model: ModelStatic>, private readonly historical = true @@ -45,6 +49,14 @@ export class PlainModel implements IModel transaction: tx, updateOnDuplicate: Object.keys(data[0]) as unknown as (keyof T)[], }); + + if (tx) { + this.exporters.forEach((store: Exporter) => { + tx.afterCommit(async () => { + await store.export(data); + }); + }); + } } async get(id: string, tx?: Transaction): Promise { @@ -166,4 +178,8 @@ export class PlainModel implements IModel } ); } + + addExporterStore(exporter: CsvStoreService): void { + this.exporters.push(exporter); + } } diff --git a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts index 09b6f6c1ce..1d59ef700b 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts @@ -29,7 +29,6 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr private readonly cacheUpperLimit: number; private _storeOperationIndex = 0; private _lastFlushedOperationIndex = 0; - private exports: Exporter[] = []; constructor( private sequelize: Sequelize, @@ -95,10 +94,6 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr this.exports.push(exporterStore); } - async flushExportStores(): Promise { - await Promise.all(this.exports.map((f) => f.shutdown())); - } - get metadata(): CacheMetadataModel { if (!this.cachedModels[METADATA_ENTITY_NAME]) { if (!this.metadataRepo) { diff --git a/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts index 8c3553c05d..0d20af80fe 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeModel.service.ts @@ -63,7 +63,8 @@ export class PlainStoreModelService extends BaseStoreModelService implements ISt } private addExporter(model: PlainModel, exporterStore: CsvStoreService): void { - throw new Error('Not implemented'); + model.addExporterStore(exporterStore); + this.exports.push(exporterStore); } async applyPendingChanges(height: number, dataSourcesCompleted: boolean, tx: Transaction): Promise { From a77b6ec448204028712aaab15395ffe7d9253ce3 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 19 Nov 2024 06:19:45 +0000 Subject: [PATCH 32/39] some improve --- .../node-core/src/indexer/store.service.spec.ts | 1 - packages/node-core/src/indexer/store.service.ts | 4 +++- .../storeModelProvider/model/cacheModel.ts | 17 ----------------- .../indexer/storeModelProvider/model/model.ts | 15 ++++----------- 4 files changed, 7 insertions(+), 30 deletions(-) diff --git a/packages/node-core/src/indexer/store.service.spec.ts b/packages/node-core/src/indexer/store.service.spec.ts index b9f6ad6679..c715ff069b 100644 --- a/packages/node-core/src/indexer/store.service.spec.ts +++ b/packages/node-core/src/indexer/store.service.spec.ts @@ -5,7 +5,6 @@ import {DataTypes, Model, ModelAttributes} from '@subql/x-sequelize'; import {addIdAndBlockRangeAttributes} from '../db'; import {StoreService} from './store.service'; -jest.setTimeout(60000); describe('Store Service', () => { let storeService: StoreService; diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 9abe5f28ad..9b99cac04c 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -375,7 +375,9 @@ export class StoreService { async setBlockHeight(blockHeight: number): Promise { this._blockHeight = blockHeight; - if (this.modelProvider instanceof PlainStoreModelService && !this.#transaction) { + if (this.modelProvider instanceof PlainStoreModelService) { + assert(!this.#transaction, new Error(`Transaction is reopening in setBlockHeight ${blockHeight}`)); + this.#transaction = await this.sequelize.transaction({ deferrable: this._historical || this.dbType === SUPPORT_DB.cockRoach ? undefined : Deferrable.SET_DEFERRED(), }); diff --git a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts index 2f7ef5771c..c9f11cf00d 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts @@ -169,23 +169,6 @@ export class CachedModel return combined; } - // async getByField( - // field: keyof T, - // value: T[keyof T] | T[keyof T][], - // options: GetOptions = defaultOptions - // ): Promise { - // return this.getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], options); - // } - - // async getOneByField(field: keyof T, value: T[keyof T]): Promise { - // const [res] = await this.getByField(field, value, { - // ...defaultOptions, - // limit: 1, - // }); - - // return res; - // } - // eslint-disable-next-line @typescript-eslint/require-await async set(id: string, data: T, blockHeight: number): Promise { if (data === undefined || data === null) { diff --git a/packages/node-core/src/indexer/storeModelProvider/model/model.ts b/packages/node-core/src/indexer/storeModelProvider/model/model.ts index 5075157a3b..fc68c0f395 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/model.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/model.ts @@ -56,6 +56,8 @@ export class PlainModel implements IModel await store.export(data); }); }); + } else { + Promise.all(this.exporters.map(async (store: Exporter) => store.export(data))); } } @@ -105,19 +107,10 @@ export class PlainModel implements IModel throw new Error(`Currently not supported: update by fields`); } - const uniqueMap = data.reduce( - (acc, curr) => { - acc[curr.id] = curr; - return acc; - }, - {} as Record - ); - const uniqueDatas = Object.values(uniqueMap); - // Batch insert every 10000 data const batchSize = 10000; - for (let i = 0; i < uniqueDatas.length; i += batchSize) { - const batchDatas = uniqueDatas.slice(i, i + batchSize); + for (let i = 0; i < data.length; i += batchSize) { + const batchDatas = data.slice(i, i + batchSize); if (!this.historical) { await this._bulkCreate(batchDatas, tx); From b84f8c627fb42dabd45d00a39377a0163be3809d Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 19 Nov 2024 06:53:22 +0000 Subject: [PATCH 33/39] Fix the issue where data is written incorrectly after being deleted and recreated in cache mode. --- .../src/indexer/storeModelProvider/model/cacheModel.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts index c9f11cf00d..3320b8aee6 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts @@ -180,10 +180,6 @@ export class CachedModel // Experimental, this means getCache keeps duplicate data from setCache, // we can remove this once memory is too full. this.getCache.set(id, copiedData); - // Handle remove cache, when removed data been created again - if (this.removeCache[id] && this.removeCache[id].removedAtBlock === blockHeight) { - delete this.removeCache[id]; - } this.flushableRecordCounter += 1; } From afaa536988f05b61d5aadaed0cecd01442225e0c Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 20 Nov 2024 01:40:44 +0000 Subject: [PATCH 34/39] remove batch size and beforeBulkDestroy hook --- .../node-core/src/indexer/store.service.ts | 11 ----- .../indexer/storeModelProvider/model/model.ts | 44 +++++++++---------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 9b99cac04c..149b3e5e38 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -292,17 +292,6 @@ export class StoreService { item.__block_range = [this.blockHeight, null]; }); }); - - sequelizeModel.addHook('beforeBulkDestroy', (instance) => { - instance.where = { - ...instance.where, - [Op.and]: this.sequelize.where( - this.sequelize.fn('lower', this.sequelize.col('_block_range')), - Op.eq, - this.blockHeight - ), - }; - }); } // TODO, remove id and block_range constraint, check id manually // see https://github.com/subquery/subql/issues/1542 diff --git a/packages/node-core/src/indexer/storeModelProvider/model/model.ts b/packages/node-core/src/indexer/storeModelProvider/model/model.ts index fc68c0f395..277d6acf8e 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/model.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/model.ts @@ -107,30 +107,28 @@ export class PlainModel implements IModel throw new Error(`Currently not supported: update by fields`); } - // Batch insert every 10000 data - const batchSize = 10000; - for (let i = 0; i < data.length; i += batchSize) { - const batchDatas = data.slice(i, i + batchSize); - - if (!this.historical) { - await this._bulkCreate(batchDatas, tx); - continue; - } - - await this.model.destroy({ - where: { - id: batchDatas.map((v) => v.id), - } as any, - limit: batchSize, - transaction: tx, - }); - await this.markAsDeleted( - batchDatas.map((v) => v.id), - blockHeight, - tx - ); - await this._bulkCreate(batchDatas, tx); + if (!this.historical) { + await this._bulkCreate(data, tx); + return; } + + await this.model.destroy({ + where: { + id: data.map((v) => v.id), + [Op.and]: this.sequelize.where( + this.sequelize.fn('lower', this.sequelize.col('_block_range')), + Op.eq, + blockHeight + ), + } as any, + transaction: tx, + }); + await this.markAsDeleted( + data.map((v) => v.id), + blockHeight, + tx + ); + await this._bulkCreate(data, tx); } async bulkRemove(ids: string[], blockHeight: number, tx?: Transaction): Promise { From 719c49ffcfbca6723514c19101f2dc33f0a496b4 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 20 Nov 2024 07:54:59 +0000 Subject: [PATCH 35/39] cacheModel delete test --- .../src/indexer/store.service.test.ts | 101 +++++++++++++++++- .../node-core/src/indexer/store.service.ts | 2 +- .../indexer/storeModelProvider/model/model.ts | 81 ++++++-------- 3 files changed, 132 insertions(+), 52 deletions(-) diff --git a/packages/node-core/src/indexer/store.service.test.ts b/packages/node-core/src/indexer/store.service.test.ts index b8c1547d74..1b42fac89b 100644 --- a/packages/node-core/src/indexer/store.service.test.ts +++ b/packages/node-core/src/indexer/store.service.test.ts @@ -1,13 +1,13 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import {EventEmitter2} from '@nestjs/event-emitter'; import {buildSchemaFromString} from '@subql/utils'; import {Sequelize, QueryTypes} from '@subql/x-sequelize'; import {NodeConfig} from '../configure'; import {DbOption} from '../db'; import {StoreService} from './store.service'; -import {PlainStoreModelService} from './storeModelProvider'; - +import {CachedModel, PlainStoreModelService, StoreCacheService} from './storeModelProvider'; const option: DbOption = { host: process.env.DB_HOST ?? '127.0.0.1', port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, @@ -163,3 +163,100 @@ describe('Check whether the db store and cache store are consistent.', () => { ]); }, 100000); }); + +describe('Cache Provider', () => { + let sequelize: Sequelize; + let storeService: StoreService; + let cacheModel: StoreCacheService; + let Account: CachedModel; + + beforeAll(async () => { + sequelize = new Sequelize( + `postgresql://${option.username}:${option.password}@${option.host}:${option.port}/${option.database}`, + option + ); + await sequelize.authenticate(); + + await sequelize.query(`CREATE SCHEMA ${testSchemaName};`); + const nodeConfig = new NodeConfig({ + subquery: 'test', + proofOfIndex: true, + enableCache: false, + storeCacheAsync: true, + storeCacheThreshold: 1, + storeCacheUpperLimit: 1, + storeFlushInterval: 0, + }); + const project = {network: {chainId: '1'}, schema} as any; + cacheModel = new StoreCacheService(sequelize, nodeConfig, new EventEmitter2(), null as any); + storeService = new StoreService(sequelize, nodeConfig, cacheModel, project); + await storeService.initCoreTables(testSchemaName); + await storeService.init(testSchemaName); + Account = cacheModel.getModel('Account') as CachedModel; + }); + afterAll(async () => { + await sequelize.query(`DROP SCHEMA ${testSchemaName} CASCADE;`); + await sequelize.close(); + }); + + async function cacheFlush(blockHeight: number, handle: (blockHeight: number) => Promise) { + const tx = await sequelize.transaction(); + tx.afterCommit(() => { + Account.clear(blockHeight); + }); + await storeService.setBlockHeight(blockHeight); + await handle(blockHeight); + await Account.runFlush(tx, blockHeight); + await tx.commit(); + } + + it('For data that already exists, if there is a delete-create-delete operation, the database should have two entries for the data.', async () => { + const getAllAccounts = () => + sequelize.query(`SELECT * FROM "${testSchemaName}"."accounts"`, { + type: QueryTypes.SELECT, + }); + + const accountEntity1 = {id: 'accountEntity-001', balance: 100}; + await cacheFlush(1, async (blockHeight) => { + await Account.set(accountEntity1.id, accountEntity1, blockHeight); + }); + + // database check + let allDatas = await getAllAccounts(); + expect(allDatas).toHaveLength(1); + + // next block 999 + const accountEntity2 = {id: 'accountEntity-002', balance: 9999}; + await cacheFlush(999, async (blockHeight) => { + await Account.remove(accountEntity1.id, blockHeight); + const oldAccunt = await Account.get(accountEntity1.id); + expect(oldAccunt).toBeUndefined(); + + await Account.set(accountEntity2.id, accountEntity2, blockHeight); + }); + + allDatas = await getAllAccounts(); + expect(allDatas).toHaveLength(2); + + // next block 99999 + await cacheFlush(99999, async (blockHeight) => { + // last block, accountEntity1 should be deleted. + const oldAccunt1 = await Account.get(accountEntity1.id); + expect(oldAccunt1).toBeUndefined(); + + let oldAccunt2 = await Account.get(accountEntity2.id); + expect(oldAccunt2.balance).toEqual(accountEntity2.balance); + + await Account.remove(accountEntity2.id, blockHeight); + oldAccunt2 = await Account.get(accountEntity2.id); + expect(oldAccunt2).toBeUndefined(); + + await Account.set(accountEntity2.id, {id: 'accountEntity-002', balance: 999999} as any, blockHeight); + oldAccunt2 = await Account.get(accountEntity2.id); + expect(oldAccunt2.balance).toEqual(999999); + }); + + allDatas = await getAllAccounts(); + expect(allDatas).toHaveLength(3); + }); +}); diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 149b3e5e38..a48119a815 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -365,7 +365,7 @@ export class StoreService { this._blockHeight = blockHeight; if (this.modelProvider instanceof PlainStoreModelService) { - assert(!this.#transaction, new Error(`Transaction is reopening in setBlockHeight ${blockHeight}`)); + assert(!this.#transaction, new Error(`Transaction already exists for height: ${blockHeight}`)); this.#transaction = await this.sequelize.transaction({ deferrable: this._historical || this.dbType === SUPPORT_DB.cockRoach ? undefined : Deferrable.SET_DEFERRED(), diff --git a/packages/node-core/src/indexer/storeModelProvider/model/model.ts b/packages/node-core/src/indexer/storeModelProvider/model/model.ts index 277d6acf8e..015cdc5e62 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/model.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/model.ts @@ -32,35 +32,6 @@ export class PlainModel implements IModel private readonly historical = true ) {} - // Only perform bulk create data, without paying attention to any exception situations. - private async _bulkCreate(data: T[], tx?: Transaction): Promise { - if (!data.length) return; - - const uniqueMap = data.reduce( - (acc, curr) => { - acc[curr.id] = curr; - return acc; - }, - {} as Record - ); - const waitCreateDatas = Object.values(uniqueMap); - - await this.model.bulkCreate(waitCreateDatas as CreationAttributes>[], { - transaction: tx, - updateOnDuplicate: Object.keys(data[0]) as unknown as (keyof T)[], - }); - - if (tx) { - this.exporters.forEach((store: Exporter) => { - tx.afterCommit(async () => { - await store.export(data); - }); - }); - } else { - Promise.all(this.exporters.map(async (store: Exporter) => store.export(data))); - } - } - async get(id: string, tx?: Transaction): Promise { const record = await this.model.findOne({ // https://github.com/sequelize/sequelize/issues/15179 @@ -107,37 +78,49 @@ export class PlainModel implements IModel throw new Error(`Currently not supported: update by fields`); } - if (!this.historical) { - await this._bulkCreate(data, tx); - return; + if (this.historical) { + // To prevent the scenario of repeated created-deleted-created, which may result in multiple entries. + await this.model.destroy({ + where: { + id: data.map((v) => v.id), + [Op.and]: this.sequelize.where( + this.sequelize.fn('lower', this.sequelize.col('_block_range')), + Op.eq, + blockHeight + ), + } as any, + transaction: tx, + }); + await this.markAsDeleted( + data.map((v) => v.id), + blockHeight, + tx + ); } - await this.model.destroy({ - where: { - id: data.map((v) => v.id), - [Op.and]: this.sequelize.where( - this.sequelize.fn('lower', this.sequelize.col('_block_range')), - Op.eq, - blockHeight - ), - } as any, + await this.model.bulkCreate(data as CreationAttributes>[], { transaction: tx, + updateOnDuplicate: Object.keys(data[0]) as unknown as (keyof T)[], }); - await this.markAsDeleted( - data.map((v) => v.id), - blockHeight, - tx - ); - await this._bulkCreate(data, tx); + + if (tx) { + this.exporters.forEach((store: Exporter) => { + tx.afterCommit(async () => { + await store.export(data); + }); + }); + } else { + Promise.all(this.exporters.map(async (store: Exporter) => store.export(data))); + } } async bulkRemove(ids: string[], blockHeight: number, tx?: Transaction): Promise { if (!ids.length) return; if (!this.historical) { await this.model.destroy({where: {id: ids} as any, transaction: tx}); - return; + } else { + await this.markAsDeleted(ids, blockHeight, tx); } - await this.markAsDeleted(ids, blockHeight, tx); } private get sequelize(): Sequelize { From 24340e5f17058bda4a380b0d81e59779f7e1283e Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 21 Nov 2024 02:33:23 +0000 Subject: [PATCH 36/39] change log --- packages/node-core/CHANGELOG.md | 4 ++++ packages/node/CHANGELOG.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index e1c9e57db7..31bf2512a2 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - lazy loading for monitor service (#2583) +- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations. + +### Fixed +- When using a GET query to retrieve an entity, it will include a “store” field. ## [14.1.7] - 2024-10-30 ### Changed diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 92a3f15a83..a29a3da388 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations. + + ## [5.2.9] - 2024-10-30 ### Changed - Bump `@subql/node-core` dependency From cd2375b71c9a83fd289e9b25517070772d2335c0 Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 21 Nov 2024 07:02:40 +0000 Subject: [PATCH 37/39] fix test --- .../src/indexer/poi/poi.service.spec.ts | 3 +- .../src/indexer/project.service.spec.ts | 28 ++++++++++++------- .../model/cacheModel.spec.ts | 8 +++--- .../storeModelProvider/model/cacheModel.ts | 5 ++++ .../storeModelProvider/storeCache.service.ts | 2 +- .../node-core/src/indexer/test.runner.spec.ts | 2 +- packages/node-core/src/indexer/test.runner.ts | 6 ++-- .../configure/SchemaMigration.service.test.ts | 2 +- 8 files changed, 35 insertions(+), 21 deletions(-) diff --git a/packages/node-core/src/indexer/poi/poi.service.spec.ts b/packages/node-core/src/indexer/poi/poi.service.spec.ts index 16cb651933..353d6e7b3d 100644 --- a/packages/node-core/src/indexer/poi/poi.service.spec.ts +++ b/packages/node-core/src/indexer/poi/poi.service.spec.ts @@ -9,6 +9,7 @@ import {Sequelize, Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; import {ProofOfIndex} from '../entities/Poi.entity'; import {StoreCacheService} from '../storeModelProvider'; +import {METADATA_ENTITY_NAME} from '../storeModelProvider/metadata/utils'; import {PoiService} from './poi.service'; jest.mock('@subql/x-sequelize', () => { @@ -154,7 +155,7 @@ describe('PoiService', () => { } as any; await service.rewind(targetBlockHeight, transaction); - expect(storeCache.metadata.bulkRemove).toHaveBeenCalledWith(['lastCreatedPoiHeight']); + expect(storeCache.metadata.bulkRemove).toHaveBeenCalledWith(['lastCreatedPoiHeight'], transaction); }); }); diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index 4870290b72..0585b3bd8b 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -7,6 +7,7 @@ import {NodeConfig, ProjectUpgradeService} from '../configure'; import {BaseDsProcessorService} from './ds-processor.service'; import {DynamicDsService} from './dynamic-ds.service'; import {BaseProjectService} from './project.service'; +import {StoreService} from './store.service'; import {Header, ISubqueryProject} from './types'; import { BaseUnfinalizedBlocksService, @@ -309,26 +310,33 @@ describe('BaseProjectService', () => { init: jest.fn(), initCoreTables: jest.fn(), historical: true, - storeCache: { + modelProvider: { metadata: { - findMany: jest.fn(() => ({})), - find: jest.fn((key: string) => { + findMany: jest.fn(async () => Promise.resolve({})), + find: jest.fn(async (key: string): Promise => { + let result: any; switch (key) { case METADATA_LAST_FINALIZED_PROCESSED_KEY: - return lastFinalizedHeight; + result = lastFinalizedHeight; + break; case METADATA_UNFINALIZED_BLOCKS_KEY: - return JSON.stringify(unfinalizedBlocks); + result = JSON.stringify(unfinalizedBlocks); + break; case 'lastProcessedHeight': - return startBlock - 1; + result = startBlock - 1; + break; case 'deployments': - return JSON.stringify({1: '1'}); + result = JSON.stringify({1: '1'}); + break; default: - return undefined; + result = undefined; + break; } + return Promise.resolve(result); }), set: jest.fn(), flush: jest.fn(), - }, + } as any, resetCache: jest.fn(), flushCache: jest.fn(), _flushCache: jest.fn(), @@ -359,7 +367,7 @@ describe('BaseProjectService', () => { resetDynamicDatasource: jest.fn(), } as unknown as DynamicDsService, // dynamicDsService new EventEmitter2(), // eventEmitter - new TestUnfinalizedBlocksService(nodeConfig, storeService.storeCache) // unfinalizedBlocksService + new TestUnfinalizedBlocksService(nodeConfig, storeService.modelProvider) // unfinalizedBlocksService ); }; diff --git a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts index ca2f9ce71c..5c1cb03b80 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts @@ -229,9 +229,9 @@ describe('cacheModel', () => { testModel = new CachedModel(sequelize.model('entity1'), true, {} as NodeConfig, () => i++); }); - it('throws when trying to set undefined', () => { - expect(() => testModel.set('0x01', undefined as any, 1)).toThrow(); - expect(() => testModel.set('0x01', null as any, 1)).toThrow(); + it('throws when trying to set undefined', async () => { + await expect(() => testModel.set('0x01', undefined as any, 1)).rejects.toThrow(); + await expect(() => testModel.set('0x01', null as any, 1)).rejects.toThrow(); }); // it should keep same behavior as hook we used @@ -287,7 +287,7 @@ describe('cacheModel', () => { }, 1 ); - const result = testModel.get('entity1_id_0x01'); + const result = await testModel.get('entity1_id_0x01'); // data should be erased from removeCache expect((testModel as any).removeCache.entity1_id_0x01).toBeUndefined(); diff --git a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts index 3320b8aee6..cc1699fb85 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts @@ -180,6 +180,11 @@ export class CachedModel // Experimental, this means getCache keeps duplicate data from setCache, // we can remove this once memory is too full. this.getCache.set(id, copiedData); + // Handle remove cache, when removed data been created again + if (this.removeCache[id] && this.removeCache[id].removedAtBlock === blockHeight) { + delete this.removeCache[id]; + } + this.flushableRecordCounter += 1; } diff --git a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts index 1d59ef700b..8b65628ba3 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts @@ -19,7 +19,7 @@ import {CacheMetadataModel} from './metadata'; import {METADATA_ENTITY_NAME} from './metadata/utils'; import {CachedModel} from './model'; import {CachePoiModel, POI_ENTITY_NAME} from './poi'; -import {Exporter, ICachedModelControl, IStoreModelProvider} from './types'; +import {ICachedModelControl, IStoreModelProvider} from './types'; const logger = getLogger('StoreCacheService'); diff --git a/packages/node-core/src/indexer/test.runner.spec.ts b/packages/node-core/src/indexer/test.runner.spec.ts index 1a977c03c6..1ee7b3aa7f 100644 --- a/packages/node-core/src/indexer/test.runner.spec.ts +++ b/packages/node-core/src/indexer/test.runner.spec.ts @@ -88,7 +88,7 @@ describe('TestRunner', () => { (testRunner as any).storeService = { getStore: () => storeMock, setBlockHeight: jest.fn(), - storeCache: mockStoreCache, + modelProvider: mockStoreCache, } as any; await testRunner.runTest(testMock, sandboxMock, indexBlock); diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 2f1157cac9..37df47af63 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -57,9 +57,9 @@ export class TestRunner { logger.debug('Fetching block'); const [block] = await this.apiService.fetchBlocks([test.blockHeight]); - this.storeService.setBlockHeight(test.blockHeight); + await this.storeService.setBlockHeight(test.blockHeight); // Ensure a block height is set so that data is flushed correctly - this.storeService.modelProvider.metadata.set('lastProcessedHeight', test.blockHeight - 1); + await this.storeService.modelProvider.metadata.set('lastProcessedHeight', test.blockHeight - 1); const store = this.storeService.getStore(); sandbox.freeze(store, 'store'); @@ -99,7 +99,7 @@ export class TestRunner { } else { Object.keys(actualEntity).forEach((attr) => { // EntityClass has private store on it, don't need to check it. - if (attr === 'store') return; + if (attr === '#store') return; const expectedAttr = (expectedEntity as Record)[attr] ?? null; const actualAttr = (actualEntity as Record)[attr] ?? null; diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index 757143fe4a..468c199142 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -99,7 +99,7 @@ describe('SchemaMigration integration tests', () => { projectService = app.get('IProjectService'); const projectUpgradeService = app.get('IProjectUpgradeService'); - const storeCache = app.get(StoreCacheService); + const storeCache = app.get('IStoreModelProvider'); const cacheSpy = jest.spyOn(storeCache, 'updateModels'); const apiService = app.get(ApiService); From 023c10a7c66c94f91829828e5d61b5758390ae0c Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 21 Nov 2024 07:59:13 +0000 Subject: [PATCH 38/39] fix unit test --- packages/node-core/src/indexer/fetch.service.ts | 2 +- packages/node-core/src/indexer/test.runner.spec.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index 9734343bd6..82eefc5127 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -208,7 +208,7 @@ export abstract class BaseFetchService { storeServiceMock = { setBlockHeight: jest.fn(), getStore: jest.fn().mockReturnValue({}), - storeCache: mockStoreCache, + modelProvider: mockStoreCache, }; sandboxMock = { @@ -123,7 +123,7 @@ describe('TestRunner', () => { (testRunner as any).storeService = { getStore: () => storeMock, setBlockHeight: jest.fn(), - storeCache: mockStoreCache, + modelProvider: mockStoreCache, } as any; await testRunner.runTest(testMock, sandboxMock, indexBlock); @@ -180,7 +180,7 @@ describe('TestRunner', () => { (testRunner as any).storeService = { getStore: () => storeMock, setBlockHeight: jest.fn(), - storeCache: mockStoreCache, + modelProvider: mockStoreCache, } as any; await testRunner.runTest(testMock, sandboxMock, indexBlock); From c42541af5f8cadf7bf563497b36e9a6c51399336 Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 21 Nov 2024 08:11:34 +0000 Subject: [PATCH 39/39] unit test fix --- packages/node/src/configure/SchemaMigration.service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index 468c199142..1a21b858d2 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -138,7 +138,7 @@ describe('SchemaMigration integration tests', () => { projectService = app.get('IProjectService'); const projectUpgradeService = app.get('IProjectUpgradeService'); - const storeCache = app.get(StoreCacheService); + const storeCache = app.get('IStoreModelProvider'); const apiService = app.get(ApiService); await apiService.init();