From 05ea8b701d7413c4bc5ee4070a0a80aed20d2378 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Mon, 29 Jul 2024 12:25:36 +1200 Subject: [PATCH 1/8] Reload chainTypes when project changed, also fixed UTC issue in dockerfile --- packages/node-core/CHANGELOG.md | 3 + packages/node-core/src/api.service.ts | 3 +- .../src/indexer/connectionPool.service.ts | 1 + packages/node/CHANGELOG.md | 4 + packages/node/Dockerfile | 4 +- packages/node/package.json | 6 +- packages/node/src/indexer/api.service.ts | 15 +- .../node/src/indexer/project.service.spec.ts | 139 ++++++++++++++++++ packages/node/src/indexer/project.service.ts | 3 +- packages/node/src/indexer/x-provider/http.ts | 8 +- 10 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 packages/node/src/indexer/project.service.spec.ts diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index 6272b50d05..1b8e93fcbf 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Fixed project upgrade missing reload network chainTypes when `onProjectChange` (#2505) + ## [13.0.0] - 2024-07-25 ### Changed - Breaking change: Require indexing environment timezone set to UTC, avoid inconsistent result from cache and database (#2495) diff --git a/packages/node-core/src/api.service.ts b/packages/node-core/src/api.service.ts index 7a1f960dc3..4f926f0f05 100644 --- a/packages/node-core/src/api.service.ts +++ b/packages/node-core/src/api.service.ts @@ -19,6 +19,7 @@ export interface IApi = any[]> { safeApi(height: number): SA; unsafeApi: A; readonly networkMeta: NetworkMetadataPayload; + reloadChainTypes?(chainTypes: unknown): Promise; } export interface IApiConnectionSpecific = any[]> extends IApi { @@ -31,7 +32,7 @@ export abstract class ApiService< A = any, SA = any, B extends Array = any[], - Connection extends IApiConnectionSpecific = IApiConnectionSpecific + Connection extends IApiConnectionSpecific = IApiConnectionSpecific, > implements IApi { constructor( diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index 5f5045e3a6..62d9c218bd 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -49,6 +49,7 @@ export class ConnectionPoolService { await Promise.all(Object.values(this.allApi).map((api) => api?.apiDisconnect())); + this.allApi = {}; } async addToConnections(api: T, endpoint: string): Promise { diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 396f2245e2..4e953beb76 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Fixed api didn't reload new deployment chainTypes when project upgraded (#2505) +- Fixed default Timezone to UTC in dockerfile and package.json (#2505) + ## [5.0.0] - 2024-07-25 ### Changed - Breaking change: Update with `@subql/node-core`, require indexing environment timezone set to UTC (#2495) diff --git a/packages/node/Dockerfile b/packages/node/Dockerfile index 6397f8b31c..70a630e727 100644 --- a/packages/node/Dockerfile +++ b/packages/node/Dockerfile @@ -1,6 +1,5 @@ # Build stage FROM node:18-alpine as builder -ENV TZ ='UTC' # Set working directory WORKDIR /app @@ -36,6 +35,9 @@ RUN mkdir -p .monitor && \ # Make the user not ROOT USER 1000 +# Fix timezone to UTC +ENV TZ ='UTC' + # Set Entry point and command ENTRYPOINT ["/sbin/tini", "--", "/bin/run"] diff --git a/packages/node/package.json b/packages/node/package.json index c08f68f7c0..142f66ee84 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -8,9 +8,9 @@ "prebuild": "rimraf dist", "build": "rm -rf dist && tsc -b", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", - "start": "nest start", - "start:dev": "nodemon", - "start:prod": "node dist/main", + "start": "TZ=utc nest start", + "start:dev": "TZ=utc nodemon", + "start:prod": "TZ=utc node dist/main", "changelog:release": "echo \"Updating changelog $npm_package_version\" && npx chan release $npm_package_version --git-url \"https://github.com/subquery/subql\" --release-prefix=\"node/\"" }, "homepage": "https://github.com/subquery/subql", diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 48fc3b4217..d5b556ba5e 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -160,11 +160,24 @@ export class ApiService } async init(): Promise { + return this.load(); + } + + async reloadChainTypes(chainTypes: unknown): Promise { + // close any existing connections + await this.connectionPoolService.onApplicationShutdown(); + logger.info(`Reload chainTypes`); + await this.load(chainTypes); + } + + private async load(newChainTypes?: unknown): Promise { overrideConsoleWarn(); let chainTypes: RegisteredTypes | undefined; let network: SubstrateNetworkConfig; try { - chainTypes = await updateChainTypesHasher(this.project.chainTypes); + chainTypes = await updateChainTypesHasher( + newChainTypes ?? this.project.chainTypes, + ); network = this.project.network; if (this.nodeConfig.primaryNetworkEndpoint) { diff --git a/packages/node/src/indexer/project.service.spec.ts b/packages/node/src/indexer/project.service.spec.ts new file mode 100644 index 0000000000..77e6cad41f --- /dev/null +++ b/packages/node/src/indexer/project.service.spec.ts @@ -0,0 +1,139 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { + ConnectionPoolService, + ConnectionPoolStateManager, + NodeConfig, + ProjectUpgradeService, +} from '@subql/node-core'; +import { SubstrateDatasourceKind, SubstrateHandlerKind } from '@subql/types'; +import { GraphQLSchema } from 'graphql'; +import { SubqueryProject } from '../configure/SubqueryProject'; +import { ApiService } from './api.service'; +import { ApiPromiseConnection } from './apiPromise.connection'; +import { DsProcessorService } from './ds-processor.service'; +import { DynamicDsService } from './dynamic-ds.service'; +import { ProjectService } from './project.service'; + +const nodeConfig = new NodeConfig({ + subquery: 'test', + subqueryName: 'test', + networkEndpoint: ['https://polkadot.api.onfinality.io/public'], + dictionaryTimeout: 10, +}); + +function testSubqueryProject(): SubqueryProject { + return { + id: 'test', + root: './', + network: { + endpoint: ['https://polkadot.api.onfinality.io/public'], + chainId: + '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3', + }, + dataSources: [ + { + kind: SubstrateDatasourceKind.Runtime, + startBlock: 1, + mapping: { + file: '', + handlers: [ + { handler: 'testSandbox', kind: SubstrateHandlerKind.Event }, + ], + }, + }, + ], + schema: new GraphQLSchema({}), + templates: [], + chainTypes: { + types: { + TestType: 'u32', + }, + }, + applyCronTimestamps: () => jest.fn(), + } as unknown as SubqueryProject; +} + +const demoProjects = [ + testSubqueryProject(), + { + parent: { + utilBlock: 5, + reference: '0', + }, + ...testSubqueryProject(), + }, +] as SubqueryProject[]; + +jest.setTimeout(10_000); + +describe('ProjectService', () => { + let service: ProjectService; + + const apiService = new ApiService( + demoProjects[0], + new ConnectionPoolService( + nodeConfig, + new ConnectionPoolStateManager(), + ), + new EventEmitter2(), + nodeConfig, + ); + + beforeEach(async () => { + // Api service init at bootstrap + await apiService.init(); + + const projectUpgradeService = await ProjectUpgradeService.create( + demoProjects[0], + (id) => Promise.resolve(demoProjects[parseInt(id, 10)]), + 1, + ); + + (projectUpgradeService as any).init = jest.fn(); + (projectUpgradeService as any).updateIndexedDeployments = jest.fn(); + + service = new ProjectService( + { + validateProjectCustomDatasources: jest.fn(), + } as unknown as DsProcessorService, + apiService, + null as unknown as any, + null as unknown as any, + { query: jest.fn() } as unknown as any, + demoProjects[0], + projectUpgradeService, + { + initCoreTables: jest.fn(), + storeCache: { metadata: {}, flushCache: jest.fn() }, + } as unknown as any, + { unsafe: false } as unknown as NodeConfig, + { + getDynamicDatasources: jest.fn().mockResolvedValue([]), + init: jest.fn(), + } as unknown as DynamicDsService, + null as unknown as any, + null as unknown as any, + ); + + // Mock db related returns + (service as any).ensureProject = jest.fn().mockResolvedValue('mock-schema'); + (service as any).ensureMetadata = jest.fn(); + (service as any).initDbSchema = jest.fn(); + (service as any).initUnfinalizedInternal = jest + .fn() + .mockResolvedValue(undefined); + }); + + it('reload chainTypes when project changed, on init', async () => { + const spyOnApiReloadChainTypes = jest.spyOn(apiService, 'reloadChainTypes'); + // mock last processed height + (service as any).getLastProcessedHeight = jest.fn().mockResolvedValue(4); + + await service.init(5); + + expect(spyOnApiReloadChainTypes).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 6ed7819e7d..a6196c534a 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -74,7 +74,8 @@ export class ProjectService extends BaseProjectService< return getTimestamp(block); } - protected onProjectChange(project: SubqueryProject): void | Promise { + protected async onProjectChange(project: SubqueryProject): Promise { + await this.apiService.reloadChainTypes(project.chainTypes); this.apiService.updateBlockFetching(); } } diff --git a/packages/node/src/indexer/x-provider/http.ts b/packages/node/src/indexer/x-provider/http.ts index e2ea2a1516..1986dedc6b 100644 --- a/packages/node/src/indexer/x-provider/http.ts +++ b/packages/node/src/indexer/x-provider/http.ts @@ -54,7 +54,7 @@ export class HttpProvider implements ProviderInterface { readonly #stats: ProviderStats; - readonly #ctx: ReturnType; + readonly ctx: ReturnType; /** * @param {string} endpoint The endpoint url starting with http:// @@ -69,7 +69,7 @@ export class HttpProvider implements ProviderInterface { ); } - this.#ctx = context({ + this.ctx = context({ http1: { keepAlive: true, maxSockets: 10, @@ -119,7 +119,7 @@ export class HttpProvider implements ProviderInterface { */ async disconnect(): Promise { // noop - await this.#ctx.disconnectAll(); + await this.ctx.disconnectAll(); } /** @@ -193,7 +193,7 @@ export class HttpProvider implements ProviderInterface { this.#stats.total.bytesSent += body.length; try { - const response = await this.#ctx.fetch(this.#endpoint, { + const response = await this.ctx.fetch(this.#endpoint, { body, headers: { Accept: 'application/json', From bd285102612b5a3ab198ac2f01bbfe83c6387174 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Mon, 29 Jul 2024 13:27:01 +1200 Subject: [PATCH 2/8] generalize load/reload --- packages/node-core/src/api.service.ts | 14 +++++++++++++- packages/node/src/indexer/api.service.ts | 14 +++----------- packages/node/src/indexer/project.service.ts | 1 + 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/packages/node-core/src/api.service.ts b/packages/node-core/src/api.service.ts index 4f926f0f05..8fd3fbb048 100644 --- a/packages/node-core/src/api.service.ts +++ b/packages/node-core/src/api.service.ts @@ -19,7 +19,6 @@ export interface IApi = any[]> { safeApi(height: number): SA; unsafeApi: A; readonly networkMeta: NetworkMetadataPayload; - reloadChainTypes?(chainTypes: unknown): Promise; } export interface IApiConnectionSpecific = any[]> extends IApi { @@ -83,6 +82,19 @@ export abstract class ApiService< return apiInstance.unsafeApi; } + // eslint-disable-next-line @typescript-eslint/require-await + protected async load(chainTypes?: unknown): Promise { + // Implement in this way rather than abstract class, so it not break other network + throw new Error('this should be implement with network api service'); + } + + async reloadChainTypes(chainTypes: unknown): Promise { + // close any existing connections + await this.connectionPoolService.onApplicationShutdown(); + logger.info(`Reload chainTypes`); + await this.load(chainTypes); + } + async createConnections( network: ProjectNetworkConfig & {chainId: string}, createConnection: (endpoint: string) => Promise, diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index d5b556ba5e..a2210dfa69 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -160,17 +160,11 @@ export class ApiService } async init(): Promise { - return this.load(); - } - - async reloadChainTypes(chainTypes: unknown): Promise { - // close any existing connections - await this.connectionPoolService.onApplicationShutdown(); - logger.info(`Reload chainTypes`); - await this.load(chainTypes); + await this.load(); + return this; } - private async load(newChainTypes?: unknown): Promise { + async load(newChainTypes?: unknown): Promise { overrideConsoleWarn(); let chainTypes: RegisteredTypes | undefined; let network: SubstrateNetworkConfig; @@ -219,8 +213,6 @@ export class ApiService }); }, ); - - return this; } updateBlockFetching(): void { diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index a6196c534a..b33c473e91 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -75,6 +75,7 @@ export class ProjectService extends BaseProjectService< } protected async onProjectChange(project: SubqueryProject): Promise { + // Only network with chainTypes require to reload await this.apiService.reloadChainTypes(project.chainTypes); this.apiService.updateBlockFetching(); } From 3ca37de24c0ae8061b90d0d244358a151bebd533 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Mon, 29 Jul 2024 18:22:18 +1200 Subject: [PATCH 3/8] update --- packages/node-core/CHANGELOG.md | 4 +++- packages/node-core/src/api.service.ts | 14 +----------- .../src/indexer/connectionPool.service.ts | 7 ++++++ packages/node/src/indexer/api.service.ts | 19 ++++++++-------- .../node/src/indexer/apiPromise.connection.ts | 22 ++++++++++++++++--- .../node/src/indexer/project.service.spec.ts | 4 ++-- packages/node/src/indexer/project.service.ts | 4 ++-- 7 files changed, 43 insertions(+), 31 deletions(-) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index de4af9c4ab..cd2e142019 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -6,9 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [13.0.1] - 2024-07-29 ### Fixed - Fixed project upgrade missing reload network chainTypes when `onProjectChange` (#2505) + +## [13.0.1] - 2024-07-29 +### Fixed - Fixed get and set data not been deepCloned and data is not mutable - Improved get bigInt type from json type diff --git a/packages/node-core/src/api.service.ts b/packages/node-core/src/api.service.ts index 8fd3fbb048..9243bdc6d7 100644 --- a/packages/node-core/src/api.service.ts +++ b/packages/node-core/src/api.service.ts @@ -25,6 +25,7 @@ export interface IApiConnectionSpecific handleError(error: Error): ApiConnectionError; apiConnect(): Promise; apiDisconnect(): Promise; + updateChainTypes?(chainTypes: unknown): Promise; } export abstract class ApiService< @@ -82,19 +83,6 @@ export abstract class ApiService< return apiInstance.unsafeApi; } - // eslint-disable-next-line @typescript-eslint/require-await - protected async load(chainTypes?: unknown): Promise { - // Implement in this way rather than abstract class, so it not break other network - throw new Error('this should be implement with network api service'); - } - - async reloadChainTypes(chainTypes: unknown): Promise { - // close any existing connections - await this.connectionPoolService.onApplicationShutdown(); - logger.info(`Reload chainTypes`); - await this.load(chainTypes); - } - async createConnections( network: ProjectNetworkConfig & {chainId: string}, createConnection: (endpoint: string) => Promise, diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index 62d9c218bd..e62d9c1a7f 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -251,4 +251,11 @@ export class ConnectionPoolService { + logger.info(`Network chain types updated!`); + for (const endpoint in this.allApi) { + await this.allApi[endpoint].updateChainTypes?.(newChainTypes); + } + } } diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index a2210dfa69..438e50f262 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -160,18 +160,11 @@ export class ApiService } async init(): Promise { - await this.load(); - return this; - } - - async load(newChainTypes?: unknown): Promise { overrideConsoleWarn(); let chainTypes: RegisteredTypes | undefined; let network: SubstrateNetworkConfig; try { - chainTypes = await updateChainTypesHasher( - newChainTypes ?? this.project.chainTypes, - ); + chainTypes = await updateChainTypesHasher(this.project.chainTypes); network = this.project.network; if (this.nodeConfig.primaryNetworkEndpoint) { @@ -213,10 +206,16 @@ export class ApiService }); }, ); + return this; + } + + async updateChainTypes(newChainTypes: unknown): Promise { + const chainTypes = await updateChainTypesHasher(newChainTypes); + await this.connectionPoolService.updateChainTypes(chainTypes); } - updateBlockFetching(): void { - const onlyEventHandlers = isOnlyEventHandlers(this.project); + updateBlockFetching(project?: SubqueryProject): void { + const onlyEventHandlers = isOnlyEventHandlers(project ?? this.project); const skipTransactions = this.nodeConfig.skipTransactions && onlyEventHandlers; diff --git a/packages/node/src/indexer/apiPromise.connection.ts b/packages/node/src/indexer/apiPromise.connection.ts index e07c8ef9d2..fb35f57a96 100644 --- a/packages/node/src/indexer/apiPromise.connection.ts +++ b/packages/node/src/indexer/apiPromise.connection.ts @@ -1,7 +1,6 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import assert from 'assert'; import { ApiPromise, WsProvider } from '@polkadot/api'; import { ProviderInterface } from '@polkadot/rpc-provider/types'; import { RegisteredTypes } from '@polkadot/types/types'; @@ -46,6 +45,7 @@ export class ApiPromiseConnection private constructor( public unsafeApi: ApiPromise, private fetchBlocksBatches: GetFetchFunc, + private endpoint: string, ) { this.networkMeta = { chain: unsafeApi.runtimeChain.toString(), @@ -59,6 +59,14 @@ export class ApiPromiseConnection fetchBlocksBatches: GetFetchFunc, args: { chainTypes?: RegisteredTypes }, ): Promise { + const api = await this.createApiPromise(endpoint, args); + return new ApiPromiseConnection(api, fetchBlocksBatches, endpoint); + } + + static async createApiPromise( + endpoint: string, + args: { chainTypes?: RegisteredTypes }, + ): Promise { let provider: ProviderInterface; let throwOnConnect = false; @@ -83,8 +91,8 @@ export class ApiPromiseConnection noInitWarn: true, ...args.chainTypes, }; - const api = await ApiPromise.create(apiOption); - return new ApiPromiseConnection(api, fetchBlocksBatches); + + return ApiPromise.create(apiOption); } safeApi(height: number): ApiAt { @@ -123,6 +131,14 @@ export class ApiPromiseConnection await this.unsafeApi.disconnect(); } + async updateRegisteredChainTypes(chainTypes: RegisteredTypes): Promise { + this.unsafeApi = await ApiPromiseConnection.createApiPromise( + this.endpoint, + { chainTypes }, + ); + await this.apiConnect(); + } + handleError = ApiPromiseConnection.handleError; static handleError(e: Error): ApiConnectionError { diff --git a/packages/node/src/indexer/project.service.spec.ts b/packages/node/src/indexer/project.service.spec.ts index 77e6cad41f..b65dd7f6c7 100644 --- a/packages/node/src/indexer/project.service.spec.ts +++ b/packages/node/src/indexer/project.service.spec.ts @@ -128,12 +128,12 @@ describe('ProjectService', () => { }); it('reload chainTypes when project changed, on init', async () => { - const spyOnApiReloadChainTypes = jest.spyOn(apiService, 'reloadChainTypes'); + const spyOnApiUpdateChainTypes = jest.spyOn(apiService, 'updateChainTypes'); // mock last processed height (service as any).getLastProcessedHeight = jest.fn().mockResolvedValue(4); await service.init(5); - expect(spyOnApiReloadChainTypes).toHaveBeenCalledTimes(1); + expect(spyOnApiUpdateChainTypes).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index b33c473e91..3b7c379d4f 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -76,7 +76,7 @@ export class ProjectService extends BaseProjectService< protected async onProjectChange(project: SubqueryProject): Promise { // Only network with chainTypes require to reload - await this.apiService.reloadChainTypes(project.chainTypes); - this.apiService.updateBlockFetching(); + await this.apiService.updateChainTypes(project.chainTypes); + this.apiService.updateBlockFetching(project); } } From 456ab1ac04900bd99362dc5ad23deadccd60585d Mon Sep 17 00:00:00 2001 From: JQQQ Date: Mon, 29 Jul 2024 18:30:24 +1200 Subject: [PATCH 4/8] update --- packages/node-core/src/indexer/connectionPool.service.ts | 1 - packages/node/src/indexer/x-provider/http.ts | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index e62d9c1a7f..efac42bdad 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -49,7 +49,6 @@ export class ConnectionPoolService { await Promise.all(Object.values(this.allApi).map((api) => api?.apiDisconnect())); - this.allApi = {}; } async addToConnections(api: T, endpoint: string): Promise { diff --git a/packages/node/src/indexer/x-provider/http.ts b/packages/node/src/indexer/x-provider/http.ts index 1986dedc6b..e2ea2a1516 100644 --- a/packages/node/src/indexer/x-provider/http.ts +++ b/packages/node/src/indexer/x-provider/http.ts @@ -54,7 +54,7 @@ export class HttpProvider implements ProviderInterface { readonly #stats: ProviderStats; - readonly ctx: ReturnType; + readonly #ctx: ReturnType; /** * @param {string} endpoint The endpoint url starting with http:// @@ -69,7 +69,7 @@ export class HttpProvider implements ProviderInterface { ); } - this.ctx = context({ + this.#ctx = context({ http1: { keepAlive: true, maxSockets: 10, @@ -119,7 +119,7 @@ export class HttpProvider implements ProviderInterface { */ async disconnect(): Promise { // noop - await this.ctx.disconnectAll(); + await this.#ctx.disconnectAll(); } /** @@ -193,7 +193,7 @@ export class HttpProvider implements ProviderInterface { this.#stats.total.bytesSent += body.length; try { - const response = await this.ctx.fetch(this.#endpoint, { + const response = await this.#ctx.fetch(this.#endpoint, { body, headers: { Accept: 'application/json', From a38a27dc4132645701bfdd53687d1965f929b2e5 Mon Sep 17 00:00:00 2001 From: Jay Ji Date: Tue, 30 Jul 2024 10:13:28 +1200 Subject: [PATCH 5/8] Update packages/node/CHANGELOG.md Co-authored-by: Scott Twiname --- packages/node/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 3327da6119..4e2d95d97e 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed -- Fixed api didn't reload new deployment chainTypes when project upgraded (#2505) +- Fixed api not reloading new deployment chainTypes when project upgrades (#2505) ## [5.0.1] - 2024-07-29 ### Fixed From e51c4cc0fb8b041edd472f8948e62f551d5c659f Mon Sep 17 00:00:00 2001 From: JQQQ Date: Tue, 30 Jul 2024 12:10:40 +1200 Subject: [PATCH 6/8] address comments --- .../src/indexer/connectionPool.service.ts | 2 +- packages/node/src/indexer/api.service.ts | 8 ++++---- .../node/src/indexer/apiPromise.connection.ts | 15 +++++++++------ packages/node/src/indexer/project.service.ts | 4 ++-- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index efac42bdad..ccf1501363 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -252,9 +252,9 @@ export class ConnectionPoolService { - logger.info(`Network chain types updated!`); for (const endpoint in this.allApi) { await this.allApi[endpoint].updateChainTypes?.(newChainTypes); } + logger.info(`Network chain types updated!`); } } diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 438e50f262..d6664a1e17 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -209,13 +209,13 @@ export class ApiService return this; } - async updateChainTypes(newChainTypes: unknown): Promise { - const chainTypes = await updateChainTypesHasher(newChainTypes); + async updateChainTypes(): Promise { + const chainTypes = await updateChainTypesHasher(this.project.chainTypes); await this.connectionPoolService.updateChainTypes(chainTypes); } - updateBlockFetching(project?: SubqueryProject): void { - const onlyEventHandlers = isOnlyEventHandlers(project ?? this.project); + updateBlockFetching(): void { + const onlyEventHandlers = isOnlyEventHandlers(this.project); const skipTransactions = this.nodeConfig.skipTransactions && onlyEventHandlers; diff --git a/packages/node/src/indexer/apiPromise.connection.ts b/packages/node/src/indexer/apiPromise.connection.ts index fb35f57a96..5466bdebfc 100644 --- a/packages/node/src/indexer/apiPromise.connection.ts +++ b/packages/node/src/indexer/apiPromise.connection.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import { ApiPromise, WsProvider } from '@polkadot/api'; +import { ApiOptions } from '@polkadot/api/types'; import { ProviderInterface } from '@polkadot/rpc-provider/types'; import { RegisteredTypes } from '@polkadot/types/types'; import { @@ -131,12 +132,14 @@ export class ApiPromiseConnection await this.unsafeApi.disconnect(); } - async updateRegisteredChainTypes(chainTypes: RegisteredTypes): Promise { - this.unsafeApi = await ApiPromiseConnection.createApiPromise( - this.endpoint, - { chainTypes }, - ); - await this.apiConnect(); + async updateChainTypes(chainTypes: RegisteredTypes): Promise { + // Typeof Decorate<'promise' | 'rxjs'>, but we need to access this private method + const currentApiOptions = (this.unsafeApi as any)._options as ApiOptions; + const apiOption = { + ...currentApiOptions, + ...chainTypes, + }; + this.unsafeApi = await ApiPromise.create(apiOption); } handleError = ApiPromiseConnection.handleError; diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 3b7c379d4f..f56fa26c6f 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -76,7 +76,7 @@ export class ProjectService extends BaseProjectService< protected async onProjectChange(project: SubqueryProject): Promise { // Only network with chainTypes require to reload - await this.apiService.updateChainTypes(project.chainTypes); - this.apiService.updateBlockFetching(project); + await this.apiService.updateChainTypes(); + this.apiService.updateBlockFetching(); } } From 9791576c75806b82488bf81288518cc6cde56e21 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Tue, 30 Jul 2024 12:18:51 +1200 Subject: [PATCH 7/8] remove unused --- packages/node/src/indexer/apiPromise.connection.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/node/src/indexer/apiPromise.connection.ts b/packages/node/src/indexer/apiPromise.connection.ts index 5466bdebfc..c126411df3 100644 --- a/packages/node/src/indexer/apiPromise.connection.ts +++ b/packages/node/src/indexer/apiPromise.connection.ts @@ -46,7 +46,6 @@ export class ApiPromiseConnection private constructor( public unsafeApi: ApiPromise, private fetchBlocksBatches: GetFetchFunc, - private endpoint: string, ) { this.networkMeta = { chain: unsafeApi.runtimeChain.toString(), @@ -61,7 +60,7 @@ export class ApiPromiseConnection args: { chainTypes?: RegisteredTypes }, ): Promise { const api = await this.createApiPromise(endpoint, args); - return new ApiPromiseConnection(api, fetchBlocksBatches, endpoint); + return new ApiPromiseConnection(api, fetchBlocksBatches); } static async createApiPromise( From 8121e48403bb73ecffbae77637dfe0b7fb2e938d Mon Sep 17 00:00:00 2001 From: JQQQ Date: Tue, 30 Jul 2024 12:25:37 +1200 Subject: [PATCH 8/8] remove unused --- packages/node/src/indexer/apiPromise.connection.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/node/src/indexer/apiPromise.connection.ts b/packages/node/src/indexer/apiPromise.connection.ts index c126411df3..0dd18f15e9 100644 --- a/packages/node/src/indexer/apiPromise.connection.ts +++ b/packages/node/src/indexer/apiPromise.connection.ts @@ -59,14 +59,6 @@ export class ApiPromiseConnection fetchBlocksBatches: GetFetchFunc, args: { chainTypes?: RegisteredTypes }, ): Promise { - const api = await this.createApiPromise(endpoint, args); - return new ApiPromiseConnection(api, fetchBlocksBatches); - } - - static async createApiPromise( - endpoint: string, - args: { chainTypes?: RegisteredTypes }, - ): Promise { let provider: ProviderInterface; let throwOnConnect = false; @@ -92,7 +84,8 @@ export class ApiPromiseConnection ...args.chainTypes, }; - return ApiPromise.create(apiOption); + const api = await ApiPromise.create(apiOption); + return new ApiPromiseConnection(api, fetchBlocksBatches); } safeApi(height: number): ApiAt {