historical timestamp and db store

packages/common-ethereum/CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- Bump `@subql/common` dependency
+
 ## [4.5.4] - 2024-10-23
 ### Changed
 - Bump `@subql/common` dependency

packages/common-ethereum/package.json

@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^5.1.4",
+    "@subql/common": "^5.2.1",
     "@subql/types-ethereum": "workspace:*",
     "@typechain/ethers-v5": "^11.1.1",
     "@zilliqa-js/crypto": "^3.5.0",

packages/node/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Support for historical indexing by timestamp as well as block height
+- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations.
+
 ## [5.1.7] - 2024-10-24
 ### Fixed
 - When there is no log data for fullBlock, it is determined as lightBlock. (#350)

packages/node/package.json

@@ -24,9 +24,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^5.1.4",
+    "@subql/common": "^5.2.1",
     "@subql/common-ethereum": "workspace:*",
-    "@subql/node-core": "^14.1.6",
+    "@subql/node-core": "^15.0.3",
     "@subql/testing": "^2.2.1",
     "@subql/types-ethereum": "workspace:*",
     "cacheable-lookup": "6",

packages/node/src/ethereum/api.ethereum.test.ts

@@ -21,6 +21,7 @@ import {
 // Add api key to work
 const HTTP_ENDPOINT = 'https://ethereum.rpc.subquery.network/public';
 const BLOCK_CONFIRMATIONS = 20;
+const MOONBEAM_ENDPOINT = 'https://rpc.api.moonbeam.network';
 
 const ds: SubqlRuntimeDatasource = {
   mapping: {
@@ -130,8 +131,11 @@ describe('Api.ethereum', () => {
   });
 
   it('Null filter support', async () => {
-    const beamEndpoint = 'https://rpc.api.moonbeam.network';
-    ethApi = new EthereumApi(beamEndpoint, BLOCK_CONFIRMATIONS, eventEmitter);
+    ethApi = new EthereumApi(
+      MOONBEAM_ENDPOINT,
+      BLOCK_CONFIRMATIONS,
+      eventEmitter,
+    );
     await ethApi.init();
     blockData = await fetchBlock(2847447);
     const result = blockData.transactions.filter((tx) => {
@@ -152,8 +156,11 @@ describe('Api.ethereum', () => {
   });
 
   it('!null filter support for logs, expect to filter out', async () => {
-    const beamEndpoint = 'https://rpc.api.moonbeam.network';
-    ethApi = new EthereumApi(beamEndpoint, BLOCK_CONFIRMATIONS, eventEmitter);
+    ethApi = new EthereumApi(
+      MOONBEAM_ENDPOINT,
+      BLOCK_CONFIRMATIONS,
+      eventEmitter,
+    );
     await ethApi.init();
     const filter_1: EthereumLogFilter = {
       topics: [
@@ -216,8 +223,11 @@ describe('Api.ethereum', () => {
   });
 
   it('Null filter support, for undefined transaction.to', async () => {
-    const beamEndpoint = 'https://rpc.api.moonbeam.network';
-    ethApi = new EthereumApi(beamEndpoint, BLOCK_CONFIRMATIONS, eventEmitter);
+    ethApi = new EthereumApi(
+      MOONBEAM_ENDPOINT,
+      BLOCK_CONFIRMATIONS,
+      eventEmitter,
+    );
     await ethApi.init();
     blockData = await fetchBlock(2847447);
     blockData.transactions[1].to = undefined;
@@ -239,8 +249,11 @@ describe('Api.ethereum', () => {
   });
 
   it('Should return all tx if filter.to is not defined', async () => {
-    const beamEndpoint = 'https://rpc.api.moonbeam.network';
-    ethApi = new EthereumApi(beamEndpoint, BLOCK_CONFIRMATIONS, eventEmitter);
+    ethApi = new EthereumApi(
+      MOONBEAM_ENDPOINT,
+      BLOCK_CONFIRMATIONS,
+      eventEmitter,
+    );
     await ethApi.init();
     blockData = await fetchBlock(2847447);
     const result = blockData.transactions.filter((tx) => {
@@ -258,8 +271,11 @@ describe('Api.ethereum', () => {
   });
 
   it('filter.to Should support only null not undefined', async () => {
-    const beamEndpoint = 'https://rpc.api.moonbeam.network';
-    ethApi = new EthereumApi(beamEndpoint, BLOCK_CONFIRMATIONS, eventEmitter);
+    ethApi = new EthereumApi(
+      MOONBEAM_ENDPOINT,
+      BLOCK_CONFIRMATIONS,
+      eventEmitter,
+    );
     await ethApi.init();
     blockData = await fetchBlock(2847447);
     const result = blockData.transactions.filter((tx) => {
@@ -277,8 +293,11 @@ describe('Api.ethereum', () => {
   });
 
   it('If transaction is undefined, with null filter, should be supported', async () => {
-    const beamEndpoint = 'https://rpc.api.moonbeam.network';
-    ethApi = new EthereumApi(beamEndpoint, BLOCK_CONFIRMATIONS, eventEmitter);
+    ethApi = new EthereumApi(
+      MOONBEAM_ENDPOINT,
+      BLOCK_CONFIRMATIONS,
+      eventEmitter,
+    );
     await ethApi.init();
     blockData = await fetchBlock(2847447);
     const result = blockData.transactions.filter((tx) => {
@@ -302,7 +321,7 @@ describe('Api.ethereum', () => {
 
     // Moonbeam
     ethApi = new EthereumApi(
-      'https://rpc.api.moonbeam.network',
+      MOONBEAM_ENDPOINT,
       BLOCK_CONFIRMATIONS,
       eventEmitter,
     );
@@ -312,7 +331,7 @@ describe('Api.ethereum', () => {
 
     // BSC
     ethApi = new EthereumApi(
-      'https://bsc-dataseed.binance.org',
+      'https://binance.llamarpc.com',
       BLOCK_CONFIRMATIONS,
       eventEmitter,
     );

packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts

@@ -5,7 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
   StoreService,
   IProjectService,
   BlockDispatcher,
@@ -40,7 +40,7 @@ export class BlockDispatcherService
     @Inject('IProjectUpgradeService')
     projectUpgradeService: IProjectUpgradeService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
   ) {
@@ -50,7 +50,7 @@ export class BlockDispatcherService
       projectService,
       projectUpgradeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       apiService.fetchBlocks.bind(apiService),

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -7,7 +7,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
   StoreService,
-  StoreCacheService,
+  IStoreModelProvider,
   IProjectService,
   WorkerBlockDispatcher,
   ConnectionPoolStateManager,
@@ -16,6 +16,7 @@ import {
   InMemoryCacheService,
   createIndexerWorker,
   MonitorServiceInterface,
+  Header,
 } from '@subql/node-core';
 import { EthereumBlock } from '@subql/types-ethereum';
 import {
@@ -46,7 +47,7 @@ export class WorkerBlockDispatcherService
     projectUpgadeService: IProjectUpgradeService,
     cacheService: InMemoryCacheService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
@@ -60,7 +61,7 @@ export class WorkerBlockDispatcherService
       projectService,
       projectUpgadeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       () =>
@@ -88,7 +89,7 @@ export class WorkerBlockDispatcherService
   protected async fetchBlock(
     worker: IndexerWorker,
     height: number,
-  ): Promise<void> {
-    await worker.fetchBlock(height, 0 /* Unused with ethereum*/);
+  ): Promise<Header> {
+    return worker.fetchBlock(height, 0 /* Unused with ethereum*/);
   }
 }

packages/node/src/indexer/fetch.module.ts

@@ -9,12 +9,12 @@ import {
   NodeConfig,
   ConnectionPoolService,
   ConnectionPoolStateManager,
-  StoreCacheService,
   IProjectUpgradeService,
   PoiSyncService,
   InMemoryCacheService,
   MonitorService,
   CoreModule,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { EthereumApiConnection } from '../ethereum/api.connection';
@@ -70,7 +70,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         indexerManager: IndexerManager,
         cacheService: InMemoryCacheService,
         storeService: StoreService,
-        storeCacheService: StoreCacheService,
+        storeModelProvider: IStoreModelProvider,
         poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
@@ -86,7 +86,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectUpgradeService,
               cacheService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
               dynamicDsService,
@@ -102,7 +102,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectService,
               projectUpgradeService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
             ),
@@ -115,7 +115,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         IndexerManager,
         InMemoryCacheService,
         StoreService,
-        StoreCacheService,
+        'IStoreModelProvider',
         PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,

packages/node/src/indexer/fetch.service.ts

@@ -11,9 +11,8 @@ import {
   BaseFetchService,
   ApiService,
   getModulos,
-  getLogger,
   Header,
-  StoreCacheService,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { EthereumBlock, SubqlDatasource } from '@subql/types-ethereum';
 import { EthereumApi } from '../ethereum';
@@ -46,7 +45,7 @@ export class FetchService extends BaseFetchService<
     unfinalizedBlocksService: UnfinalizedBlocksService,
     eventEmitter: EventEmitter2,
     schedulerRegistry: SchedulerRegistry,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
     super(
       nodeConfig,
@@ -56,7 +55,7 @@ export class FetchService extends BaseFetchService<
       eventEmitter,
       schedulerRegistry,
       unfinalizedBlocksService,
-      storeCacheService,
+      storeModelProvider,
     );
   }
 

packages/node/src/indexer/project.service.ts

@@ -13,6 +13,7 @@ import {
   IProjectUpgradeService,
   ApiService,
   profiler,
+  Header,
 } from '@subql/node-core';
 import { Sequelize } from '@subql/x-sequelize';
 import {
@@ -83,7 +84,7 @@ export class ProjectService extends BaseProjectService<
     this.apiService.updateBlockFetching();
   }
 
-  protected async initUnfinalized(): Promise<number | undefined> {
+  protected async initUnfinalized(): Promise<Header | undefined> {
     return this.unfinalizedBlockService.init(
       this.reindex.bind(this),
       this.apiService.api.supportsFinalization,

packages/node/src/indexer/unfinalizedBlocks.service.spec.ts

@@ -112,7 +112,7 @@ describe('UnfinalizedBlockService', () => {
       makeHeader(111, true),
     );
 
-    expect(rewind).toEqual(103);
+    expect(rewind?.blockHeight).toEqual(103);
   });
 
   it('uses POI blocks if there are not enough cached unfinalized blocks', async () => {
@@ -138,7 +138,7 @@ describe('UnfinalizedBlockService', () => {
       makeHeader(111, true),
     );
 
-    expect(rewind).toEqual(99);
+    expect(rewind?.blockHeight).toEqual(99);
     expect(spy).toHaveBeenCalled();
   });
 
@@ -160,7 +160,11 @@ describe('UnfinalizedBlockService', () => {
     await unfinalizedBlocks.init(rewind);
 
     // It should fall back to poi in this case
-    expect(rewind).toHaveBeenCalledWith(99);
+    expect(rewind).toHaveBeenCalledWith({
+      blockHash: '0x00ABC99f',
+      blockHeight: 99,
+      parentHash: '0x00ABC98f',
+    });
   });
 
   it('startup, correctly checks for forks within cached unfinalized blocks', async () => {
@@ -180,7 +184,11 @@ describe('UnfinalizedBlockService', () => {
     await unfinalizedBlocks.init(rewind);
 
     // It should fall back to poi in this case
-    expect(rewind).toHaveBeenCalledWith(99);
+    expect(rewind).toHaveBeenCalledWith({
+      blockHash: '0x00ABC99f',
+      blockHeight: 99,
+      parentHash: '0x00ABC98f',
+    });
   });
 
   it('doesnt throw if there are no unfinalized blocks on startup', async () => {
@@ -209,6 +217,6 @@ describe('UnfinalizedBlockService', () => {
     await unfinalizedBlocks.init(rewind);
 
     // It should fall back to poi in this case
-    expect(rewind).toHaveBeenCalledWith(0);
+    expect(rewind).toHaveBeenCalledWith({ blockHeight: 0 });
   });
 });