Historical by timestamp

packages/common-substrate/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Update common (#2584)
 
 ## [4.3.2] - 2024-10-23
 ### Changed

packages/common/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Fixed
 - Removed imports of `@polkadot/util` without it being a dependency (#2592)
+### Removed
+- Support for cockroach DB (#2584)
 
 ## [5.1.4] - 2024-10-23
 ### Fixed

packages/common/src/constants.ts

@@ -39,12 +39,6 @@ export const runnerMapping = {
   '@subql/node-concordium': NETWORK_FAMILY.concordium,
 };
 
-// DATABASE TYPE
-export enum SUPPORT_DB {
-  cockRoach = 'CockroachDB',
-  postgres = 'PostgreSQL',
-}
-
 // DATABASE ERROR REGEX
 export const CONNECTION_SSL_ERROR_REGEX = 'not support SSL';
 

packages/common/src/project/index.ts

@@ -4,6 +4,5 @@
 export * from './load';
 export * from './versioned';
 export * from './readers';
-export * from './database';
 export * from './utils';
 export * from './IpfsHttpClientLite';

packages/node-core/CHANGELOG.md

@@ -7,10 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - lazy loading for monitor service (#2583)
-- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations. 
+- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations.
 
 ### Fixed
 - When using a GET query to retrieve an entity, it will include a “store” field.
+- Support for historical indexing by timestamp as well as block height (#2584)
+
+### Removed
+- Support for cockroach DB (#2584)
 
 ### Fixed
 - When configuring multiple endpoints, poor network conditions may lead to block crawling delays. (#2572)

packages/node-core/src/configure/NodeConfig.ts

@@ -8,6 +8,7 @@ import {getFileContent, loadFromJsonOrYaml, normalizeNetworkEndpoints} from '@su
 import {IEndpointConfig} from '@subql/types-core';
 import {last} from 'lodash';
 import {LevelWithSilent} from 'pino';
+import {HistoricalMode} from '../indexer';
 import {getLogger} from '../logger';
 import {assign} from '../utils/object';
 
@@ -38,7 +39,7 @@ export interface IConfig {
   readonly profiler?: boolean;
   readonly unsafe?: boolean;
   readonly subscription: boolean;
-  readonly disableHistorical: boolean;
+  readonly historical: HistoricalMode;
   readonly multiChain: boolean;
   readonly reindex?: number;
   readonly unfinalizedBlocks?: boolean;
@@ -75,7 +76,7 @@ const DEFAULT_CONFIG = {
   dictionaryQuerySize: 10000,
   profiler: false,
   subscription: false,
-  disableHistorical: false,
+  historical: 'height',
   multiChain: false,
   unfinalizedBlocks: false,
   storeCacheThreshold: 1000,
@@ -257,8 +258,15 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     return this._config.subscription;
   }
 
-  get disableHistorical(): boolean {
-    return this._isTest ? true : this._config.disableHistorical;
+  get historical(): HistoricalMode {
+    if (this._isTest) return false;
+
+    const val = this._config.historical;
+    // Runtime check, option can come from cli, project or config file
+    if (val !== false && val !== 'height' && val !== 'timestamp') {
+      throw new Error(`Historical mode is invalid. Received: ${val}`);
+    }
+    return val;
   }
 
   get multiChain(): boolean {
@@ -326,7 +334,7 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     const defaultMonitorFileSize = 200;
     // If user passed though yarg, we will record monitor file by this size, no matter poi or not
     // if user didn't pass through yarg, we will record monitor file by this default size only when poi is enabled
-    return this._config.monitorFileSize ?? this._config.proofOfIndex ? defaultMonitorFileSize : 0;
+    return (this._config.monitorFileSize ?? this._config.proofOfIndex) ? defaultMonitorFileSize : 0;
   }
 
   get enableCache(): boolean {

packages/node-core/src/configure/configure.module.ts

@@ -38,8 +38,8 @@ export function validDbSchemaName(name: string): boolean {
   }
 }
 
-// TODO once yargs is in node core we can update
-type Args = Record<string, any>; // typeof yargsOptions.argv['argv']
+// Cant seem to use the inferred types, strings arent converted to unions
+type Args = Record<string, any>; //ReturnType<typeof yargsBuilder>['argv']
 
 function processEndpointConfig(raw?: string | string[]): IEndpointConfig[] {
   if (!raw) return [];
@@ -67,10 +67,13 @@ export function yargsToIConfig(yargs: Args, nameMapping: Record<string, string>
         value = [value];
       }
       if (Array.isArray(value)) {
-        value = value.reduce((acc, endpoint, index) => {
-          acc[endpoint] = endpointConfig[index] ?? {};
-          return acc;
-        }, {} as Record<string, IEndpointConfig>);
+        value = value.reduce(
+          (acc, endpoint, index) => {
+            acc[endpoint] = endpointConfig[index] ?? {};
+            return acc;
+          },
+          {} as Record<string, IEndpointConfig>
+        );
       }
     }
     if (key === 'primary-network-endpoint') {
@@ -79,6 +82,13 @@ export function yargsToIConfig(yargs: Args, nameMapping: Record<string, string>
     }
     if (['network-endpoint-config', 'primary-network-endpoint-config'].includes(key)) return acc;
 
+    if (key === 'disable-historical' && value) {
+      acc.historical = false;
+    }
+    if (key === 'historical' && value === 'false') {
+      value = false;
+    }
+
     acc[nameMapping[key] ?? camelCase(key)] = value;
     return acc;
   }, {} as any);

packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts

@@ -227,7 +227,7 @@ describe('SchemaMigration integration tests', () => {
       schemaName,
       initialSchema,
       sequelize,
-      new NodeConfig({disableHistorical: true} as any)
+      new NodeConfig({historical: false} as any)
     );
 
     await migrationService.run(initialSchema, loadGqlSchema('test_10_1000.graphql'));
@@ -323,7 +323,7 @@ WHERE
       schemaName,
       initialSchema,
       sequelize,
-      new NodeConfig({disableHistorical: true} as any)
+      new NodeConfig({historical: false} as any)
     );
 
     await migrationService.run(initialSchema, loadGqlSchema('test_13_2000.graphql'));
@@ -363,7 +363,7 @@ WHERE
       schemaName,
       initialSchema,
       sequelize,
-      new NodeConfig({disableHistorical: true} as any)
+      new NodeConfig({historical: false} as any)
     );
 
     await migrationService.run(initialSchema, loadGqlSchema('test_14_1000.graphql'));

packages/node-core/src/db/migration-service/SchemaMigration.service.ts

@@ -1,7 +1,6 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {SUPPORT_DB} from '@subql/common';
 import {getAllEntitiesRelations, GraphQLModelsType, GraphQLRelationsType} from '@subql/utils';
 import {Sequelize, Transaction} from '@subql/x-sequelize';
 import {GraphQLSchema} from 'graphql';
@@ -30,8 +29,7 @@ export class SchemaMigrationService {
     private sequelize: Sequelize,
     private storeService: StoreService,
     private dbSchema: string,
-    private config: NodeConfig,
-    private dbType: SUPPORT_DB = SUPPORT_DB.postgres
+    private config: NodeConfig
   ) {}
 
   static validateSchemaChanges(currentSchema: GraphQLSchema, nextSchema: GraphQLSchema): boolean {
@@ -116,13 +114,7 @@ export class SchemaMigrationService {
 
     await cacheProviderFlushData(this.storeService.modelProvider, true);
 
-    const migrationAction = await Migration.create(
-      this.sequelize,
-      this.storeService,
-      this.dbSchema,
-      this.config,
-      this.dbType
-    );
+    const migrationAction = await Migration.create(this.sequelize, this.storeService, this.dbSchema, this.config);
 
     if (this.config.debug) {
       logger.debug(`${schemaChangesLoggerMessage(schemaDifference)}`);