Fix substrate timestamp undefined (#2514)

* Fix substrate block timestamp can be undefined * update * update * update
subquery · Aug 1, 2024 · 24d66c2 · 24d66c2
1 parent 4f93f34
commit 24d66c2
Show file tree

Hide file tree

Showing 10 changed files with 39 additions and 13 deletions.
diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md
@@ -7,10 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - A more useful error message when failing to require modules from the VM (#2512)
-
-### Added
 - Support for endpoint configs (#2511)
 
+### Fixed
+- Handle when block timestamp can be undefined (#2513)
+
 ## [13.0.2] - 2024-07-31
 ### Fixed
 - Fixed project upgrade missing reload network chainTypes when `onProjectChange` (#2505)

diff --git a/packages/node-core/src/configure/SubqueryProject.ts b/packages/node-core/src/configure/SubqueryProject.ts
@@ -124,7 +124,7 @@ export class BaseSubqueryProject<
     return this.#dataSources;
   }
 
-  async applyCronTimestamps(getTimestamp: (height: number) => Promise<Date>): Promise<void> {
+  async applyCronTimestamps(getTimestamp: (height: number) => Promise<Date | undefined>): Promise<void> {
     this.#dataSources = await insertBlockFiltersCronSchedules(
       this.dataSources,
       getTimestamp,

diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts
@@ -41,7 +41,7 @@ export abstract class BaseProjectService<
   private _blockOffset?: number;
 
   protected abstract packageVersion: string;
-  protected abstract getBlockTimestamp(height: number): Promise<Date>;
+  protected abstract getBlockTimestamp(height: number): Promise<Date | undefined>;
   protected abstract onProjectChange(project: ISubqueryProject<IProjectNetworkConfig, DS>): void | Promise<void>;
 
   constructor(

diff --git a/packages/node-core/src/indexer/types.ts b/packages/node-core/src/indexer/types.ts
@@ -16,10 +16,10 @@ export interface ISubqueryProject<
   N extends IProjectNetworkConfig = IProjectNetworkConfig,
   DS extends BaseDataSource = BaseDataSource,
   T extends BaseTemplateDataSource<DS> = BaseTemplateDataSource<DS>,
-  C = unknown
+  C = unknown,
 > extends Omit<CommonSubqueryProject<N, DS, T>, 'schema' | 'version' | 'name' | 'specVersion' | 'description'> {
   readonly schema: GraphQLSchema;
-  applyCronTimestamps: (getBlockTimestamp: (height: number) => Promise<Date>) => Promise<void>;
+  applyCronTimestamps: (getBlockTimestamp: (height: number) => Promise<Date | undefined>) => Promise<void>;
   readonly id: string;
   chainTypes?: C; // The chainTypes after loaded
   readonly root: string;

diff --git a/packages/node-core/src/utils/project.ts b/packages/node-core/src/utils/project.ts
@@ -232,7 +232,7 @@ export type IsRuntimeDs<DS> = (ds: DS) => ds is DS;
 // eslint-disable-next-line @typescript-eslint/require-await
 export async function insertBlockFiltersCronSchedules<DS extends BaseDataSource = BaseDataSource>(
   dataSources: DS[],
-  getBlockTimestamp: (height: number) => Promise<Date>,
+  getBlockTimestamp: (height: number) => Promise<Date | undefined>,
   isRuntimeDs: IsRuntimeDs<DS>,
   blockHandlerKind: string
 ): Promise<DS[]> {
@@ -247,8 +247,16 @@ export async function insertBlockFiltersCronSchedules<DS extends BaseDataSource
             if (handler.kind === blockHandlerKind) {
               if (handler.filter?.timestamp) {
                 if (!timestampReference) {
-                  timestampReference = await getBlockTimestamp(startBlock);
+                  const blockTimestamp = await getBlockTimestamp(startBlock);
+                  if (!blockTimestamp) {
+                    throw new Error(
+                      `Could not apply cronSchedule, failed to get block timestamp for block ${startBlock}`
+                    );
+                  } else {
+                    timestampReference = blockTimestamp;
+                  }
                 }
+
                 let cronArr: number[][];
                 try {
                   cronArr = stringToArray(handler.filter.timestamp);

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Support endpoint configs and specifying headers for network endpoints (#2511)
 
+### Fixed
+- Fixed timestamp can be undefined in some network blocks, it should return undefined (#2513)
+
 ## [5.0.2] - 2024-07-31
 ### Fixed
 - Fixed api not reloading new deployment chainTypes when project upgrades (#2505)

diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts
@@ -69,7 +69,7 @@ export class ProjectService extends BaseProjectService<
     return super.init(startHeight);
   }
 
-  protected async getBlockTimestamp(height: number): Promise<Date> {
+  protected async getBlockTimestamp(height: number): Promise<Date | undefined> {
     const block = await getBlockByHeight(this.apiService.api, height);
     return getTimestamp(block);
   }

diff --git a/packages/node/src/utils/substrate.test.ts b/packages/node/src/utils/substrate.test.ts
@@ -9,11 +9,12 @@ import {
   fetchBlocksBatches,
   filterExtrinsic,
   getBlockByHeight,
+  getTimestamp,
 } from './substrate';
 
 const ENDPOINT_POLKADOT = 'wss://rpc.polkadot.io';
 const ENDPOINT_KARURA = 'wss://karura-rpc-0.aca-api.network';
-
+const ENDPOINT_SHIDEN = 'wss://rpc.shiden.astar.network';
 jest.setTimeout(100000);
 
 describe('substrate utils', () => {
@@ -93,4 +94,12 @@ describe('substrate utils', () => {
     expect(await getBlockByHeight(api, 50710)).toBeTruthy();
     await api.disconnect();
   });
+
+  it('return undefined if no timestamp set extrinsic', async () => {
+    const provider = new WsProvider(ENDPOINT_SHIDEN);
+    const api = await ApiPromise.create({ provider });
+    const block1 = await getBlockByHeight(api, 1);
+    expect(getTimestamp(block1)).toBeUndefined();
+    await api.disconnect();
+  });
 });
diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts
@@ -59,7 +59,9 @@ export function wrapBlock(
   });
 }
 
-export function getTimestamp({ block: { extrinsics } }: SignedBlock): Date {
+export function getTimestamp({
+  block: { extrinsics },
+}: SignedBlock): Date | undefined {
   for (const e of extrinsics) {
     const {
       method: { method, section },
@@ -72,7 +74,9 @@ export function getTimestamp({ block: { extrinsics } }: SignedBlock): Date {
       return date;
     }
   }
-  throw new Error('timestamp not found');
+  // For network that doesn't use timestamp-set, return undefined
+  // See test `return undefined if no timestamp set extrinsic`
+  return undefined;
 }
 
 export function wrapExtrinsics(
@@ -143,6 +147,7 @@ export function filterBlock(
   if (!filter) return block;
   if (!filterBlockModulo(block, filter)) return;
   if (
+    block.timestamp &&
     !filterBlockTimestamp(
       block.timestamp.getTime(),
       filter as SubqlProjectBlockFilter,

diff --git a/packages/types/src/interfaces.ts b/packages/types/src/interfaces.ts
@@ -9,7 +9,7 @@ import {IEvent} from '@polkadot/types/types';
 export interface SubstrateBlock extends SignedBlock {
   // parent block's spec version, can be used to decide the correct metadata that should be used for this block.
   specVersion: number;
-  timestamp: Date;
+  timestamp: Date | undefined;
   events: EventRecord[];
 }