Fix datasource processor types not being able to distinguish input ty…

…pes and filter types (#2522)

* Fix datasource processor types not being able to distinguish input types and filter types

* Update changelog

packages/node-core/CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Fixed
 - revert yargs version as it was returning a promise
+- DS Processor types not being able to distinguish input and filter types (#2522)
 
 ## [14.1.0] - 2024-08-05
 ### Changed

packages/node-core/src/indexer/ds-processor.service.ts

@@ -5,7 +5,6 @@ import fs from 'fs';
 import path from 'path';
 import {Inject} from '@nestjs/common';
 import {
-  HandlerInputMap,
   BaseCustomDataSource,
   SecondLayerHandlerProcessor_0_0_0,
   SecondLayerHandlerProcessor_1_0_0,
@@ -22,48 +21,51 @@ import {ISubqueryProject} from './types';
 const logger = getLogger('ds-sandbox');
 
 function isSecondLayerHandlerProcessor_0_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  InputKinds extends string | symbol,
+  HandlerInput extends Record<InputKinds, any>,
+  BaseHandlerFilters extends Record<InputKinds, any>,
   F extends Record<string, unknown>,
   E,
   API,
   DS extends BaseCustomDataSource = BaseCustomDataSource
 >(
   processor:
-    | SecondLayerHandlerProcessor_0_0_0<IM, K, F, E, DS, API>
-    | SecondLayerHandlerProcessor_1_0_0<IM, K, F, E, DS, API>
-): processor is SecondLayerHandlerProcessor_0_0_0<IM, K, F, E, DS, API> {
+    | SecondLayerHandlerProcessor_0_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+    | SecondLayerHandlerProcessor_1_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+): processor is SecondLayerHandlerProcessor_0_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API> {
   // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0
   return processor.specVersion === undefined;
 }
 
 function isSecondLayerHandlerProcessor_1_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  InputKinds extends string | symbol,
+  HandlerInput extends Record<InputKinds, any>,
+  BaseHandlerFilters extends Record<InputKinds, any>,
   F extends Record<string, unknown>,
   E,
   API,
   DS extends BaseCustomDataSource = BaseCustomDataSource
 >(
   processor:
-    | SecondLayerHandlerProcessor_0_0_0<IM, K, F, E, DS, API>
-    | SecondLayerHandlerProcessor_1_0_0<IM, K, F, E, DS, API>
-): processor is SecondLayerHandlerProcessor_1_0_0<IM, K, F, E, DS, API> {
+    | SecondLayerHandlerProcessor_0_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+    | SecondLayerHandlerProcessor_1_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+): processor is SecondLayerHandlerProcessor_1_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API> {
   return processor.specVersion === '1.0.0';
 }
 
 export function asSecondLayerHandlerProcessor_1_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  InputKinds extends string | symbol,
+  HandlerInput extends Record<InputKinds, any>,
+  BaseHandlerFilters extends Record<InputKinds, any>,
   F extends Record<string, unknown>,
   E,
   API,
   DS extends BaseCustomDataSource = BaseCustomDataSource
 >(
   processor:
-    | SecondLayerHandlerProcessor_0_0_0<IM, K, F, E, DS, API>
-    | SecondLayerHandlerProcessor_1_0_0<IM, K, F, E, DS, API>
-): SecondLayerHandlerProcessor_1_0_0<IM, K, F, E, DS, API> {
+    | SecondLayerHandlerProcessor_0_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+    | SecondLayerHandlerProcessor_1_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+): SecondLayerHandlerProcessor_1_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API> {
   if (isSecondLayerHandlerProcessor_1_0_0(processor)) {
     return processor;
   }

packages/types-core/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- DS Processor types not being able to distinguish input and filter types (#2522)
 
 ## [1.1.0] - 2024-08-05
 ### Changed

packages/types-core/src/project/datasourceProcessors.ts

@@ -14,70 +14,73 @@ import {BaseCustomDataSource} from './versioned';
 export type HandlerInputMap = Record<string, any>;
 
 export interface HandlerInputTransformer_0_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  Input,
   DS extends BaseCustomDataSource,
   API,
   E // Output type
 > {
-  (input: IM[K], ds: DS, api: API, assets?: Record<string, string>): Promise<E>;
+  (input: Input, ds: DS, api: API, assets?: Record<string, string>): Promise<E>;
 }
 
 export interface HandlerInputTransformer_1_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  Input,
   DS extends BaseCustomDataSource,
   API,
   F extends Record<string, unknown>,
   E // Output type
 > {
-  (params: {input: IM[K]; ds: DS; api: API; filter?: F; assets?: Record<string, string>}): Promise<E[]>;
+  (params: {input: Input; ds: DS; api: API; filter?: F; assets?: Record<string, string>}): Promise<E[]>;
 }
 
 interface SecondLayerHandlerProcessorBase<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  BaseFilterMap extends HandlerInputMap,
+  K extends keyof BaseFilterMap,
   F extends Record<string, unknown>,
   DS extends BaseCustomDataSource
 > {
   baseHandlerKind: K;
-  baseFilter: IM[K] | IM[K][];
+  baseFilter: BaseFilterMap[K] | BaseFilterMap[K][];
   filterValidator: (filter?: F) => void;
   dictionaryQuery?: (filter: F, ds: DS) => DictionaryQueryEntry | undefined;
 }
 
 // only allow one custom handler for each baseHandler kind
 export interface SecondLayerHandlerProcessor_0_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  InputKinds extends string | symbol,
+  HandlerInput extends Record<InputKinds, any>,
+  BaseHandlerFilters extends Record<InputKinds, any>,
   F extends Record<string, unknown>,
   E,
   DS extends BaseCustomDataSource,
   API
-> extends SecondLayerHandlerProcessorBase<IM, K, F, DS> {
+> extends SecondLayerHandlerProcessorBase<BaseHandlerFilters, InputKinds, F, DS> {
   specVersion: undefined;
-  transformer: HandlerInputTransformer_0_0_0<IM, K, DS, API, E>;
-  filterProcessor: (filter: F | undefined, input: IM[K], ds: DS) => boolean;
+  transformer: HandlerInputTransformer_0_0_0<HandlerInput[InputKinds], DS, API, E>;
+  filterProcessor: (filter: F | undefined, input: HandlerInput[InputKinds], ds: DS) => boolean;
 }
 
 export interface SecondLayerHandlerProcessor_1_0_0<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  InputKinds extends string | symbol,
+  HandlerInput extends Record<InputKinds, any>,
+  BaseHandlerFilters extends Record<InputKinds, any>,
   F extends Record<string, unknown>,
   E,
   DS extends BaseCustomDataSource,
   API
-> extends SecondLayerHandlerProcessorBase<IM, K, F, DS> {
+> extends SecondLayerHandlerProcessorBase<BaseHandlerFilters, InputKinds, F, DS> {
   specVersion: '1.0.0';
-  transformer: HandlerInputTransformer_1_0_0<IM, K, DS, API, F, E>;
-  filterProcessor: (params: {filter: F | undefined; input: IM[K]; ds: DS}) => boolean;
+  transformer: HandlerInputTransformer_1_0_0<HandlerInput[InputKinds], DS, API, F, E>;
+  filterProcessor: (params: {filter: F | undefined; input: HandlerInput[InputKinds]; ds: DS}) => boolean;
 }
 
 export type SecondLayerHandlerProcessor<
-  IM extends HandlerInputMap,
-  K extends keyof IM,
+  InputKinds extends string | symbol,
+  HandlerInput extends Record<InputKinds, any>,
+  BaseHandlerFilters extends Record<InputKinds, any>,
   F extends Record<string, unknown>,
   E,
   DS extends BaseCustomDataSource,
   API
-> = SecondLayerHandlerProcessor_0_0_0<IM, K, F, E, DS, API> | SecondLayerHandlerProcessor_1_0_0<IM, K, F, E, DS, API>;
+> =
+  | SecondLayerHandlerProcessor_0_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>
+  | SecondLayerHandlerProcessor_1_0_0<InputKinds, HandlerInput, BaseHandlerFilters, F, E, DS, API>;

packages/types/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- DS Processor types not being able to distinguish input and filter types (#2522)
 
 ## [3.11.0] - 2024-08-05
 ### Changed

packages/types/src/project.ts

@@ -237,7 +237,7 @@ type ISubstrateDatasource<M extends SubstrateMapping> = BaseDataSource<Substrate
  * @template M - The mapping type for the datasource (default: SubstrateMapping<SubstrateRuntimeHandler>).
  */
 export interface SubstrateRuntimeDatasource<
-  M extends SubstrateMapping<SubstrateRuntimeHandler> = SubstrateMapping<SubstrateRuntimeHandler>,
+  M extends SubstrateMapping<SubstrateRuntimeHandler> = SubstrateMapping<SubstrateRuntimeHandler>
 > extends ISubstrateDatasource<M> {
   /**
    * The kind of the datasource, which is `substrate/Runtime`.
@@ -262,7 +262,7 @@ export type SubstrateDatasource = SubstrateRuntimeDatasource | SubstrateCustomDa
 export interface SubstrateCustomDatasource<
   K extends string = string,
   M extends SubstrateMapping = SubstrateMapping<SubstrateCustomHandler>,
-  O = any,
+  O = any
 > extends BaseCustomDataSource<SubstrateHandler, M> {
   /**
    * The kind of the custom datasource. This should follow the pattern `substrate/*`.
@@ -293,8 +293,8 @@ export type HandlerInputTransformer_0_0_0<
   IM extends RuntimeHandlerInputMap<IT>,
   T extends SubstrateHandlerKind,
   E,
-  DS extends SubstrateCustomDatasource = SubstrateCustomDatasource,
-> = BaseHandlerInputTransformer_0_0_0<IM, T, DS, ApiPromise, E>;
+  DS extends SubstrateCustomDatasource = SubstrateCustomDatasource
+> = BaseHandlerInputTransformer_0_0_0<IM[T], DS, ApiPromise, E>;
 
 /**
  * @deprecated use types core version. datasource processors need updating before this can be removed
@@ -305,14 +305,14 @@ export type HandlerInputTransformer_1_0_0<
   T extends SubstrateHandlerKind,
   F extends Record<string, unknown>,
   E,
-  DS extends SubstrateCustomDatasource = SubstrateCustomDatasource,
-> = BaseHandlerInputTransformer_1_0_0<IM, T, DS, ApiPromise, F, E>;
+  DS extends SubstrateCustomDatasource = SubstrateCustomDatasource
+> = BaseHandlerInputTransformer_1_0_0<IM[T], DS, ApiPromise, F, E>;
 
 export type SecondLayerHandlerProcessorArray<
   K extends string,
   F extends Record<string, unknown>,
   T,
-  DS extends SubstrateCustomDatasource<K> = SubstrateCustomDatasource<K>,
+  DS extends SubstrateCustomDatasource<K> = SubstrateCustomDatasource<K>
 > =
   | SecondLayerHandlerProcessor<SubstrateHandlerKind.Block, F, T, DS>
   | SecondLayerHandlerProcessor<SubstrateHandlerKind.Call, F, T, DS>
@@ -328,17 +328,17 @@ export type SubstrateDatasourceProcessor<
   P extends Record<string, SecondLayerHandlerProcessorArray<K, F, any, DS>> = Record<
     string,
     SecondLayerHandlerProcessorArray<K, F, any, DS>
-  >,
+  >
 > = DsProcessor<DS, P, ApiPromise>;
 
 export type SecondLayerHandlerProcessor<
   K extends SubstrateHandlerKind,
   F extends Record<string, unknown>,
   E,
-  DS extends SubstrateCustomDatasource = SubstrateCustomDatasource,
+  DS extends SubstrateCustomDatasource = SubstrateCustomDatasource
 > =
-  | SecondLayerHandlerProcessor_0_0_0<RuntimeFilterMap, K, F, E, DS, ApiPromise>
-  | SecondLayerHandlerProcessor_1_0_0<RuntimeFilterMap, K, F, E, DS, ApiPromise>;
+  | SecondLayerHandlerProcessor_0_0_0<K, RuntimeHandlerInputMap, RuntimeFilterMap, F, E, DS, ApiPromise>
+  | SecondLayerHandlerProcessor_1_0_0<K, RuntimeHandlerInputMap, RuntimeFilterMap, F, E, DS, ApiPromise>;
 
 /**
  * Represents a Substrate subquery network configuration, which is based on the CommonSubqueryNetworkConfig template.