Fixes and optimisations to dictionary queries

package.json

@@ -33,7 +33,7 @@
     "ts-loader": "^9.2.6",
     "ts-node": "^10.4.0",
     "tsconfig-paths": "^3.12.0",
-    "typescript": "^4.9.5"
+    "typescript": "^5.5.4"
   },
   "resolutions": {
     "node-fetch": "2.6.7"

packages/node/CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Dictionary grouping optimisation regression (#344)
+- Dictionary query not being used if address specified but no other filters (#344)
 
 ## [5.1.2] - 2024-08-27
 ### Changed

packages/node/src/indexer/dictionary/utils.spec.ts

@@ -7,7 +7,7 @@ import {
   SubqlRuntimeDatasource,
 } from '@subql/types-ethereum';
 import { EthereumProjectDsTemplate } from '../../configure/SubqueryProject';
-import { ethFilterDs } from './utils';
+import { groupedDataSources } from './utils';
 
 describe('Dictionary utils', () => {
   it('can filter eth ds with multiple dynamic ds/templates', () => {
@@ -80,12 +80,31 @@ describe('Dictionary utils', () => {
     // Runtime + ERC721 + ERC721 + ERC1155
     expect(dataSources.length).toBe(4);
 
-    const filteredDs = ethFilterDs(dataSources);
-    // Runtime + ERC721 (groupedOptions) + ERC1155
-    expect(filteredDs.length).toBe(3);
-    expect((filteredDs[1] as any).groupedOptions).toStrictEqual([
-      { address: 'address1' },
-      { address: 'address2' },
+    const grouped = groupedDataSources(dataSources);
+
+    expect(grouped).toEqual([
+      [
+        {
+          handler: 'handleDyanmicDs',
+          kind: 'ethereum/LogHandler',
+          filter: {
+            topics: [
+              'TransferSingle(address, address, address, uint256, uint256)',
+            ],
+          },
+        },
+        [undefined, 'address3'],
+      ],
+      [
+        {
+          handler: 'handleERC721',
+          kind: 'ethereum/LogHandler',
+          filter: {
+            topics: ['Transfer(address, address, uint256)'],
+          },
+        },
+        ['address1', 'address2'],
+      ],
     ]);
   });
 });

packages/node/src/indexer/dictionary/utils.ts

@@ -1,46 +1,44 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import { SubqlDatasource } from '@subql/types-ethereum';
-import { groupBy, partition } from 'lodash';
-import {
-  EthereumProjectDsTemplate,
-  EthereumProjectDs,
-} from '../../configure/SubqueryProject';
+import { SubqlRuntimeHandler } from '@subql/types-ethereum';
+import { groupBy } from 'lodash';
+import { EthereumProjectDs } from '../../configure/SubqueryProject';
 
-function isTemplateDs(
-  ds: EthereumProjectDs | EthereumProjectDsTemplate,
-): ds is EthereumProjectDsTemplate {
-  return !!(ds as EthereumProjectDsTemplate)?.name;
-}
-
-export function ethFilterDs(
-  dataSources: (EthereumProjectDs | EthereumProjectDsTemplate)[],
-): SubqlDatasource[] {
-  const [normalDataSources, templateDataSources] = partition(
-    dataSources,
-    (ds) => !isTemplateDs(ds),
-  );
-
-  // Group templ
-  const groupedDataSources = Object.values(
-    groupBy(
-      templateDataSources as EthereumProjectDsTemplate[],
-      (ds) => ds.name,
-    ),
-  ).map((grouped) => {
-    if (grouped.length === 1) {
-      return grouped[0];
-    }
+// Gets all the unique handlers and the contract addresses that go with them
+export function groupedDataSources(
+  dataSources: EthereumProjectDs[],
+): [SubqlRuntimeHandler, Array<string | undefined>][] {
+  const addressHandlers: [string | undefined, SubqlRuntimeHandler][] =
+    dataSources
+      .map((ds) =>
+        ds.mapping.handlers.map(
+          (h) =>
+            [ds.options?.address, h] as [
+              string | undefined,
+              SubqlRuntimeHandler,
+            ],
+        ),
+      )
+      .flat();
 
-    const options = grouped.map((ds) => ds.options);
-    const ref = grouped[0];
+  const grouped = groupBy(addressHandlers, ([, handler]) => {
+    return `${handler.kind}:${JSON.stringify(handler.filter)}`;
+  });
 
-    return {
-      ...ref,
-      groupedOptions: options,
-    };
+  const res = Object.values(grouped).map((grouped) => {
+    const addresses = [...new Set(grouped.map(([address]) => address))]; // Make unique, could be duplicate handler in the same DS
+    return [grouped[0][1], addresses] as [
+      SubqlRuntimeHandler,
+      Array<string | undefined>,
+    ];
   });
 
-  return [...normalDataSources, ...groupedDataSources];
+  return res;
+}
+
+export function validAddresses(
+  addresses?: (string | undefined | null)[],
+): string[] {
+  return (addresses ?? []).filter((a) => !!a) as string[];
 }

packages/node/src/indexer/dictionary/v1/ethDictionaryV1.spec.ts

@@ -336,6 +336,55 @@ describe('buildDictionaryV1QueryEntries', () => {
       ]);
     });
 
+    it('Creates a valid filter with a single event handler that has 0 filters but a contract address', () => {
+      const ds: SubqlRuntimeDatasource = {
+        kind: EthereumDatasourceKind.Runtime,
+        assets: new Map(),
+        options: {
+          abi: 'erc20',
+          address: '0x7ceb23fd6bc0add59e62ac25578270cff1b9f619',
+        },
+        startBlock: 1,
+        mapping: {
+          file: '',
+          handlers: [
+            {
+              handler: 'handleTransfer',
+              kind: EthereumHandlerKind.Event,
+            },
+            {
+              handler: 'handleTransfer',
+              kind: EthereumHandlerKind.Call,
+            },
+          ],
+        },
+      };
+
+      const result = buildDictionaryV1QueryEntries([ds]);
+      expect(result).toEqual([
+        {
+          entity: 'evmLogs',
+          conditions: [
+            {
+              field: 'address',
+              matcher: 'equalTo',
+              value: '0x7ceb23fd6bc0add59e62ac25578270cff1b9f619',
+            },
+          ],
+        },
+        {
+          entity: 'evmTransactions',
+          conditions: [
+            {
+              field: 'to',
+              matcher: 'equalTo',
+              value: '0x7ceb23fd6bc0add59e62ac25578270cff1b9f619',
+            },
+          ],
+        },
+      ]);
+    });
+
     it('builds a filter when theres a block handler with modulo filter', () => {
       const ds: SubqlRuntimeDatasource = {
         kind: EthereumDatasourceKind.Runtime,
@@ -433,7 +482,7 @@ describe('buildDictionaryV1QueryEntries', () => {
               kind: EthereumHandlerKind.Event,
               filter: {
                 topics: [
-                  'TransferSingle(address, address, address, uint256, uint256)',
+                  'TransferMultiple(address, address, address, uint256, uint256)',
                 ],
               },
             },
@@ -458,7 +507,7 @@ describe('buildDictionaryV1QueryEntries', () => {
               field: 'topics0',
               matcher: 'equalTo',
               value:
-                '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62',
+                '0xeb9b7dd0d144caae51d14067d0d112bc1839fbf62e856fca78f6b4d9bfb51962',
             },
           ],
           entity: 'evmLogs',
@@ -467,8 +516,8 @@ describe('buildDictionaryV1QueryEntries', () => {
           conditions: [
             {
               field: 'address',
-              matcher: 'equalTo',
-              value: 'address1',
+              matcher: 'in',
+              value: ['address1', 'address2'],
             },
             {
               field: 'topics0',
@@ -484,29 +533,37 @@ describe('buildDictionaryV1QueryEntries', () => {
             {
               field: 'address',
               matcher: 'equalTo',
-              value: 'address2',
+              value: 'address3',
             },
             {
               field: 'topics0',
               matcher: 'equalTo',
               value:
-                '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
+                '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62',
             },
           ],
           entity: 'evmLogs',
         },
+      ]);
+    });
+
+    it('Drops the address if multiple datasources have the same filters, but one without an address', () => {
+      const duplicateDataSources = [
+        { ...mockTempDs[0], options: { address: 'address1' } },
+        { ...mockTempDs[0], options: { address: 'address2' } },
+        { ...mockTempDs[0], options: { address: undefined } },
+      ] as SubqlRuntimeDatasource[];
+
+      const queryEntry = buildDictionaryV1QueryEntries(duplicateDataSources);
+
+      expect(queryEntry).toEqual([
         {
           conditions: [
-            {
-              field: 'address',
-              matcher: 'equalTo',
-              value: 'address3',
-            },
             {
               field: 'topics0',
               matcher: 'equalTo',
               value:
-                '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62',
+                '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
             },
           ],
           entity: 'evmLogs',