Merge remote-tracking branch 'origin/main' into fix-poi-write-perform…

…ance

.yarn/versions/7a978424.yml

@@ -0,0 +1,5 @@
+undecided:
+  - "@subql/node"
+  - "@subql/node-core"
+  - "@subql/types"
+  - "@subql/utils"

.yarn/versions/9aef76dc.yml

@@ -0,0 +1,2 @@
+undecided:
+  - "@subql/node-core"

packages/cli/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Fix `codegen` logic for Cosmos Manifests (#2073)
 
 ## [4.0.0] - 2023-10-04
 ### Fixed

packages/cli/README.md

@@ -23,7 +23,7 @@ $ npm install -g @subql/cli
 $ subql COMMAND
 running command...
 $ subql (--version)
-@subql/cli/3.6.2-13 linux-x64 node-v18.18.0
+@subql/cli/4.0.1-2 linux-x64 node-v18.18.0
 $ subql --help [COMMAND]
 USAGE
   $ subql COMMAND
@@ -71,7 +71,7 @@ DESCRIPTION
   Build this SubQuery project code
 ```
 
-_See code: [lib/commands/build/index.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/build/index.js)_
+_See code: [lib/commands/build/index.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/build/index.js)_
 
 ## `subql codegen`
 
@@ -89,7 +89,7 @@ DESCRIPTION
   Generate schemas for graph node
 ```
 
-_See code: [lib/commands/codegen/index.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/codegen/index.js)_
+_See code: [lib/commands/codegen/index.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/codegen/index.js)_
 
 ## `subql codegen:generate`
 
@@ -112,7 +112,7 @@ DESCRIPTION
   Generate Project.yaml and mapping functions based on provided ABI
 ```
 
-_See code: [lib/commands/codegen/generate.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/codegen/generate.js)_
+_See code: [lib/commands/codegen/generate.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/codegen/generate.js)_
 
 ## `subql deployment`
 
@@ -155,7 +155,7 @@ DESCRIPTION
   Deploy to hosted service
 ```
 
-_See code: [lib/commands/deployment/index.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/deployment/index.js)_
+_See code: [lib/commands/deployment/index.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/deployment/index.js)_
 
 ## `subql deployment:delete`
 
@@ -174,7 +174,7 @@ DESCRIPTION
   Delete Deployment
 ```
 
-_See code: [lib/commands/deployment/delete.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/deployment/delete.js)_
+_See code: [lib/commands/deployment/delete.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/deployment/delete.js)_
 
 ## `subql deployment:deploy`
 
@@ -214,7 +214,7 @@ DESCRIPTION
   Deployment to hosted service
 ```
 
-_See code: [lib/commands/deployment/deploy.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/deployment/deploy.js)_
+_See code: [lib/commands/deployment/deploy.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/deployment/deploy.js)_
 
 ## `subql deployment:promote`
 
@@ -233,7 +233,7 @@ DESCRIPTION
   Promote Deployment
 ```
 
-_See code: [lib/commands/deployment/promote.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/deployment/promote.js)_
+_See code: [lib/commands/deployment/promote.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/deployment/promote.js)_
 
 ## `subql help [COMMAND]`
 
@@ -277,7 +277,7 @@ DESCRIPTION
   Initialize a scaffold subquery project
 ```
 
-_See code: [lib/commands/init.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/init.js)_
+_See code: [lib/commands/init.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/init.js)_
 
 ## `subql multi-chain:add`
 
@@ -296,7 +296,7 @@ DESCRIPTION
   Add new chain manifest to multi-chain configuration
 ```
 
-_See code: [lib/commands/multi-chain/add.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/multi-chain/add.js)_
+_See code: [lib/commands/multi-chain/add.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/multi-chain/add.js)_
 
 ## `subql project`
 
@@ -322,7 +322,7 @@ DESCRIPTION
   Create/Delete project
 ```
 
-_See code: [lib/commands/project/index.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/project/index.js)_
+_See code: [lib/commands/project/index.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/project/index.js)_
 
 ## `subql project:create-project`
 
@@ -347,7 +347,7 @@ DESCRIPTION
   Create Project on Hosted Service
 ```
 
-_See code: [lib/commands/project/create-project.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/project/create-project.js)_
+_See code: [lib/commands/project/create-project.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/project/create-project.js)_
 
 ## `subql project:delete-project`
 
@@ -365,7 +365,7 @@ DESCRIPTION
   Delete Project on Hosted Service
 ```
 
-_See code: [lib/commands/project/delete-project.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/project/delete-project.js)_
+_See code: [lib/commands/project/delete-project.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/project/delete-project.js)_
 
 ## `subql publish`
 
@@ -384,7 +384,7 @@ DESCRIPTION
   Upload this SubQuery project to IPFS
 ```
 
-_See code: [lib/commands/publish.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/publish.js)_
+_See code: [lib/commands/publish.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/publish.js)_
 
 ## `subql validate`
 
@@ -406,6 +406,6 @@ DESCRIPTION
   Check a folder or github repo is a validate subquery project
 ```
 
-_See code: [lib/commands/validate.js](https://github.com/packages/cli/blob/v3.6.2-13/lib/commands/validate.js)_
+_See code: [lib/commands/validate.js](https://github.com/packages/cli/blob/v4.0.1-2/lib/commands/validate.js)_
 
 <!-- commandsstop -->

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@subql/cli",
   "description": "cli for subquery",
-  "version": "4.0.0",
+  "version": "4.0.1-2",
   "author": "Ian He",
   "bin": {
     "subql": "./bin/run"
@@ -90,5 +90,6 @@
   "types": "lib/index.d.ts",
   "resolutions": {
     "node-fetch": "2.6.7"
-  }
+  },
+  "stableVersion": "4.0.1-1"
 }

packages/cli/src/controller/codegen-controller.test.ts

@@ -79,4 +79,9 @@ describe('Codegen can generate schema', () => {
     await codegen(projectPath, ['non-evm-project.yaml']);
     expect(fs.existsSync(`${projectPath}/src/types/abi-interfaces/`)).toBeFalsy();
   });
+  it('Should not generate proto-interfaces if no chaintypes are provided', async () => {
+    const projectPath = path.join(__dirname, '../../test/schemaTest');
+    await codegen(projectPath, ['project-cosmos.yaml']);
+    expect(fs.existsSync(`${projectPath}/src/types/proto-interfaces/`)).toBeFalsy();
+  });
 });

packages/cli/src/controller/codegen-controller.ts

@@ -272,9 +272,10 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL
 
   const chainTypes = plainManifests
     .filter((m) => validateCosmosManifest(m))
-    .map((m) => (m as CosmosManifest).network.chaintypes);
+    .map((m) => (m as CosmosManifest).network.chaintypes)
+    .filter((value) => value && value.size);
 
-  if (chainTypes.length !== 0) {
+  if (chainTypes.length) {
     await generateProto(chainTypes, projectPath, prepareDirPath, renderTemplate, upperFirst, tempProtoDir);
   }
   await generateCosmwasm(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);

packages/cli/src/utils/build.ts

@@ -9,14 +9,12 @@ import {Command} from '@oclif/core';
 import {DEFAULT_TS_MANIFEST, extensionIsTs, tsProjectYamlPath} from '@subql/common';
 
 const requireScriptWrapper = (scriptPath: string, outputPath: string): string =>
-  `
-  import {toJsonObject} from '@subql/common'
-  const {writeFileSync} =  require('fs');
-  const yaml = require('js-yaml');
-  const project = toJsonObject((require('${scriptPath}')).default);
-  const yamlOutput = yaml.dump(project);
-  writeFileSync('${outputPath}', \`# // Auto-generated , DO NOT EDIT\n\${yamlOutput}\`);
-`;
+  `import {toJsonObject} from '@subql/common';` +
+  `const {writeFileSync} = require('fs');` +
+  `const yaml = require('js-yaml');` +
+  `const project = toJsonObject((require('${scriptPath}')).default);` +
+  `const yamlOutput = yaml.dump(project);` +
+  `writeFileSync('${outputPath}', '# // Auto-generated , DO NOT EDIT\\n' + yamlOutput);`;
 
 export async function buildManifestFromLocation(location: string, command: Command): Promise<string> {
   let directory: string;
@@ -31,17 +29,11 @@ export async function buildManifestFromLocation(location: string, command: Comma
   } else {
     command.error('Argument `location` is not a valid directory or file');
   }
-  // Only build when ProjectYamlPath not exist and projectManifestEntry is in typescript
-  if (
-    !existsSync(tsProjectYamlPath(projectManifestEntry)) &&
-    existsSync(projectManifestEntry) &&
-    extensionIsTs(path.extname(projectManifestEntry))
-  ) {
-    try {
-      await generateManifestFromTs(projectManifestEntry, command);
-    } catch (e) {
-      throw new Error(`Failed to generate manifest from typescript ${projectManifestEntry}, ${e.message}`);
-    }
+  // We compile from TypeScript every time, even if the current YAML file exists, to ensure that the YAML file remains up-to-date with the latest changes
+  try {
+    await generateManifestFromTs(projectManifestEntry, command);
+  } catch (e) {
+    throw new Error(`Failed to generate manifest from typescript ${projectManifestEntry}, ${e.message}`);
   }
   return directory;
 }

packages/cli/test/schemaTest/project-cosmos.yaml

@@ -0,0 +1,29 @@
+specVersion: 1.0.0
+name: axelar-subql-starter
+version: 0.0.1
+runner:
+  node:
+    name: '@subql/node-cosmos'
+    version: '*'
+  query:
+    name: '@subql/query'
+    version: '*'
+description: >-
+  This project can be use as a starting point for developing your Cosmos (Axelar) based SubQuery project
+repository: 'https://github.com/subquery/cosmos-subql-starter'
+schema:
+  file: ./schema.graphql
+network:
+  chainId: axelar-dojo-1
+  endpoint: ['https://axelar-archrpc.chainode.tech/']
+  chaintypes: {}
+dataSources:
+  - kind: cosmos/Runtime
+    startBlock: 5262 # The first deposit event occurs on this block
+    mapping:
+      file: ./dist/index.js
+      handlers:
+        - handler: handleEvent
+          kind: cosmos/EventHandler
+          filter:
+            type: 'depositConfirmation'

packages/node-core/CHANGELOG.md

@@ -5,6 +5,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Support scoped debug logging (#2077)
+
+### Added
+- Debug logging for dictionary queries (#2077)
+
+### Fixed
+- Fixed Poi migration performance issue with `sqlIterator` 
+- Fixed AutoQueue timeout issue, align setting with nodeConfig.
 
 ### Fixed
 - Fixed Poi migration performance issue with `sqlIterator` 
@@ -13,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [5.0.3] - 2023-10-03
 ### Fixed
 - Fix reindex service without poi feature (2062)
+
 ### Changed
 - Version bump with `types-core` 0.1.1
 
@@ -414,7 +424,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Move blockchain agnostic code from `node` to `node-core` package. (#1222)
 
 [Unreleased]: https://github.com/subquery/subql/compare/node-core/5.0.4...HEAD
-[5.0.4]: https://github.com/subquery/subql/compare/node-core/5.0.3...node-core/5.0.4
 [5.0.3]: https://github.com/subquery/subql/compare/node-core/5.0.2...node-core/5.0.3
 [5.0.2]: https://github.com/subquery/subql/compare/node-core/5.0.1...node-core/5.0.2
 [5.0.1]: https://github.com/subquery/subql/compare/node-core/4.2.3...node-core/5.0.1

packages/node-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-core",
-  "version": "5.0.4-0",
+  "version": "5.0.4-2",
   "description": "Common node features that are agnostic to blockchains",
   "homepage": "https://github.com/subquery/subql",
   "repository": "github:subquery/subql",
@@ -45,5 +45,5 @@
   "devDependencies": {
     "@types/async-lock": "^1"
   },
-  "stableVersion": "5.0.3"
+  "stableVersion": "5.0.4-1"
 }

packages/node-core/src/configure/NodeConfig.ts

@@ -19,7 +19,7 @@ export interface IConfig {
   readonly batchSize: number;
   readonly timeout: number;
   readonly blockTime: number;
-  readonly debug: boolean;
+  readonly debug?: string;
   readonly preferRange: boolean;
   readonly networkEndpoint?: string[];
   readonly primaryNetworkEndpoint?: string;
@@ -62,7 +62,7 @@ const DEFAULT_CONFIG = {
   timeout: 900,
   blockTime: 6000,
   preferRange: false,
-  debug: false,
+  debug: undefined,
   queryLimit: 100,
   indexCountLimit: 10,
   timestampField: true,
@@ -177,7 +177,7 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     return this._config.blockTime;
   }
 
-  get debug(): boolean {
+  get debug(): string | undefined {
     return this._config.debug;
   }
 

packages/node-core/src/configure/configure.module.ts

@@ -5,7 +5,7 @@ import {handleCreateSubqueryProjectError, LocalReader, makeTempDir, ReaderFactor
 import {Reader} from '@subql/types-core';
 import {camelCase, isNil, omitBy} from 'lodash';
 import {ISubqueryProject} from '../indexer';
-import {getLogger, setLevel} from '../logger';
+import {getLogger, setDebugFilter} from '../logger';
 import {defaultSubqueryName, rebaseArgsWithManifest} from '../utils';
 import {IConfig, NodeConfig} from './NodeConfig';
 import {IProjectUpgradeService, ProjectUpgradeSevice, upgradableSubqueryProject} from './ProjectUpgrade.service';
@@ -126,7 +126,7 @@ export async function registerApp<P extends ISubqueryProject>(
   }
 
   if (config.debug) {
-    setLevel('debug');
+    setDebugFilter(config.debug);
   }
 
   const project = await createProject(

packages/node-core/src/db/db.module.ts

@@ -59,7 +59,8 @@ const sequelizeFactory = (option: SequelizeOption) => async () => {
 };
 
 const buildSequelizeOptions = (nodeConfig: NodeConfig, option: DbOption): SequelizeOption => {
-  const logger = getLogger('db');
+  const logger = getLogger('SQL');
+
   return {
     ...option,
     dialect: 'postgres',
@@ -72,11 +73,9 @@ const buildSequelizeOptions = (nodeConfig: NodeConfig, option: DbOption): Sequel
         cert: nodeConfig.postgresClientCert,
       },
     },
-    logging: nodeConfig.debug
-      ? (sql: string, timing?: number) => {
-          logger.debug(sql);
-        }
-      : false,
+    logging: (sql: string, timing?: number) => {
+      logger.debug(sql);
+    },
   };
 };
 
@@ -102,7 +101,6 @@ export class DbModule {
   }
 
   static forRoot(option: DbOption = DEFAULT_DB_OPTION): DynamicModule {
-    const logger = getLogger('db');
     return {
       module: DbModule,
       providers: [