add ts manifest support for multichain projects

packages/cli/CHANGELOG.md

@@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Multichain support for TypeScript manifest (#2097)
 - Support for multi endpoints CLI deployment (#2117)
 
 ## [4.0.5] - 2023-10-18

packages/cli/src/commands/build/index.ts

@@ -7,7 +7,7 @@ import path from 'path';
 import {Command, Flags} from '@oclif/core';
 import glob from 'glob';
 import {runWebpack} from '../../controller/build-controller';
-import {resolveToAbsolutePath, buildManifestFromLocation, checkForTsManifest} from '../../utils';
+import {resolveToAbsolutePath, buildManifestFromLocation, getTsManifest} from '../../utils';
 
 export default class Build extends Command {
   static description = 'Build this SubQuery project code';
@@ -28,8 +28,10 @@ export default class Build extends Command {
       assert(existsSync(location), 'Argument `location` is not a valid directory or file');
       const directory = lstatSync(location).isDirectory() ? location : path.dirname(location);
 
-      if (checkForTsManifest(location)) {
-        await buildManifestFromLocation(location, this);
+      const tsManifest = getTsManifest(location, this);
+
+      if (tsManifest) {
+        await buildManifestFromLocation(tsManifest, this);
       }
 
       // Get the output location from the project package.json main field

packages/cli/src/commands/codegen/index.ts

@@ -4,7 +4,7 @@
 import {Command, Flags} from '@oclif/core';
 import {getProjectRootAndManifest, getSchemaPath} from '@subql/common';
 import {codegen} from '../../controller/codegen-controller';
-import {resolveToAbsolutePath, buildManifestFromLocation, checkForTsManifest} from '../../utils';
+import {resolveToAbsolutePath, buildManifestFromLocation, getTsManifest} from '../../utils';
 
 export default class Codegen extends Command {
   static description = 'Generate schemas for graph node';
@@ -27,8 +27,15 @@ export default class Codegen extends Command {
 
     const projectPath = resolveToAbsolutePath(file ?? location ?? process.cwd());
 
-    if (checkForTsManifest(projectPath)) {
-      await buildManifestFromLocation(projectPath, this);
+    /*
+      ts manifest can be either single chain ts manifest
+      or multichain ts manifest
+      or multichain yaml manifest containing single chain ts project paths
+    */
+    const tsManifest = getTsManifest(projectPath, this);
+
+    if (tsManifest) {
+      await buildManifestFromLocation(tsManifest, this);
     }
 
     const {manifests, root} = getProjectRootAndManifest(projectPath);

packages/cli/src/controller/init-controller.test.ts

@@ -123,9 +123,10 @@ describe('Cli can create project', () => {
       projects.find((p) => p.name === 'Polkadot-starter')
     );
     await prepare(projectPath, projectSpec);
-    const [specVersion, endpoint, author, description] = await readDefaults(projectPath);
+    const [endpoint, author, description] = await readDefaults(projectPath);
 
-    expect(projectSpec.specVersion).toEqual(specVersion);
+    //spec version is  not returned from readDefaults
+    //expect(projectSpec.specVersion).toEqual(specVersion);
     expect(projectSpec.endpoint).toEqual(endpoint);
     expect(projectSpec.author).toEqual(author);
     expect(projectSpec.description).toEqual(description);

packages/cli/src/controller/init-controller.ts

@@ -195,7 +195,11 @@ export async function prepareManifest(projectPath: string, project: ProjectSpecB
 
   if (isTs) {
     const tsManifest = (await fs.promises.readFile(tsPath, 'utf8')).toString();
-    const formattedEndpoint = `[ ${JSON.stringify(project.endpoint).slice(1, -1)} ]`;
+    //converting string endpoint to array of string.
+    const formattedEndpoint = Array.isArray(project.endpoint)
+      ? JSON.stringify(project.endpoint)
+      : `[ "${project.endpoint}" ]`;
+
     manifestData = findReplace(tsManifest, ENDPOINT_REG, `endpoint: ${formattedEndpoint}`);
   } else {
     //load and write manifest(project.yaml)

packages/cli/src/utils/build.spec.ts

@@ -0,0 +1,78 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import {existsSync, readFileSync, writeFileSync} from 'fs';
+import path from 'path';
+import {Command} from '@oclif/core';
+import {MultichainProjectManifest} from '@subql/types-core';
+import * as yaml from 'js-yaml';
+import rimraf from 'rimraf';
+import {buildManifestFromLocation} from './build';
+
+describe('Manifest generation', () => {
+  afterEach(() => {
+    const projectPath = path.join(__dirname, '../../test/tsManifestTest');
+    rimraf.sync(path.join(projectPath, 'project1.yaml'));
+    rimraf.sync(path.join(projectPath, 'subquery-multichain2.yaml'));
+    rimraf.sync(path.join(projectPath, 'project2.yaml'));
+    rimraf.sync(path.join(projectPath, 'subquery-multichain.yaml'));
+    rimraf.sync(path.join(projectPath, 'subquery-multichain3.yaml'));
+  });
+
+  it('should build ts manifest from multichain file', async () => {
+    const projectPath = path.join(__dirname, '../../test/tsManifestTest/subquery-multichain.ts');
+    await expect(
+      buildManifestFromLocation(projectPath, {log: console.log} as unknown as Command)
+    ).resolves.toBeDefined();
+    expect(existsSync(path.join(projectPath, '../project1.yaml'))).toBe(true);
+    expect(existsSync(path.join(projectPath, '../project2.yaml'))).toBe(true);
+    expect(existsSync(path.join(projectPath, '../subquery-multichain.yaml'))).toBe(true);
+
+    //ts files are replaced with yaml files
+    const multichainContent = yaml.load(
+      readFileSync(path.join(projectPath, '../subquery-multichain.yaml'), 'utf8')
+    ) as MultichainProjectManifest;
+    multichainContent.projects.forEach((project) => project.endsWith('.yaml'));
+  }, 50000);
+
+  it('throws error on unknown file in multichain manifest', async () => {
+    const projectPath = path.join(__dirname, '../../test/tsManifestTest/subquery-multichain2.ts');
+    await expect(buildManifestFromLocation(projectPath, {log: console.log} as unknown as Command)).rejects.toThrow();
+  }, 50000);
+
+  it('allows both ts and yaml file in multichain manifest', async () => {
+    const projectPath = path.join(__dirname, '../../test/tsManifestTest/subquery-multichain3.ts');
+    await expect(
+      buildManifestFromLocation(projectPath, {log: console.log} as unknown as Command)
+    ).resolves.toBeDefined();
+    expect(existsSync(path.join(projectPath, '../project1.yaml'))).toBe(true);
+    expect(existsSync(path.join(projectPath, '../project3.yaml'))).toBe(true);
+    expect(existsSync(path.join(projectPath, '../subquery-multichain3.yaml'))).toBe(true);
+
+    //ts files are replaced with yaml files
+    const multichainContent = yaml.load(
+      readFileSync(path.join(projectPath, '../subquery-multichain3.yaml'), 'utf8')
+    ) as MultichainProjectManifest;
+    multichainContent.projects.forEach((project) => project.endsWith('.yaml'));
+  }, 50000);
+
+  it('should build ts manifest from yaml multichain file', async () => {
+    const projectPath = path.join(__dirname, '../../test/tsManifestTest/subquery-multichain4.yaml');
+    await expect(
+      buildManifestFromLocation(projectPath, {log: console.log} as unknown as Command)
+    ).resolves.toBeDefined();
+    expect(existsSync(path.join(projectPath, '../project1.yaml'))).toBe(true);
+    expect(existsSync(path.join(projectPath, '../project2.yaml'))).toBe(true);
+
+    //ts files are replaced with yaml files
+    const multichainContent = yaml.load(
+      readFileSync(path.join(projectPath, '../subquery-multichain4.yaml'), 'utf8')
+    ) as MultichainProjectManifest;
+    multichainContent.projects.forEach((project) => expect(project.endsWith('.yaml')).toBe(true));
+
+    //revert yaml to ts
+    multichainContent.projects = multichainContent.projects.map((project) => project.replace('.yaml', '.ts'));
+
+    writeFileSync(path.join(projectPath, '../subquery-multichain4.yaml'), yaml.dump(multichainContent));
+  }, 50000);
+});

packages/cli/src/utils/build.ts

@@ -2,11 +2,19 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {execFile} from 'child_process';
-import {existsSync, lstatSync} from 'fs';
+import {assert} from 'console';
+import {existsSync, lstatSync, readFileSync, writeFileSync} from 'fs';
 import util from 'node:util';
 import path from 'path';
 import {Command} from '@oclif/core';
-import {DEFAULT_TS_MANIFEST, extensionIsTs, tsProjectYamlPath} from '@subql/common';
+import {
+  DEFAULT_MULTICHAIN_MANIFEST,
+  DEFAULT_MULTICHAIN_TS_MANIFEST,
+  DEFAULT_TS_MANIFEST,
+  tsProjectYamlPath,
+} from '@subql/common';
+import {MultichainProjectManifest} from '@subql/types-core';
+import * as yaml from 'js-yaml';
 
 const requireScriptWrapper = (scriptPath: string, outputPath: string): string =>
   `import {toJsonObject} from '@subql/common';` +
@@ -29,16 +37,27 @@ export async function buildManifestFromLocation(location: string, command: Comma
   } else {
     command.error('Argument `location` is not a valid directory or file');
   }
+
   // We compile from TypeScript every time, even if the current YAML file exists, to ensure that the YAML file remains up-to-date with the latest changes
   try {
-    await generateManifestFromTs(projectManifestEntry, command);
+    //we could have a multichain yaml with ts projects inside it
+    const projectYamlPath = projectManifestEntry.endsWith('.ts')
+      ? await generateManifestFromTs(projectManifestEntry, command)
+      : projectManifestEntry;
+
+    if (isMultichain(projectYamlPath)) {
+      const tsManifests = getTsManifestsFromMultichain(projectYamlPath, command);
+      await Promise.all(tsManifests.map((manifest) => generateManifestFromTs(manifest, command)));
+      replaceTsReferencesInMultichain(projectYamlPath);
+    }
   } catch (e) {
     throw new Error(`Failed to generate manifest from typescript ${projectManifestEntry}, ${e.message}`);
   }
   return directory;
 }
 
-async function generateManifestFromTs(projectManifestEntry: string, command: Command): Promise<void> {
+export async function generateManifestFromTs(projectManifestEntry: string, command: Command): Promise<string> {
+  assert(existsSync(projectManifestEntry), `${projectManifestEntry} does not exist`);
   const projectYamlPath = tsProjectYamlPath(projectManifestEntry);
   try {
     await util.promisify(execFile)(
@@ -47,20 +66,65 @@ async function generateManifestFromTs(projectManifestEntry: string, command: Com
       {cwd: path.dirname(projectManifestEntry)}
     );
     command.log(`Project manifest generated to ${projectYamlPath}`);
+
+    return projectYamlPath;
   } catch (error) {
     throw new Error(`Failed to build ${projectManifestEntry}: ${error}`);
   }
 }
 
-export function checkForTsManifest(location: string): boolean {
-  let projectManifestEntry: string;
+//Returns either the single chain ts manifest or the multichain ts/yaml manifest
+export function getTsManifest(location: string, command: Command): string {
+  let manifest: string;
+
   if (lstatSync(location).isDirectory()) {
-    projectManifestEntry = path.join(location, DEFAULT_TS_MANIFEST);
+    //default ts manifest
+    manifest = path.join(location, DEFAULT_TS_MANIFEST);
+    if (existsSync(manifest)) {
+      return manifest;
+    } else {
+      //default multichain ts manifest
+      manifest = path.join(location, DEFAULT_MULTICHAIN_TS_MANIFEST);
+      if (existsSync(manifest)) {
+        return manifest;
+      } else {
+        //default yaml multichain manifest
+        manifest = path.join(location, DEFAULT_MULTICHAIN_MANIFEST);
+        if (existsSync(manifest)) {
+          return manifest;
+        }
+      }
+    }
   } else if (lstatSync(location).isFile()) {
-    projectManifestEntry = location;
-  } else {
-    throw new Error('Argument `location` is not a valid directory or file');
+    if (location.endsWith('.ts') || isMultichain(location)) {
+      return location;
+    }
   }
 
-  return existsSync(projectManifestEntry) && projectManifestEntry.endsWith('.ts');
+  return null;
+}
+
+function getTsManifestsFromMultichain(location: string, command: Command): string[] {
+  const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest;
+
+  if (!multichainContent || !multichainContent.projects) {
+    return [];
+  }
+
+  return multichainContent.projects
+    .filter((project) => project.endsWith('.ts'))
+    .map((project) => path.resolve(path.dirname(location), project));
+}
+
+function isMultichain(location: string): boolean {
+  const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest;
+
+  return !!multichainContent && !!multichainContent.projects;
+}
+
+function replaceTsReferencesInMultichain(location: string): void {
+  const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest;
+  multichainContent.projects = multichainContent.projects.map((project) => tsProjectYamlPath(project));
+  const yamlOutput = yaml.dump(multichainContent);
+  writeFileSync(location, yamlOutput);
 }

packages/cli/test/tsManifestTest/project1.ts

@@ -0,0 +1,49 @@
+import {SubstrateDatasourceKind, SubstrateHandlerKind, SubstrateProject} from '@subql/types';
+
+const project: SubstrateProject = {
+  specVersion: '1.0.0',
+  name: 'multichain-transfers-polkadot',
+  version: '0.0.1',
+  runner: {
+    node: {
+      name: '@subql/node',
+      version: '>=1.0.0',
+    },
+    query: {
+      name: '@subql/query',
+      version: '*',
+    },
+  },
+  description:
+    'This project is an example of a multichain project that indexes multiple networks into the same database. Read more about it at https://academy.subquery.network/build/multi-chain.html',
+  repository: 'https://github.com/subquery/multi-networks-transfers.git',
+  schema: {
+    file: './schema.graphql',
+  },
+  network: {
+    chainId: '0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe',
+    endpoint: ['wss://kusama.api.onfinality.io/public-ws', 'wss://kusama-rpc.polkadot.io'],
+    dictionary: 'https://api.subquery.network/sq/subquery/kusama-dictionary',
+  },
+  dataSources: [
+    {
+      kind: SubstrateDatasourceKind.Runtime,
+      startBlock: 1,
+      mapping: {
+        file: './dist/index.js',
+        handlers: [
+          {
+            handler: 'handleKusamaEvent',
+            kind: SubstrateHandlerKind.Event,
+            filter: {
+              module: 'balances',
+              method: 'Transfer',
+            },
+          },
+        ],
+      },
+    },
+  ],
+};
+
+export default project;

packages/cli/test/tsManifestTest/project2.ts

@@ -0,0 +1,49 @@
+import {SubstrateDatasourceKind, SubstrateHandlerKind, SubstrateProject} from '@subql/types';
+
+const project: SubstrateProject = {
+  specVersion: '1.0.0',
+  name: 'multichain-transfers-polkadot',
+  version: '0.0.1',
+  runner: {
+    node: {
+      name: '@subql/node',
+      version: '>=1.0.0',
+    },
+    query: {
+      name: '@subql/query',
+      version: '*',
+    },
+  },
+  description:
+    'This project is an example of a multichain project that indexes multiple networks into the same database. Read more about it at https://academy.subquery.network/build/multi-chain.html',
+  repository: 'https://github.com/subquery/multi-networks-transfers.git',
+  schema: {
+    file: './schema.graphql',
+  },
+  network: {
+    chainId: '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3',
+    endpoint: ['wss://polkadot.api.onfinality.io/public-ws', 'wss://rpc.polkadot.io'],
+    dictionary: 'https://api.subquery.network/sq/subquery/polkadot-dictionary',
+  },
+  dataSources: [
+    {
+      kind: SubstrateDatasourceKind.Runtime,
+      startBlock: 1,
+      mapping: {
+        file: './dist/index.js',
+        handlers: [
+          {
+            handler: 'handlePolkadotEvent',
+            kind: SubstrateHandlerKind.Event,
+            filter: {
+              module: 'balances',
+              method: 'Transfer',
+            },
+          },
+        ],
+      },
+    },
+  ],
+};
+
+export default project;