add ts manifest support for multichain projects

subquery · Oct 12, 2023 · 604f2a0 · 604f2a0
1 parent 95e60f3
commit 604f2a0
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 29 deletions.
diff --git a/packages/cli/src/commands/build/index.ts b/packages/cli/src/commands/build/index.ts
@@ -7,7 +7,7 @@ import path from 'path';
 import {Command, Flags} from '@oclif/core';
 import glob from 'glob';
 import {runWebpack} from '../../controller/build-controller';
-import {resolveToAbsolutePath, buildManifestFromLocation, checkForTsManifest} from '../../utils';
+import {resolveToAbsolutePath, buildManifestFromLocation, getTsManifest} from '../../utils';
 
 export default class Build extends Command {
   static description = 'Build this SubQuery project code';
@@ -28,8 +28,10 @@ export default class Build extends Command {
       assert(existsSync(location), 'Argument `location` is not a valid directory or file');
       const directory = lstatSync(location).isDirectory() ? location : path.dirname(location);
 
-      if (checkForTsManifest(location)) {
-        await buildManifestFromLocation(location, this);
+      const tsManifest = getTsManifest(location, this);
+
+      if (tsManifest) {
+        await buildManifestFromLocation(tsManifest, this);
       }
 
       // Get the output location from the project package.json main field

diff --git a/packages/cli/src/commands/codegen/index.ts b/packages/cli/src/commands/codegen/index.ts
@@ -4,7 +4,7 @@
 import {Command, Flags} from '@oclif/core';
 import {getProjectRootAndManifest, getSchemaPath} from '@subql/common';
 import {codegen} from '../../controller/codegen-controller';
-import {resolveToAbsolutePath, buildManifestFromLocation, checkForTsManifest} from '../../utils';
+import {resolveToAbsolutePath, buildManifestFromLocation, getTsManifest} from '../../utils';
 
 export default class Codegen extends Command {
   static description = 'Generate schemas for graph node';
@@ -27,8 +27,15 @@ export default class Codegen extends Command {
 
     const projectPath = resolveToAbsolutePath(file ?? location ?? process.cwd());
 
-    if (checkForTsManifest(projectPath)) {
-      await buildManifestFromLocation(projectPath, this);
+    /*
+      ts manifest can be either single chain ts manifest
+      or multichain ts manifest
+      or multichain yaml manifest containing single chain ts project paths
+    */
+    const tsManifest = getTsManifest(projectPath, this);
+
+    if (tsManifest) {
+      await buildManifestFromLocation(tsManifest, this);
     }
 
     const {manifests, root} = getProjectRootAndManifest(projectPath);

diff --git a/packages/cli/src/utils/build.ts b/packages/cli/src/utils/build.ts
@@ -2,11 +2,20 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {execFile} from 'child_process';
-import {existsSync, lstatSync} from 'fs';
+import {assert} from 'console';
+import {existsSync, lstatSync, readFileSync, writeFileSync} from 'fs';
 import util from 'node:util';
 import path from 'path';
 import {Command} from '@oclif/core';
-import {DEFAULT_TS_MANIFEST, extensionIsTs, tsProjectYamlPath} from '@subql/common';
+import {
+  DEFAULT_MULTICHAIN_MANIFEST,
+  DEFAULT_MULTICHAIN_TS_MANIFEST,
+  DEFAULT_TS_MANIFEST,
+  extensionIsTs,
+  tsProjectYamlPath,
+} from '@subql/common';
+import {MultichainProjectManifest} from '@subql/types-core';
+import * as yaml from 'js-yaml';
 
 const requireScriptWrapper = (scriptPath: string, outputPath: string): string =>
   `import {toJsonObject} from '@subql/common';` +
@@ -29,16 +38,27 @@ export async function buildManifestFromLocation(location: string, command: Comma
   } else {
     command.error('Argument `location` is not a valid directory or file');
   }
+
   // We compile from TypeScript every time, even if the current YAML file exists, to ensure that the YAML file remains up-to-date with the latest changes
   try {
-    await generateManifestFromTs(projectManifestEntry, command);
+    //we could have a multichain yaml with ts projects inside it
+    const projectYamlPath = projectManifestEntry.endsWith('.ts')
+      ? await generateManifestFromTs(projectManifestEntry, command)
+      : projectManifestEntry;
+
+    if (isMultichain(projectYamlPath)) {
+      const tsManifests = getTsManifestsFromMultichain(projectYamlPath, command);
+      await Promise.all(tsManifests.map((manifest) => generateManifestFromTs(manifest, command)));
+      replaceTsReferencesInMultichain(projectYamlPath);
+    }
   } catch (e) {
     throw new Error(`Failed to generate manifest from typescript ${projectManifestEntry}, ${e.message}`);
   }
   return directory;
 }
 
-async function generateManifestFromTs(projectManifestEntry: string, command: Command): Promise<void> {
+async function generateManifestFromTs(projectManifestEntry: string, command: Command): Promise<string> {
+  assert(existsSync(projectManifestEntry), `${projectManifestEntry} does not exist`);
   const projectYamlPath = tsProjectYamlPath(projectManifestEntry);
   try {
     await util.promisify(execFile)(
@@ -47,20 +67,67 @@ async function generateManifestFromTs(projectManifestEntry: string, command: Com
       {cwd: path.dirname(projectManifestEntry)}
     );
     command.log(`Project manifest generated to ${projectYamlPath}`);
+
+    return projectYamlPath;
   } catch (error) {
     throw new Error(`Failed to build ${projectManifestEntry}: ${error}`);
   }
 }
 
-export function checkForTsManifest(location: string): boolean {
-  let projectManifestEntry: string;
+//Returns either the single chain ts manifest or the multichain ts/yaml manifest
+export function getTsManifest(location: string, command: Command): string {
+  let manifest: string;
+
   if (lstatSync(location).isDirectory()) {
-    projectManifestEntry = path.join(location, DEFAULT_TS_MANIFEST);
+    //default ts manifest
+    manifest = path.join(location, DEFAULT_TS_MANIFEST);
+    if (existsSync(manifest)) {
+      return manifest;
+    } else {
+      //default multichain ts manifest
+      manifest = path.join(location, DEFAULT_MULTICHAIN_TS_MANIFEST);
+      if (existsSync(manifest)) {
+        return manifest;
+      } else {
+        //default yaml multichain manifest
+        manifest = path.join(location, DEFAULT_MULTICHAIN_MANIFEST);
+        if (existsSync(manifest)) {
+          return manifest;
+        }
+      }
+    }
   } else if (lstatSync(location).isFile()) {
-    projectManifestEntry = location;
-  } else {
-    throw new Error('Argument `location` is not a valid directory or file');
+    if (location.endsWith('.ts')) {
+      return location;
+    } else if (isMultichain(location)) {
+      return location;
+    }
   }
 
-  return existsSync(projectManifestEntry) && projectManifestEntry.endsWith('.ts');
+  return null;
+}
+
+function getTsManifestsFromMultichain(location: string, command: Command): string[] {
+  const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest;
+
+  if (!multichainContent || !multichainContent.projects) {
+    return [];
+  }
+
+  return multichainContent.projects
+    .filter((project) => project.endsWith('.ts'))
+    .map((project) => path.resolve(path.dirname(location), project));
+}
+
+function isMultichain(location: string): boolean {
+  const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest;
+
+  return !!multichainContent && !!multichainContent.projects;
+}
+
+function replaceTsReferencesInMultichain(location: string): void {
+  const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest;
+  multichainContent.projects = multichainContent.projects.map((project) => tsProjectYamlPath(project));
+  const yamlOutput = yaml.dump(multichainContent);
+  writeFileSync(location, yamlOutput);
 }
diff --git a/packages/common/src/project/utils.ts b/packages/common/src/project/utils.ts
@@ -21,6 +21,7 @@ import updateNotifier, {Package} from 'update-notifier';
 import {RUNNER_ERROR_REGEX} from '../constants';
 
 export const DEFAULT_MULTICHAIN_MANIFEST = 'subquery-multichain.yaml';
+export const DEFAULT_MULTICHAIN_TS_MANIFEST = 'subquery-multichain.ts';
 export const DEFAULT_MANIFEST = 'project.yaml';
 export const DEFAULT_TS_MANIFEST = 'project.ts';
 
@@ -93,25 +94,25 @@ export function getProjectRootAndManifest(subquery: string): ProjectRootAndManif
       throw new Error(`Extension ${ext} not supported for project ${subquery}`);
     }
     project.root = dir;
+    let projectYamlPath = subquery;
+
     if (extensionIsTs(ext)) {
-      const projectYamlPath = tsProjectYamlPath(subquery);
+      projectYamlPath = tsProjectYamlPath(subquery);
       if (!fs.existsSync(projectYamlPath)) {
         throw new Error(
           `Could not find manifest ${projectYamlPath}, if pointing to a typescript manifest, please ensure build successfully`
         );
       }
-      project.manifests.push(projectYamlPath);
+    }
+
+    const multichainManifestContent = yaml.load(fs.readFileSync(projectYamlPath, 'utf8')) as MultichainProjectManifest;
+    // The project manifest could be empty
+    if (multichainManifestContent === null) {
+      throw new Error(`Read manifest content is null, ${projectYamlPath}`);
+    } else if (multichainManifestContent.projects && Array.isArray(multichainManifestContent.projects)) {
+      addMultichainManifestProjects(dir, multichainManifestContent, project);
     } else {
-      // when file path is yaml
-      const multichainManifestContent = yaml.load(fs.readFileSync(subquery, 'utf8')) as MultichainProjectManifest;
-      // The project manifest could be empty
-      if (multichainManifestContent === null) {
-        throw new Error(`Read manifest content is null, ${subquery}`);
-      } else if (multichainManifestContent.projects && Array.isArray(multichainManifestContent.projects)) {
-        addMultichainManifestProjects(dir, multichainManifestContent, project);
-      } else {
-        project.manifests.push(subquery);
-      }
+      project.manifests.push(projectYamlPath);
     }
   }