From 604f2a031d731ff5daa157b66e72da57952a9156 Mon Sep 17 00:00:00 2001 From: Naveen V Date: Thu, 12 Oct 2023 08:10:34 +0000 Subject: [PATCH] add ts manifest support for multichain projects --- packages/cli/src/commands/build/index.ts | 8 +- packages/cli/src/commands/codegen/index.ts | 13 +++- packages/cli/src/utils/build.ts | 89 +++++++++++++++++++--- packages/common/src/project/utils.ts | 25 +++--- 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/packages/cli/src/commands/build/index.ts b/packages/cli/src/commands/build/index.ts index b3e7d17844..96d219fd48 100644 --- a/packages/cli/src/commands/build/index.ts +++ b/packages/cli/src/commands/build/index.ts @@ -7,7 +7,7 @@ import path from 'path'; import {Command, Flags} from '@oclif/core'; import glob from 'glob'; import {runWebpack} from '../../controller/build-controller'; -import {resolveToAbsolutePath, buildManifestFromLocation, checkForTsManifest} from '../../utils'; +import {resolveToAbsolutePath, buildManifestFromLocation, getTsManifest} from '../../utils'; export default class Build extends Command { static description = 'Build this SubQuery project code'; @@ -28,8 +28,10 @@ export default class Build extends Command { assert(existsSync(location), 'Argument `location` is not a valid directory or file'); const directory = lstatSync(location).isDirectory() ? location : path.dirname(location); - if (checkForTsManifest(location)) { - await buildManifestFromLocation(location, this); + const tsManifest = getTsManifest(location, this); + + if (tsManifest) { + await buildManifestFromLocation(tsManifest, this); } // Get the output location from the project package.json main field diff --git a/packages/cli/src/commands/codegen/index.ts b/packages/cli/src/commands/codegen/index.ts index 65f38a33e6..0666c82b4f 100644 --- a/packages/cli/src/commands/codegen/index.ts +++ b/packages/cli/src/commands/codegen/index.ts @@ -4,7 +4,7 @@ import {Command, Flags} from '@oclif/core'; import {getProjectRootAndManifest, getSchemaPath} from '@subql/common'; import {codegen} from '../../controller/codegen-controller'; -import {resolveToAbsolutePath, buildManifestFromLocation, checkForTsManifest} from '../../utils'; +import {resolveToAbsolutePath, buildManifestFromLocation, getTsManifest} from '../../utils'; export default class Codegen extends Command { static description = 'Generate schemas for graph node'; @@ -27,8 +27,15 @@ export default class Codegen extends Command { const projectPath = resolveToAbsolutePath(file ?? location ?? process.cwd()); - if (checkForTsManifest(projectPath)) { - await buildManifestFromLocation(projectPath, this); + /* + ts manifest can be either single chain ts manifest + or multichain ts manifest + or multichain yaml manifest containing single chain ts project paths + */ + const tsManifest = getTsManifest(projectPath, this); + + if (tsManifest) { + await buildManifestFromLocation(tsManifest, this); } const {manifests, root} = getProjectRootAndManifest(projectPath); diff --git a/packages/cli/src/utils/build.ts b/packages/cli/src/utils/build.ts index 56219a74e1..d4a7cabe95 100644 --- a/packages/cli/src/utils/build.ts +++ b/packages/cli/src/utils/build.ts @@ -2,11 +2,20 @@ // SPDX-License-Identifier: GPL-3.0 import {execFile} from 'child_process'; -import {existsSync, lstatSync} from 'fs'; +import {assert} from 'console'; +import {existsSync, lstatSync, readFileSync, writeFileSync} from 'fs'; import util from 'node:util'; import path from 'path'; import {Command} from '@oclif/core'; -import {DEFAULT_TS_MANIFEST, extensionIsTs, tsProjectYamlPath} from '@subql/common'; +import { + DEFAULT_MULTICHAIN_MANIFEST, + DEFAULT_MULTICHAIN_TS_MANIFEST, + DEFAULT_TS_MANIFEST, + extensionIsTs, + tsProjectYamlPath, +} from '@subql/common'; +import {MultichainProjectManifest} from '@subql/types-core'; +import * as yaml from 'js-yaml'; const requireScriptWrapper = (scriptPath: string, outputPath: string): string => `import {toJsonObject} from '@subql/common';` + @@ -29,16 +38,27 @@ export async function buildManifestFromLocation(location: string, command: Comma } else { command.error('Argument `location` is not a valid directory or file'); } + // We compile from TypeScript every time, even if the current YAML file exists, to ensure that the YAML file remains up-to-date with the latest changes try { - await generateManifestFromTs(projectManifestEntry, command); + //we could have a multichain yaml with ts projects inside it + const projectYamlPath = projectManifestEntry.endsWith('.ts') + ? await generateManifestFromTs(projectManifestEntry, command) + : projectManifestEntry; + + if (isMultichain(projectYamlPath)) { + const tsManifests = getTsManifestsFromMultichain(projectYamlPath, command); + await Promise.all(tsManifests.map((manifest) => generateManifestFromTs(manifest, command))); + replaceTsReferencesInMultichain(projectYamlPath); + } } catch (e) { throw new Error(`Failed to generate manifest from typescript ${projectManifestEntry}, ${e.message}`); } return directory; } -async function generateManifestFromTs(projectManifestEntry: string, command: Command): Promise { +async function generateManifestFromTs(projectManifestEntry: string, command: Command): Promise { + assert(existsSync(projectManifestEntry), `${projectManifestEntry} does not exist`); const projectYamlPath = tsProjectYamlPath(projectManifestEntry); try { await util.promisify(execFile)( @@ -47,20 +67,67 @@ async function generateManifestFromTs(projectManifestEntry: string, command: Com {cwd: path.dirname(projectManifestEntry)} ); command.log(`Project manifest generated to ${projectYamlPath}`); + + return projectYamlPath; } catch (error) { throw new Error(`Failed to build ${projectManifestEntry}: ${error}`); } } -export function checkForTsManifest(location: string): boolean { - let projectManifestEntry: string; +//Returns either the single chain ts manifest or the multichain ts/yaml manifest +export function getTsManifest(location: string, command: Command): string { + let manifest: string; + if (lstatSync(location).isDirectory()) { - projectManifestEntry = path.join(location, DEFAULT_TS_MANIFEST); + //default ts manifest + manifest = path.join(location, DEFAULT_TS_MANIFEST); + if (existsSync(manifest)) { + return manifest; + } else { + //default multichain ts manifest + manifest = path.join(location, DEFAULT_MULTICHAIN_TS_MANIFEST); + if (existsSync(manifest)) { + return manifest; + } else { + //default yaml multichain manifest + manifest = path.join(location, DEFAULT_MULTICHAIN_MANIFEST); + if (existsSync(manifest)) { + return manifest; + } + } + } } else if (lstatSync(location).isFile()) { - projectManifestEntry = location; - } else { - throw new Error('Argument `location` is not a valid directory or file'); + if (location.endsWith('.ts')) { + return location; + } else if (isMultichain(location)) { + return location; + } } - return existsSync(projectManifestEntry) && projectManifestEntry.endsWith('.ts'); + return null; +} + +function getTsManifestsFromMultichain(location: string, command: Command): string[] { + const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest; + + if (!multichainContent || !multichainContent.projects) { + return []; + } + + return multichainContent.projects + .filter((project) => project.endsWith('.ts')) + .map((project) => path.resolve(path.dirname(location), project)); +} + +function isMultichain(location: string): boolean { + const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest; + + return !!multichainContent && !!multichainContent.projects; +} + +function replaceTsReferencesInMultichain(location: string): void { + const multichainContent = yaml.load(readFileSync(location, 'utf8')) as MultichainProjectManifest; + multichainContent.projects = multichainContent.projects.map((project) => tsProjectYamlPath(project)); + const yamlOutput = yaml.dump(multichainContent); + writeFileSync(location, yamlOutput); } diff --git a/packages/common/src/project/utils.ts b/packages/common/src/project/utils.ts index efc576db46..911b023b3f 100644 --- a/packages/common/src/project/utils.ts +++ b/packages/common/src/project/utils.ts @@ -21,6 +21,7 @@ import updateNotifier, {Package} from 'update-notifier'; import {RUNNER_ERROR_REGEX} from '../constants'; export const DEFAULT_MULTICHAIN_MANIFEST = 'subquery-multichain.yaml'; +export const DEFAULT_MULTICHAIN_TS_MANIFEST = 'subquery-multichain.ts'; export const DEFAULT_MANIFEST = 'project.yaml'; export const DEFAULT_TS_MANIFEST = 'project.ts'; @@ -93,25 +94,25 @@ export function getProjectRootAndManifest(subquery: string): ProjectRootAndManif throw new Error(`Extension ${ext} not supported for project ${subquery}`); } project.root = dir; + let projectYamlPath = subquery; + if (extensionIsTs(ext)) { - const projectYamlPath = tsProjectYamlPath(subquery); + projectYamlPath = tsProjectYamlPath(subquery); if (!fs.existsSync(projectYamlPath)) { throw new Error( `Could not find manifest ${projectYamlPath}, if pointing to a typescript manifest, please ensure build successfully` ); } - project.manifests.push(projectYamlPath); + } + + const multichainManifestContent = yaml.load(fs.readFileSync(projectYamlPath, 'utf8')) as MultichainProjectManifest; + // The project manifest could be empty + if (multichainManifestContent === null) { + throw new Error(`Read manifest content is null, ${projectYamlPath}`); + } else if (multichainManifestContent.projects && Array.isArray(multichainManifestContent.projects)) { + addMultichainManifestProjects(dir, multichainManifestContent, project); } else { - // when file path is yaml - const multichainManifestContent = yaml.load(fs.readFileSync(subquery, 'utf8')) as MultichainProjectManifest; - // The project manifest could be empty - if (multichainManifestContent === null) { - throw new Error(`Read manifest content is null, ${subquery}`); - } else if (multichainManifestContent.projects && Array.isArray(multichainManifestContent.projects)) { - addMultichainManifestProjects(dir, multichainManifestContent, project); - } else { - project.manifests.push(subquery); - } + project.manifests.push(projectYamlPath); } }