[WIP] modular CLI

packages/cli/package.json

@@ -10,13 +10,6 @@
   "dependencies": {
     "@oclif/core": "^2.15.0",
     "@subql/common": "workspace:*",
-    "@subql/common-algorand": "^3.4.0",
-    "@subql/common-concordium": "^3.6.0",
-    "@subql/common-cosmos": "^4.3.0",
-    "@subql/common-ethereum": "^3.8.3",
-    "@subql/common-near": "^3.5.0",
-    "@subql/common-stellar": "^3.5.0",
-    "@subql/common-substrate": "workspace:*",
     "@subql/utils": "workspace:*",
     "algosdk": "^1.19.0",
     "boxen": "5.1.2",
@@ -25,7 +18,7 @@
     "ethers": "^5.7.0",
     "fuzzy": "^0.1.3",
     "inquirer": "^8.2.0",
-    "inquirer-autocomplete-prompt": "^1.4.0",
+    "inquirer-autocomplete-prompt": "^2.0.0",
     "json5": "^2.2.3",
     "node-fetch": "2.6.7",
     "rimraf": "^3.0.2",
@@ -44,6 +37,13 @@
     "yaml-loader": "^0.6.0"
   },
   "devDependencies": {
+    "@subql/common-algorand": "^3.4.0",
+    "@subql/common-concordium": "^3.6.0",
+    "@subql/common-cosmos": "^4.4.1-0",
+    "@subql/common-ethereum": "^3.10.1-2",
+    "@subql/common-near": "^3.5.0",
+    "@subql/common-stellar": "^3.5.0",
+    "@subql/common-substrate": "workspace:*",
     "@types/ejs": "^3.1.0",
     "@types/inquirer": "^8.2.0",
     "@types/node": "^18.16.10",

packages/cli/src/commands/codegen/generate.ts

@@ -3,11 +3,10 @@
 
 import fs, {lstatSync} from 'fs';
 import path from 'path';
-import {EventFragment, FunctionFragment} from '@ethersproject/abi/src.ts/fragments';
+import type {EventFragment, FunctionFragment} from '@ethersproject/abi/src.ts/fragments';
 import {Command, Flags} from '@oclif/core';
-import {DEFAULT_MANIFEST, DEFAULT_TS_MANIFEST, extensionIsTs} from '@subql/common';
-import {SubqlRuntimeDatasource as EthereumDs} from '@subql/types-ethereum';
-import {parseContractPath} from 'typechain';
+import {DEFAULT_MANIFEST, DEFAULT_TS_MANIFEST, extensionIsTs, NETWORK_FAMILY} from '@subql/common';
+import type {SubqlRuntimeDatasource as EthereumDs} from '@subql/types-ethereum';
 import {
   constructMethod,
   filterExistingMethods,
@@ -23,6 +22,7 @@
   tsExtractor,
   yamlExtractor,
 } from '../../controller/generate-controller';
+import {loadDependency} from '../../modulars';
 import {extractFromTs} from '../../utils';
 
 export interface SelectedMethod {
@@ -102,8 +102,8 @@
     } else {
       this.error('Invalid manifest path');
     }
-
-    const abiName = parseContractPath(abiPath).name;
+    const ethModule = loadDependency(NETWORK_FAMILY.ethereum);
+    const abiName = ethModule.parseContractPath(abiPath).name;
 
     if (fs.existsSync(path.join(root, 'src/mappings/', `${abiName}Handlers.ts`))) {
       throw new Error(`file: ${abiName}Handlers.ts already exists`);
@@ -144,7 +144,7 @@
       } else {
         // yaml
         const existingManifest = await getManifestData(manifest);
         const existingDs = ((existingManifest.get('dataSources') as any)?.toJSON() as EthereumDs[]) ?? [];
 
         userInput = this.prepareUserInput(
           selectedEvents,

packages/cli/src/commands/codegen/index.ts

@@ -55,7 +55,7 @@ export default class Codegen extends Command {
     try {
       await codegen(root, manifests);
     } catch (err) {
-      console.error(err.message);
+      console.error(err.message, err.cause);
       process.exit(1);
     }
   }

packages/cli/src/controller/codegen-controller.ts

@@ -2,81 +2,37 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import path from 'path';
-import {DEFAULT_MANIFEST, getManifestPath, getSchemaPath, loadFromJsonOrYaml} from '@subql/common';
-import {isCustomDs as isCustomConcordiumDs, isRuntimeDs as isRuntimeConcordiumDs} from '@subql/common-concordium';
 import {
-  isCustomCosmosDs,
-  isRuntimeCosmosDs,
-  generateProto,
-  tempProtoDir,
-  validateCosmosManifest,
-  ProjectManifestImpls as CosmosManifest,
-  generateCosmwasm,
-  CosmosCustomModuleImpl,
-} from '@subql/common-cosmos';
-import {
-  isCustomDs as isCustomEthereumDs,
-  isRuntimeDs as isRuntimeEthereumDs,
-  generateAbis,
-} from '@subql/common-ethereum';
-import {isCustomDs as isCustomNearDs, isRuntimeDs as isRuntimeNearDs} from '@subql/common-near';
-import {isCustomDs as isCustomStellarDs, isRuntimeDs as isRuntimeStellarDs} from '@subql/common-stellar';
-import {isCustomDs as isCustomSubstrateDs, SubstrateCustomDataSource} from '@subql/common-substrate';
-import {
-  RuntimeDatasourceTemplate as SubstrateDsTemplate,
-  CustomDatasourceTemplate as SubstrateCustomDsTemplate,
-} from '@subql/types';
-import {
-  RuntimeDatasourceTemplate as ConcordiumDsTemplate,
-  CustomDatasourceTemplate as ConcordiumCustomDsTemplate,
-} from '@subql/types-concordium';
-import {TemplateBase} from '@subql/types-core';
-import {
-  RuntimeDatasourceTemplate as CosmosDsTemplate,
-  CustomDatasourceTemplate as CosmosCustomDsTemplate,
-} from '@subql/types-cosmos';
-import {
-  RuntimeDatasourceTemplate as EthereumDsTemplate,
-  CustomDatasourceTemplate as EthereumCustomDsTemplate,
-  SubqlRuntimeDatasource as EthereumDs,
+  DEFAULT_MANIFEST,
+  getManifestPath,
+  getProjectNetwork,
+  getSchemaPath,
+  loadFromJsonOrYaml,
+  NETWORK_FAMILY,
+} from '@subql/common';
+import type {SubstrateCustomDatasource} from '@subql/types';
+import {BaseTemplateDataSource, ProjectManifestV1_0_0, TemplateBase} from '@subql/types-core';
+import type {
   SubqlCustomDatasource as EthereumCustomDs,
+  SubqlRuntimeDatasource as EthereumDs,
 } from '@subql/types-ethereum';
-import {
-  RuntimeDatasourceTemplate as NearDsTemplate,
-  CustomDatasourceTemplate as NearCustomDsTemplate,
-} from '@subql/types-near';
-import {
-  RuntimeDatasourceTemplate as StellarDsTemplate,
-  CustomDatasourceTemplate as StellarCustomDsTemplate,
-} from '@subql/types-stellar';
 import {
   getAllEntitiesRelations,
+  getAllEnums,
   getAllJsonObjects,
-  setJsonObjectType,
   getTypeByScalarName,
   GraphQLEntityField,
-  GraphQLJsonFieldType,
   GraphQLEntityIndex,
-  getAllEnums,
+  GraphQLJsonFieldType,
+  setJsonObjectType,
 } from '@subql/utils';
-import {upperFirst, uniq, uniqBy} from 'lodash';
-import {renderTemplate, prepareDirPath} from '../utils';
+import {uniq, uniqBy, upperFirst} from 'lodash';
+import {loadDependency} from '../modulars';
+import {prepareDirPath, renderTemplate} from '../utils';
 
-export type TemplateKind =
-  | SubstrateDsTemplate
-  | SubstrateCustomDsTemplate
-  | CosmosDsTemplate
-  | CosmosCustomDsTemplate
-  | EthereumDsTemplate
-  | EthereumCustomDsTemplate
-  | NearDsTemplate
-  | NearCustomDsTemplate
-  | StellarDsTemplate
-  | StellarCustomDsTemplate
-  | ConcordiumDsTemplate
-  | ConcordiumCustomDsTemplate;
+export type TemplateKind = BaseTemplateDataSource;
 
-export type DatasourceKind = SubstrateCustomDataSource | EthereumDs | EthereumCustomDs;
+export type DatasourceKind = SubstrateCustomDatasource | EthereumDs | EthereumCustomDs;
 
 const MODEL_TEMPLATE_PATH = path.resolve(__dirname, '../template/model.ts.ejs');
 const MODELS_INDEX_TEMPLATE_PATH = path.resolve(__dirname, '../template/models-index.ts.ejs');
@@ -233,12 +189,11 @@
   await prepareDirPath(modelDir, true);
   await prepareDirPath(interfacesPath, false);
   const plainManifests = fileNames.map((fileName) => {
-    const project = loadFromJsonOrYaml(getManifestPath(projectPath, fileName));
-    return project as {
-      specVersion: string;
-      templates?: TemplateKind[];
-      dataSources: DatasourceKind[];
+    const project = loadFromJsonOrYaml(getManifestPath(projectPath, fileName)) as ProjectManifestV1_0_0 & {
+      networkFamily: NETWORK_FAMILY;
     };
+    project.networkFamily = getProjectNetwork(project);
+    return project;
   });
 
   const expectKeys = ['datasources', 'templates'];
@@ -247,7 +202,7 @@
     return Object.keys(plainManifest)
       .filter((key) => !expectKeys.includes(key))
       .map((dsKey) => {
         const value = (plainManifest as any)[dsKey];
         if (typeof value === 'object' && value) {
           return !!Object.keys(value).find((d) => d === 'assets') && value;
         }
@@ -278,14 +233,23 @@
     datasources = datasources.concat(customDatasources);
   }
 
-  const chainTypes = getChaintypes(plainManifests);
-
-  if (chainTypes.length) {
-    await generateProto(chainTypes, projectPath, prepareDirPath, renderTemplate, upperFirst, tempProtoDir);
+  const cosmosManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.cosmos);
+  if (cosmosManifests.length > 0) {
+    const cosmosModule = loadDependency(NETWORK_FAMILY.cosmos);
+    await cosmosModule.projectCodegen(
+      plainManifests,
+      projectPath,
+      prepareDirPath,
+      renderTemplate,
+      upperFirst,
+      datasources
+    );
+  }
+  const ethManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.ethereum);
+  if (ethManifests.length >= 0 || !!datasources.find((d) => d?.assets)) {
+    const ethModule = loadDependency(NETWORK_FAMILY.ethereum);
+    await ethModule.generateAbis(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
   }
-  await generateCosmwasm(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
-
-  await generateAbis(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
 
   if (exportTypes.interfaces || exportTypes.models || exportTypes.enums || exportTypes.datasources) {
     try {
@@ -301,15 +265,6 @@
   }
 }
 
-export function getChaintypes(
-  manifest: {templates?: TemplateKind[]; dataSources: DatasourceKind[]}[]
-): Map<string, CosmosCustomModuleImpl>[] {
-  return manifest
-    .filter((m) => validateCosmosManifest(m))
-    .map((m) => (m as CosmosManifest).network.chaintypes)
-    .filter((value) => value && Object.keys(value).length !== 0);
-}
-
 export async function generateSchemaModels(projectPath: string, schemaPath: string): Promise<void> {
   const modelDir = path.join(projectPath, MODEL_ROOT_DIR);
   const interfacesPath = path.join(projectPath, TYPE_ROOT_DIR, `interfaces.ts`);
@@ -390,7 +345,7 @@
 export async function generateDatasourceTemplates(projectPath: string, templates: TemplateKind[]): Promise<void> {
   const props = templates.map((t) => ({
     name: (t as TemplateBase).name,
-    args: hasParameters(t) ? 'Record<string, unknown>' : undefined,
+    args: 'Record<string, unknown>',
   }));
 
   const propsWithoutDuplicates = uniqBy(props, (prop) => `${prop.name}-${prop.args}`);
@@ -406,19 +361,3 @@
   }
   console.log(`* Datasource template constructors generated !`);
 }
-
-function hasParameters(t: TemplateKind): boolean {
-  return (
-    isRuntimeCosmosDs(t as CosmosDsTemplate) ||
-    isCustomCosmosDs(t as CosmosDsTemplate) ||
-    isRuntimeEthereumDs(t as EthereumDsTemplate) ||
-    isCustomEthereumDs(t as EthereumDsTemplate) ||
-    isCustomSubstrateDs(t as SubstrateDsTemplate) ||
-    isRuntimeNearDs(t as NearDsTemplate) ||
-    isCustomNearDs(t as NearDsTemplate) ||
-    isRuntimeStellarDs(t as StellarDsTemplate) ||
-    isCustomStellarDs(t as StellarDsTemplate) ||
-    isRuntimeConcordiumDs(t as ConcordiumDsTemplate) ||
-    isCustomConcordiumDs(t as ConcordiumDsTemplate)
-  );
-}

packages/cli/src/controller/codegen-cosmos.test.ts

@@ -4,26 +4,24 @@
 import fs from 'fs';
 import path from 'path';
 import {promisify} from 'util';
-import {generateProto, tempProtoDir} from '@subql/common-cosmos';
+import {loadFromJsonOrYaml, NETWORK_FAMILY} from '@subql/common';
+import {ProjectManifestV1_0_0} from '@subql/types-core';
 import {upperFirst} from 'lodash';
 import rimraf from 'rimraf';
+import {loadDependency} from '../modulars';
 import {prepareDirPath, renderTemplate} from '../utils';
 
 const PROJECT_PATH = path.join(__dirname, '../../test/protoTest1');
-const MOCK_CHAINTYPES = [
-  {
-    'osmosis.gamm.v1beta1': {
-      file: './proto/osmosis/gamm/v1beta1/tx.proto',
-      messages: ['MsgSwapExactAmountIn'],
-    },
+const MOCK_CHAINTYPES = {
+  'osmosis.gamm.v1beta1': {
+    file: './proto/osmosis/gamm/v1beta1/tx.proto',
+    messages: ['MsgSwapExactAmountIn'],
   },
-  {
-    'osmosis.poolmanager.v1beta1': {
-      file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto',
-      messages: ['SwapAmountInRoute'],
-    },
+  'osmosis.poolmanager.v1beta1': {
+    file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto',
+    messages: ['SwapAmountInRoute'],
   },
-];
+};
 
 jest.setTimeout(30000);
 
@@ -32,6 +30,9 @@
     await promisify(rimraf)(path.join(__dirname, '../../test/protoTest1/src'));
   });
 
+  const cosmosModule = loadDependency(NETWORK_FAMILY.cosmos);
+  const manifest = loadFromJsonOrYaml(path.join(PROJECT_PATH, 'project.yaml')) as ProjectManifestV1_0_0<any, any, any>;
+
   it('Able to generate ts types from protobufs', async () => {
     const expectedGeneratedCode =
       '' +
@@ -49,34 +50,37 @@
 }
 
 `;
-    await generateProto(MOCK_CHAINTYPES, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst);
+
+    manifest.network.chaintypes = MOCK_CHAINTYPES;
+    await cosmosModule.projectCodegen([manifest], PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, []);
     const codegenResult = await fs.promises.readFile(path.join(PROJECT_PATH, '/src/types/CosmosMessageTypes.ts'));
     expect(fs.existsSync(`${PROJECT_PATH}/src/types/CosmosMessageTypes.ts`)).toBeTruthy();
     expect(codegenResult.toString()).toBe(expectedGeneratedCode);
   });
 
   it('On missing protobuf dependency should throw', async () => {
-    const badChainTypes = [
-      {
-        'osmosis.gamm.v1beta1': {
-          file: './proto/cosmos/osmosis/gamm/v1beta1/tx.proto',
-          messages: ['MsgSwapExactAmountIn'],
-        },
+    const badChainType = {
+      'osmosis.gamm.v1beta1': {
+        file: './proto/cosmos/osmosis/gamm/v1beta1/tx.proto',
+        messages: ['MsgSwapExactAmountIn'],
       },
-    ];
-    await expect(
-      generateProto(badChainTypes, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst)
+    };
+    manifest.network.chaintypes = badChainType;
+    await expect(() =>
+      cosmosModule.projectCodegen([manifest], PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, [])
     ).rejects.toThrow(
       'Failed to generate from protobufs. Error: chainType osmosis.gamm.v1beta1, file ./proto/cosmos/osmosis/gamm/v1beta1/tx.proto does not exist'
     );
   });
 
-  it('create temp dir with all protobufs', async () => {
-    // user Protobufs should not be overwritten
-    const preFile = await fs.promises.readFile(path.join(PROJECT_PATH, 'proto/osmosis/gamm/v1beta1/tx.proto'));
-    const tmpDir = await tempProtoDir(PROJECT_PATH);
-    const afterFile = await fs.promises.readFile(path.join(tmpDir, 'osmosis/gamm/v1beta1/tx.proto'));
-    expect(preFile.toString()).toBe(afterFile.toString());
-    await promisify(rimraf)(tmpDir);
-  });
+  // TODO, we lost tmpDir path here, will move this test to cosmos
+  // it('create temp dir with all protobufs', async () => {
+  //   // user Protobufs should not be overwritten
+  //   const preFile = await fs.promises.readFile(path.join(PROJECT_PATH, 'proto/osmosis/gamm/v1beta1/tx.proto'));
+  //   await cosmosModule.projectCodegen([manifest], PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst,[]);
+  //   const tmpDir = await tempProtoDir(PROJECT_PATH);
+  //   const afterFile = await fs.promises.readFile(path.join(tmpDir, 'osmosis/gamm/v1beta1/tx.proto'));
+  //   expect(preFile.toString()).toBe(afterFile.toString());
+  //   await promisify(rimraf)(tmpDir);
+  // });
 });