Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] modular CLI #2456

Merged
merged 18 commits into from
Jul 3, 2024
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,6 @@
"dependencies": {
"@oclif/core": "^2.15.0",
"@subql/common": "workspace:*",
"@subql/common-algorand": "^3.4.0",
"@subql/common-concordium": "^3.6.0",
"@subql/common-cosmos": "^4.3.0",
"@subql/common-ethereum": "^3.8.3",
"@subql/common-near": "^3.5.0",
"@subql/common-stellar": "^3.5.0",
"@subql/common-substrate": "workspace:*",
"@subql/utils": "workspace:*",
"algosdk": "^1.19.0",
"boxen": "5.1.2",
Expand All @@ -25,7 +18,7 @@
"ethers": "^5.7.0",
"fuzzy": "^0.1.3",
"inquirer": "^8.2.0",
"inquirer-autocomplete-prompt": "^1.4.0",
"inquirer-autocomplete-prompt": "^2.0.0",
"json5": "^2.2.3",
"node-fetch": "2.6.7",
"rimraf": "^3.0.2",
Expand All @@ -44,6 +37,13 @@
"yaml-loader": "^0.6.0"
},
"devDependencies": {
"@subql/common-algorand": "^3.4.0",
"@subql/common-concordium": "^3.6.0",
"@subql/common-cosmos": "^4.4.1-0",
"@subql/common-ethereum": "^3.10.1-2",
"@subql/common-near": "^3.5.0",
"@subql/common-stellar": "^3.5.0",
"@subql/common-substrate": "workspace:*",
"@types/ejs": "^3.1.0",
"@types/inquirer": "^8.2.0",
"@types/node": "^18.16.10",
Expand Down
12 changes: 6 additions & 6 deletions packages/cli/src/commands/codegen/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@

import fs, {lstatSync} from 'fs';
import path from 'path';
import {EventFragment, FunctionFragment} from '@ethersproject/abi/src.ts/fragments';
import type {EventFragment, FunctionFragment} from '@ethersproject/abi/src.ts/fragments';
import {Command, Flags} from '@oclif/core';
import {DEFAULT_MANIFEST, DEFAULT_TS_MANIFEST, extensionIsTs} from '@subql/common';
import {SubqlRuntimeDatasource as EthereumDs} from '@subql/types-ethereum';
import {parseContractPath} from 'typechain';
import {DEFAULT_MANIFEST, DEFAULT_TS_MANIFEST, extensionIsTs, NETWORK_FAMILY} from '@subql/common';
import type {SubqlRuntimeDatasource as EthereumDs} from '@subql/types-ethereum';
import {
constructMethod,
filterExistingMethods,
Expand All @@ -23,6 +22,7 @@
tsExtractor,
yamlExtractor,
} from '../../controller/generate-controller';
import {loadDependency} from '../../modulars';
import {extractFromTs} from '../../utils';

export interface SelectedMethod {
Expand Down Expand Up @@ -102,8 +102,8 @@
} else {
this.error('Invalid manifest path');
}

const abiName = parseContractPath(abiPath).name;
const ethModule = loadDependency(NETWORK_FAMILY.ethereum);
const abiName = ethModule.parseContractPath(abiPath).name;

if (fs.existsSync(path.join(root, 'src/mappings/', `${abiName}Handlers.ts`))) {
throw new Error(`file: ${abiName}Handlers.ts already exists`);
Expand Down Expand Up @@ -144,7 +144,7 @@
} else {
// yaml
const existingManifest = await getManifestData(manifest);
const existingDs = ((existingManifest.get('dataSources') as any)?.toJSON() as EthereumDs[]) ?? [];

Check warning on line 147 in packages/cli/src/commands/codegen/generate.ts

View workflow job for this annotation

GitHub Actions / code-style

Unexpected any. Specify a different type

userInput = this.prepareUserInput(
selectedEvents,
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/commands/codegen/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ export default class Codegen extends Command {
try {
await codegen(root, manifests);
} catch (err) {
console.error(err.message);
console.error(err.message, err.cause);
process.exit(1);
}
}
Expand Down
141 changes: 40 additions & 101 deletions packages/cli/src/controller/codegen-controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,81 +2,37 @@
// SPDX-License-Identifier: GPL-3.0

import path from 'path';
import {DEFAULT_MANIFEST, getManifestPath, getSchemaPath, loadFromJsonOrYaml} from '@subql/common';
import {isCustomDs as isCustomConcordiumDs, isRuntimeDs as isRuntimeConcordiumDs} from '@subql/common-concordium';
import {
isCustomCosmosDs,
isRuntimeCosmosDs,
generateProto,
tempProtoDir,
validateCosmosManifest,
ProjectManifestImpls as CosmosManifest,
generateCosmwasm,
CosmosCustomModuleImpl,
} from '@subql/common-cosmos';
import {
isCustomDs as isCustomEthereumDs,
isRuntimeDs as isRuntimeEthereumDs,
generateAbis,
} from '@subql/common-ethereum';
import {isCustomDs as isCustomNearDs, isRuntimeDs as isRuntimeNearDs} from '@subql/common-near';
import {isCustomDs as isCustomStellarDs, isRuntimeDs as isRuntimeStellarDs} from '@subql/common-stellar';
import {isCustomDs as isCustomSubstrateDs, SubstrateCustomDataSource} from '@subql/common-substrate';
import {
RuntimeDatasourceTemplate as SubstrateDsTemplate,
CustomDatasourceTemplate as SubstrateCustomDsTemplate,
} from '@subql/types';
import {
RuntimeDatasourceTemplate as ConcordiumDsTemplate,
CustomDatasourceTemplate as ConcordiumCustomDsTemplate,
} from '@subql/types-concordium';
import {TemplateBase} from '@subql/types-core';
import {
RuntimeDatasourceTemplate as CosmosDsTemplate,
CustomDatasourceTemplate as CosmosCustomDsTemplate,
} from '@subql/types-cosmos';
import {
RuntimeDatasourceTemplate as EthereumDsTemplate,
CustomDatasourceTemplate as EthereumCustomDsTemplate,
SubqlRuntimeDatasource as EthereumDs,
DEFAULT_MANIFEST,
getManifestPath,
getProjectNetwork,
getSchemaPath,
loadFromJsonOrYaml,
NETWORK_FAMILY,
} from '@subql/common';
import type {SubstrateCustomDatasource} from '@subql/types';
import {BaseTemplateDataSource, ProjectManifestV1_0_0, TemplateBase} from '@subql/types-core';
import type {
SubqlCustomDatasource as EthereumCustomDs,
SubqlRuntimeDatasource as EthereumDs,
} from '@subql/types-ethereum';
import {
RuntimeDatasourceTemplate as NearDsTemplate,
CustomDatasourceTemplate as NearCustomDsTemplate,
} from '@subql/types-near';
import {
RuntimeDatasourceTemplate as StellarDsTemplate,
CustomDatasourceTemplate as StellarCustomDsTemplate,
} from '@subql/types-stellar';
import {
getAllEntitiesRelations,
getAllEnums,
getAllJsonObjects,
setJsonObjectType,
getTypeByScalarName,
GraphQLEntityField,
GraphQLJsonFieldType,
GraphQLEntityIndex,
getAllEnums,
GraphQLJsonFieldType,
setJsonObjectType,
} from '@subql/utils';
import {upperFirst, uniq, uniqBy} from 'lodash';
import {renderTemplate, prepareDirPath} from '../utils';
import {uniq, uniqBy, upperFirst} from 'lodash';
import {loadDependency} from '../modulars';
import {prepareDirPath, renderTemplate} from '../utils';

export type TemplateKind =
| SubstrateDsTemplate
| SubstrateCustomDsTemplate
| CosmosDsTemplate
| CosmosCustomDsTemplate
| EthereumDsTemplate
| EthereumCustomDsTemplate
| NearDsTemplate
| NearCustomDsTemplate
| StellarDsTemplate
| StellarCustomDsTemplate
| ConcordiumDsTemplate
| ConcordiumCustomDsTemplate;
export type TemplateKind = BaseTemplateDataSource;

export type DatasourceKind = SubstrateCustomDataSource | EthereumDs | EthereumCustomDs;
export type DatasourceKind = SubstrateCustomDatasource | EthereumDs | EthereumCustomDs;

const MODEL_TEMPLATE_PATH = path.resolve(__dirname, '../template/model.ts.ejs');
const MODELS_INDEX_TEMPLATE_PATH = path.resolve(__dirname, '../template/models-index.ts.ejs');
Expand Down Expand Up @@ -233,12 +189,11 @@
await prepareDirPath(modelDir, true);
await prepareDirPath(interfacesPath, false);
const plainManifests = fileNames.map((fileName) => {
const project = loadFromJsonOrYaml(getManifestPath(projectPath, fileName));
return project as {
specVersion: string;
templates?: TemplateKind[];
dataSources: DatasourceKind[];
const project = loadFromJsonOrYaml(getManifestPath(projectPath, fileName)) as ProjectManifestV1_0_0 & {
networkFamily: NETWORK_FAMILY;
};
project.networkFamily = getProjectNetwork(project);
return project;
});

const expectKeys = ['datasources', 'templates'];
Expand All @@ -247,7 +202,7 @@
return Object.keys(plainManifest)
.filter((key) => !expectKeys.includes(key))
.map((dsKey) => {
const value = (plainManifest as any)[dsKey];

Check warning on line 205 in packages/cli/src/controller/codegen-controller.ts

View workflow job for this annotation

GitHub Actions / code-style

Unexpected any. Specify a different type
if (typeof value === 'object' && value) {
return !!Object.keys(value).find((d) => d === 'assets') && value;
}
Expand Down Expand Up @@ -278,14 +233,23 @@
datasources = datasources.concat(customDatasources);
}

const chainTypes = getChaintypes(plainManifests);

if (chainTypes.length) {
await generateProto(chainTypes, projectPath, prepareDirPath, renderTemplate, upperFirst, tempProtoDir);
const cosmosManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.cosmos);
if (cosmosManifests.length > 0) {
const cosmosModule = loadDependency(NETWORK_FAMILY.cosmos);
await cosmosModule.projectCodegen(
plainManifests,
projectPath,
prepareDirPath,
renderTemplate,
upperFirst,
datasources
);
}
const ethManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.ethereum);
if (ethManifests.length >= 0 || !!datasources.find((d) => d?.assets)) {
const ethModule = loadDependency(NETWORK_FAMILY.ethereum);
await ethModule.generateAbis(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
}
await generateCosmwasm(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);

await generateAbis(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);

if (exportTypes.interfaces || exportTypes.models || exportTypes.enums || exportTypes.datasources) {
try {
Expand All @@ -301,15 +265,6 @@
}
}

export function getChaintypes(
manifest: {templates?: TemplateKind[]; dataSources: DatasourceKind[]}[]
): Map<string, CosmosCustomModuleImpl>[] {
return manifest
.filter((m) => validateCosmosManifest(m))
.map((m) => (m as CosmosManifest).network.chaintypes)
.filter((value) => value && Object.keys(value).length !== 0);
}

export async function generateSchemaModels(projectPath: string, schemaPath: string): Promise<void> {
const modelDir = path.join(projectPath, MODEL_ROOT_DIR);
const interfacesPath = path.join(projectPath, TYPE_ROOT_DIR, `interfaces.ts`);
Expand Down Expand Up @@ -390,7 +345,7 @@
export async function generateDatasourceTemplates(projectPath: string, templates: TemplateKind[]): Promise<void> {
const props = templates.map((t) => ({
name: (t as TemplateBase).name,
args: hasParameters(t) ? 'Record<string, unknown>' : undefined,
args: 'Record<string, unknown>',
}));

const propsWithoutDuplicates = uniqBy(props, (prop) => `${prop.name}-${prop.args}`);
Expand All @@ -406,19 +361,3 @@
}
console.log(`* Datasource template constructors generated !`);
}

function hasParameters(t: TemplateKind): boolean {
return (
isRuntimeCosmosDs(t as CosmosDsTemplate) ||
isCustomCosmosDs(t as CosmosDsTemplate) ||
isRuntimeEthereumDs(t as EthereumDsTemplate) ||
isCustomEthereumDs(t as EthereumDsTemplate) ||
isCustomSubstrateDs(t as SubstrateDsTemplate) ||
isRuntimeNearDs(t as NearDsTemplate) ||
isCustomNearDs(t as NearDsTemplate) ||
isRuntimeStellarDs(t as StellarDsTemplate) ||
isCustomStellarDs(t as StellarDsTemplate) ||
isRuntimeConcordiumDs(t as ConcordiumDsTemplate) ||
isCustomConcordiumDs(t as ConcordiumDsTemplate)
);
}
62 changes: 30 additions & 32 deletions packages/cli/src/controller/codegen-cosmos.test.ts
stwiname marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,25 @@
import fs from 'fs';
import path from 'path';
import {promisify} from 'util';
import {generateProto, tempProtoDir} from '@subql/common-cosmos';
import {loadFromJsonOrYaml, NETWORK_FAMILY} from '@subql/common';
import {ProjectManifestV1_0_0} from '@subql/types-core';
import type {CosmosDatasource, CustomDatasourceTemplate, RuntimeDatasourceTemplate} from '@subql/types-cosmos';
import {upperFirst} from 'lodash';
import rimraf from 'rimraf';
import {loadDependency} from '../modulars';
import {prepareDirPath, renderTemplate} from '../utils';

const PROJECT_PATH = path.join(__dirname, '../../test/protoTest1');
const MOCK_CHAINTYPES = [
{
'osmosis.gamm.v1beta1': {
file: './proto/osmosis/gamm/v1beta1/tx.proto',
messages: ['MsgSwapExactAmountIn'],
},
const MOCK_CHAINTYPES = {
'osmosis.gamm.v1beta1': {
file: './proto/osmosis/gamm/v1beta1/tx.proto',
messages: ['MsgSwapExactAmountIn'],
},
{
'osmosis.poolmanager.v1beta1': {
file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto',
messages: ['SwapAmountInRoute'],
},
'osmosis.poolmanager.v1beta1': {
file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto',
messages: ['SwapAmountInRoute'],
},
];
};

jest.setTimeout(30000);

Expand All @@ -32,6 +31,13 @@
await promisify(rimraf)(path.join(__dirname, '../../test/protoTest1/src'));
});

const cosmosModule = loadDependency(NETWORK_FAMILY.cosmos);
const manifest = loadFromJsonOrYaml(path.join(PROJECT_PATH, 'project.yaml')) as ProjectManifestV1_0_0<
CosmosDatasource,
RuntimeDatasourceTemplate | CustomDatasourceTemplate,
any

Check warning on line 38 in packages/cli/src/controller/codegen-cosmos.test.ts

View workflow job for this annotation

GitHub Actions / code-style

Unexpected any. Specify a different type
>;

it('Able to generate ts types from protobufs', async () => {
const expectedGeneratedCode =
'' +
Expand All @@ -49,34 +55,26 @@
}

`;
await generateProto(MOCK_CHAINTYPES, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst);

manifest.network.chaintypes = MOCK_CHAINTYPES;
await cosmosModule.projectCodegen([manifest], PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, []);
const codegenResult = await fs.promises.readFile(path.join(PROJECT_PATH, '/src/types/CosmosMessageTypes.ts'));
expect(fs.existsSync(`${PROJECT_PATH}/src/types/CosmosMessageTypes.ts`)).toBeTruthy();
expect(codegenResult.toString()).toBe(expectedGeneratedCode);
});

it('On missing protobuf dependency should throw', async () => {
const badChainTypes = [
{
'osmosis.gamm.v1beta1': {
file: './proto/cosmos/osmosis/gamm/v1beta1/tx.proto',
messages: ['MsgSwapExactAmountIn'],
},
const badChainType = {
'osmosis.gamm.v1beta1': {
file: './proto/cosmos/osmosis/gamm/v1beta1/tx.proto',
messages: ['MsgSwapExactAmountIn'],
},
];
await expect(
generateProto(badChainTypes, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst)
};
manifest.network.chaintypes = badChainType;
await expect(() =>
cosmosModule.projectCodegen([manifest], PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, [])
).rejects.toThrow(
'Failed to generate from protobufs. Error: chainType osmosis.gamm.v1beta1, file ./proto/cosmos/osmosis/gamm/v1beta1/tx.proto does not exist'
);
});

it('create temp dir with all protobufs', async () => {
// user Protobufs should not be overwritten
const preFile = await fs.promises.readFile(path.join(PROJECT_PATH, 'proto/osmosis/gamm/v1beta1/tx.proto'));
const tmpDir = await tempProtoDir(PROJECT_PATH);
const afterFile = await fs.promises.readFile(path.join(tmpDir, 'osmosis/gamm/v1beta1/tx.proto'));
expect(preFile.toString()).toBe(afterFile.toString());
await promisify(rimraf)(tmpDir);
});
});
Loading
Loading