From f474ac27d522ec09e4cfee5d0d133f941e414015 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 16 Sep 2024 22:07:09 +0800 Subject: [PATCH] Problem: bulk-add-genesis-account is not used in testground (#1582) * Problem: bulk-add-genesis-account is not used in testground Solution: - extract standalone changes from #1575 - use a simple and deterministic way to generate test accounts * fix * fix * fix coins * fix comment * multi coins * cleanup * fix py lint --------- Co-authored-by: mmsqe --- go.mod | 8 +-- go.sum | 16 +++--- gomod2nix.toml | 10 ++-- testground/benchmark/benchmark/main.py | 2 +- testground/benchmark/benchmark/peer.py | 54 ++++++++++++++------- testground/benchmark/benchmark/sendtx.py | 29 ++--------- testground/benchmark/benchmark/stateless.py | 40 +++++++++++---- testground/benchmark/benchmark/types.py | 10 +++- testground/benchmark/benchmark/utils.py | 15 ++++++ 9 files changed, 113 insertions(+), 71 deletions(-) diff --git a/go.mod b/go.mod index 45e1c5cba1..22f5881c57 100644 --- a/go.mod +++ b/go.mod @@ -254,10 +254,10 @@ require ( // release/v0.50.x replace ( - cosmossdk.io/client/v2 => github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2 - cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2 - cosmossdk.io/x/tx => github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2 - github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2 + cosmossdk.io/client/v2 => github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240916022730-3317ca17f0f1 + cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240916022730-3317ca17f0f1 + cosmossdk.io/x/tx => github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240916022730-3317ca17f0f1 + github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240916022730-3317ca17f0f1 ) replace ( diff --git a/go.sum b/go.sum index 1c600a83d9..2bb00018c1 100644 --- a/go.sum +++ b/go.sum @@ -423,14 +423,14 @@ github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c h1:MOgfS4+F github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/crypto-org-chain/cometbft-db v0.0.0-20231011055109-57922ac52a63 h1:R1QJ9a3XdYMSKo+1RdFifxb/g3lNypC52L/rpYrWoKo= github.com/crypto-org-chain/cometbft-db v0.0.0-20231011055109-57922ac52a63/go.mod h1:rocwIfnS+kA060x64gkSIRvWB9StSppIkJuo5MWzL24= -github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2 h1:4SoAvnxDaiIWcgm6XOmPDIdCf4/WNhNYLXGbij1eaA0= -github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2/go.mod h1:Rb43DdB0i/rKcCN69Tg2X3+zA4WhJ7MC8K3a6Ezh38E= -github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2 h1:5oGZtOUcauk9dtv+8BCfj2PEQyXEEEV+K3sP4OSvBmg= -github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2/go.mod h1:W5sR4asmVDUhJpEmuXTUBkk/yEefKlXTjVWcNciVSR0= -github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2 h1:CGh5I0L6IYhe0AJevb4vf5TE3ru+qAgMs437BlWCwo8= -github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg= -github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2 h1:mxlOSCru7YgmX055rrlkCSUu0D8lAqJ8Dnhp0yXCBuM= -github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY= +github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240916022730-3317ca17f0f1 h1:lWStPYj3ULmU5hRBFxR06QPK8Tp3v2t0qEBcEUHUjlk= +github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240916022730-3317ca17f0f1/go.mod h1:Rb43DdB0i/rKcCN69Tg2X3+zA4WhJ7MC8K3a6Ezh38E= +github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240916022730-3317ca17f0f1 h1:m1ayhc+L/fZ+1uregeAZt4GB7Il8oviJ/znWZyK4X94= +github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240916022730-3317ca17f0f1/go.mod h1:W5sR4asmVDUhJpEmuXTUBkk/yEefKlXTjVWcNciVSR0= +github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240916022730-3317ca17f0f1 h1:27XoLx7tmWPV/yBV21zVLoOyQVpB2vY7fvfI9DZk0+0= +github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240916022730-3317ca17f0f1/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg= +github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240916022730-3317ca17f0f1 h1:yKiDUGCgeFiM5RLwS8IPp0BmdQANVJN7qDTCdFCPLyE= +github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240916022730-3317ca17f0f1/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY= github.com/crypto-org-chain/ethermint v0.6.1-0.20240913100216-dbc7eb41488c h1:pJJNL+ZganmfcxzEijVNqwNDhzXsTyMk/Of1/lUvxlM= github.com/crypto-org-chain/ethermint v0.6.1-0.20240913100216-dbc7eb41488c/go.mod h1:D2lnc8ARuVmgc2/2IWla2Ky1o8/pjmyrnIt+d46Clco= github.com/crypto-org-chain/go-block-stm v0.0.0-20240912024944-1cd89976aa5e h1:FFpE6+Y4o5GxkeGwUcETM6amgohh7msWvWf1MDqueVc= diff --git a/gomod2nix.toml b/gomod2nix.toml index e08795eb15..3b2744bf5a 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -23,7 +23,7 @@ schema = 3 version = "v0.7.5" hash = "sha256-Nuw697sJr56kU3EU7DV1eYNkyI76psznIVqYAV6RfbQ=" [mod."cosmossdk.io/client/v2"] - version = "v2.0.0-20240911084450-6870ba130be2" + version = "v2.0.0-20240916022730-3317ca17f0f1" hash = "sha256-60hmufv3Ml4Pv3zNwgn8eeqlEINOR6n9MKr2QHddoxo=" replaced = "github.com/crypto-org-chain/cosmos-sdk/client/v2" [mod."cosmossdk.io/collections"] @@ -48,7 +48,7 @@ schema = 3 version = "v0.0.0-20230608160436-666c345ad23d" hash = "sha256-6BMBA98BpK3jG6++ZE4LdPQwwpS+lZ0GLMRF1fO4UfM=" [mod."cosmossdk.io/store"] - version = "v0.0.0-20240911084450-6870ba130be2" + version = "v0.0.0-20240916022730-3317ca17f0f1" hash = "sha256-Dm3sSZNJBcnBF33PULoTpK4rkNQbsZl0DfTqH1GPCQM=" replaced = "github.com/crypto-org-chain/cosmos-sdk/store" [mod."cosmossdk.io/tools/confix"] @@ -61,7 +61,7 @@ schema = 3 version = "v0.1.1" hash = "sha256-aps3LfnQau1TYeccGwtqHQvy1Rudc9+O+iVAwXBKyDw=" [mod."cosmossdk.io/x/tx"] - version = "v0.0.0-20240911084450-6870ba130be2" + version = "v0.0.0-20240916022730-3317ca17f0f1" hash = "sha256-xT5IdapEx1h46ofBpxcBQfzGF2EntmC8xZl7aym/6xE=" replaced = "github.com/crypto-org-chain/cosmos-sdk/x/tx" [mod."cosmossdk.io/x/upgrade"] @@ -173,8 +173,8 @@ schema = 3 version = "v1.0.0-beta.5" hash = "sha256-Fy/PbsOsd6iq0Njy3DVWK6HqWsogI+MkE8QslHGWyVg=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.50.6-0.20240911084450-6870ba130be2" - hash = "sha256-kl2sLe8vITIguRGtdeviDeP3R5JIbRg+eDsheGd4PqM=" + version = "v0.50.6-0.20240916022730-3317ca17f0f1" + hash = "sha256-ehNXD1RsXdC+Et+RrwuycvJBQ2yQnPNQIAb1JCUM77U=" replaced = "github.com/crypto-org-chain/cosmos-sdk" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" diff --git a/testground/benchmark/benchmark/main.py b/testground/benchmark/benchmark/main.py index 16c1a652dd..372ad3db4c 100644 --- a/testground/benchmark/benchmark/main.py +++ b/testground/benchmark/benchmark/main.py @@ -36,7 +36,7 @@ def entrypoint(ctx: Context): test_finish_entry = f"finish-test-{ctx.params.test_group_id}" if not ctx.is_validator: - generate_load(cli, ctx.params.num_accounts, ctx.params.num_txs) + generate_load(cli, ctx.params.num_accounts, ctx.params.num_txs, ctx.global_seq) print("finish test", ctx.group_seq) ctx.sync.signal_and_wait( test_finish_entry, ctx.params.test_group_instance_count diff --git a/testground/benchmark/benchmark/peer.py b/testground/benchmark/benchmark/peer.py index eb49fd5a7c..cacc84d9b6 100644 --- a/testground/benchmark/benchmark/peer.py +++ b/testground/benchmark/benchmark/peer.py @@ -3,19 +3,21 @@ from pathlib import Path from typing import List +from pydantic.json import pydantic_encoder + from .cli import ChainCommand from .context import Context from .network import get_data_ip from .topology import connect_all -from .types import GenesisAccount, PeerPacket -from .utils import patch_json, patch_toml +from .types import Balance, GenesisAccount, PeerPacket +from .utils import eth_to_bech32, gen_account, patch_json, patch_toml +DEFAULT_DENOM = "basecro" VAL_ACCOUNT = "validator" -VAL_INITIAL_AMOUNT = "100000000000000000000basecro" -VAL_STAKED_AMOUNT = "10000000000000000000basecro" -ACC_INITIAL_AMOUNT = "10000000000000000000000000basecro" +VAL_INITIAL_AMOUNT = Balance(amount="100000000000000000000", denom=DEFAULT_DENOM) +VAL_STAKED_AMOUNT = Balance(amount="10000000000000000000", denom=DEFAULT_DENOM) +ACC_INITIAL_AMOUNT = Balance(amount="10000000000000000000000000", denom=DEFAULT_DENOM) MEMPOOL_SIZE = 10000 -DEFAULT_DENOM = "basecro" VALIDATOR_GROUP = "validators" FULLNODE_GROUP = "fullnodes" CONTAINER_CRONOSD_PATH = "/bin/cronosd" @@ -30,6 +32,7 @@ def bootstrap(ctx: Context, cli) -> PeerPacket: ctx.params.chain_id, ctx.params.test_group_id, ctx.group_seq, + ctx.global_seq, ) data = ctx.sync.publish_subscribe_simple( @@ -58,6 +61,8 @@ def init_node( chain_id: str, group: str, group_seq: int, + global_seq: int, + num_accounts: int = 1, ) -> PeerPacket: default_kwargs = { "home": home, @@ -70,13 +75,27 @@ def init_node( default_denom=DEFAULT_DENOM, **default_kwargs, ) - cli("keys", "add", VAL_ACCOUNT, **default_kwargs) - cli("keys", "add", "account", **default_kwargs) - validator_addr = cli("keys", "show", VAL_ACCOUNT, "--address", **default_kwargs) - account_addr = cli("keys", "show", "account", "--address", **default_kwargs) + + val_acct = gen_account(global_seq, 0) + cli( + "keys", + "unsafe-import-eth-key", + VAL_ACCOUNT, + val_acct.key.hex(), + stdin=b"00000000\n", + **default_kwargs, + ) accounts = [ - GenesisAccount(address=validator_addr, balance=VAL_INITIAL_AMOUNT), - GenesisAccount(address=account_addr, balance=ACC_INITIAL_AMOUNT), + GenesisAccount( + address=eth_to_bech32(val_acct.address), + coins=[VAL_INITIAL_AMOUNT], + ), + ] + [ + GenesisAccount( + address=eth_to_bech32(gen_account(global_seq, i + 1).address), + coins=[ACC_INITIAL_AMOUNT], + ) + for i in range(num_accounts) ] node_id = cli("comet", "show-node-id", **default_kwargs) @@ -98,12 +117,13 @@ def gen_genesis( cli: ChainCommand, leader_home: Path, peers: List[PeerPacket], genesis_patch: dict ): for peer in peers: - for account in peer.accounts: + with tempfile.NamedTemporaryFile() as fp: + fp.write(json.dumps(peer.accounts, default=pydantic_encoder).encode()) + fp.flush() cli( "genesis", - "add-genesis-account", - account.address, - account.balance, + "bulk-add-genesis-account", + fp.name, home=leader_home, ) collect_gen_tx(cli, peers, home=leader_home) @@ -154,7 +174,7 @@ def gentx(cli, **kwargs): "genesis", "add-genesis-account", VAL_ACCOUNT, - VAL_INITIAL_AMOUNT, + str(VAL_INITIAL_AMOUNT), **kwargs, ) with tempfile.TemporaryDirectory() as tmp: diff --git a/testground/benchmark/benchmark/sendtx.py b/testground/benchmark/benchmark/sendtx.py index 34bb9f2cd0..8a313e70c9 100644 --- a/testground/benchmark/benchmark/sendtx.py +++ b/testground/benchmark/benchmark/sendtx.py @@ -4,33 +4,11 @@ import web3 from eth_account import Account -from .utils import export_eth_account, send_transaction, send_transactions +from .utils import gen_account, send_transaction -TEST_AMOUNT = 1000000000000000000 GAS_PRICE = 1000000000 -def fund_test_accounts(w3, from_account, num_accounts) -> [Account]: - accounts = [] - nonce = w3.eth.get_transaction_count(from_account.address) - txs = [] - for i in range(num_accounts): - acct = Account.create() - tx = { - "to": acct.address, - "value": TEST_AMOUNT, - "gas": 21000, - "gasPrice": GAS_PRICE, - "nonce": nonce + i, - } - txs.append(tx) - accounts.append(acct) - receipts = send_transactions(w3, txs, from_account) - for receipt in receipts: - assert receipt["status"] == 1 - return accounts - - def sendtx(w3: web3.Web3, acct: Account, tx_amount: int): initial_nonce = w3.eth.get_transaction_count(acct.address) print( @@ -77,11 +55,10 @@ def sendtx(w3: web3.Web3, acct: Account, tx_amount: int): ) -def generate_load(cli, num_accounts, num_txs, **kwargs): +def generate_load(cli, num_accounts, num_txs, global_seq, **kwargs): w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545")) assert w3.eth.chain_id == 777 - genesis_account = export_eth_account(cli, "account", **kwargs) - accounts = fund_test_accounts(w3, genesis_account, num_accounts) + accounts = [gen_account(global_seq, i + 1) for i in range(num_accounts)] with ThreadPoolExecutor(max_workers=num_accounts) as executor: futs = (executor.submit(sendtx, w3, acct, num_txs) for acct in accounts) for fut in as_completed(futs): diff --git a/testground/benchmark/benchmark/stateless.py b/testground/benchmark/benchmark/stateless.py index 42f64f392d..45df0b2e3a 100644 --- a/testground/benchmark/benchmark/stateless.py +++ b/testground/benchmark/benchmark/stateless.py @@ -31,7 +31,6 @@ # use cronosd on host machine LOCAL_CRONOSD_PATH = "cronosd" DEFAULT_CHAIN_ID = "cronos_777-1" -DEFAULT_DENOM = "basecro" # the container must be deployed with the prefixed name HOSTNAME_TEMPLATE = "testplan-{index}" LOCAL_RPC = "http://localhost:26657" @@ -103,12 +102,22 @@ def _gen( peers = [] for i in range(validators): print("init validator", i) - ip = hostname_template.format(index=i) - peers.append(init_node_local(cli, outdir, VALIDATOR_GROUP, i, ip)) + global_seq = i + ip = hostname_template.format(index=global_seq) + peers.append( + init_node_local( + cli, outdir, VALIDATOR_GROUP, i, global_seq, ip, num_accounts + ) + ) for i in range(fullnodes): print("init fullnode", i) - ip = hostname_template.format(index=i + validators) - peers.append(init_node_local(cli, outdir, FULLNODE_GROUP, i, ip)) + global_seq = i + validators + ip = hostname_template.format(index=global_seq) + peers.append( + init_node_local( + cli, outdir, FULLNODE_GROUP, i, global_seq, ip, num_accounts + ) + ) print("prepare genesis") # use a full node directory to prepare the genesis file @@ -183,7 +192,7 @@ def run(outdir: str, datadir: str, cronosd, global_seq): home = datadir / group / str(group_seq) try: - return do_run(home, cronosd, group, cfg) + return do_run(home, cronosd, group, global_seq, cfg) finally: # collect outputs output = Path("/data.tar.bz2") @@ -197,7 +206,7 @@ def run(outdir: str, datadir: str, cronosd, global_seq): shutil.copy(output, filename) -def do_run(home: str, cronosd: str, group: str, cfg: dict): +def do_run(home: str, cronosd: str, group: str, global_seq: int, cfg: dict): run_echo_server(ECHO_SERVER_PORT) # wait for persistent peers to be ready @@ -218,7 +227,12 @@ def do_run(home: str, cronosd: str, group: str, cfg: dict): if group == FULLNODE_GROUP or cfg.get("validator-generate-load", True): wait_for_w3() generate_load( - cli, cfg["num_accounts"], cfg["num_txs"], home=home, output="json" + cli, + cfg["num_accounts"], + cfg["num_txs"], + global_seq, + home=home, + output="json", ) # node quit when the chain is idle or halted for a while @@ -309,7 +323,13 @@ def block_txs(height): def init_node_local( - cli: ChainCommand, outdir: Path, group: str, group_seq: int, ip: str + cli: ChainCommand, + outdir: Path, + group: str, + group_seq: int, + global_seq: int, + ip: str, + num_accounts: int, ) -> PeerPacket: return init_node( cli, @@ -318,6 +338,8 @@ def init_node_local( DEFAULT_CHAIN_ID, group, group_seq, + global_seq, + num_accounts=num_accounts, ) diff --git a/testground/benchmark/benchmark/types.py b/testground/benchmark/benchmark/types.py index 4025df380d..14bc02e600 100644 --- a/testground/benchmark/benchmark/types.py +++ b/testground/benchmark/benchmark/types.py @@ -5,9 +5,17 @@ from .utils import bech32_to_eth +class Balance(BaseModel): + amount: str + denom: str + + def __str__(self): + return f"{self.amount}{self.denom}" + + class GenesisAccount(BaseModel): address: str - balance: str + coins: List[Balance] @property def eth_address(self) -> str: diff --git a/testground/benchmark/benchmark/utils.py b/testground/benchmark/benchmark/utils.py index 0b0580e20b..0d890c4999 100644 --- a/testground/benchmark/benchmark/utils.py +++ b/testground/benchmark/benchmark/utils.py @@ -10,6 +10,8 @@ from hexbytes import HexBytes from web3._utils.transactions import fill_nonce, fill_transaction_defaults +CRONOS_ADDRESS_PREFIX = "crc" + def patch_dict(doc, kwargs): for k, v in kwargs.items(): @@ -92,6 +94,11 @@ def bech32_to_eth(addr): return decode_bech32(addr).hex() +def eth_to_bech32(addr, prefix=CRONOS_ADDRESS_PREFIX): + bz = bech32.convertbits(HexBytes(addr), 8, 5) + return bech32.bech32_encode(prefix, bz) + + def sign_transaction(w3, tx, acct): "fill default fields and sign" tx["from"] = acct.address @@ -124,3 +131,11 @@ def send_transactions(w3, txs, acct, wait=True): def export_eth_account(cli, name: str, **kwargs) -> Account: kwargs.setdefault("keyring_backend", "test") return Account.from_key(cli("keys", "unsafe-export-eth-key", name, **kwargs)) + + +def gen_account(global_seq: int, index: int) -> Account: + """ + deterministically generate test private keys, + index 0 is reserved for validator account. + """ + return Account.from_key(((global_seq + 1) << 32 | index).to_bytes(32))