From 31639e1eb481cdd944fe585630d5e636775e5f08 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Mar 2023 14:03:33 +0200 Subject: [PATCH 1/8] Add Wyrm support. --- moonworm/crawler/moonstream_ethereum_state_provider.py | 7 ++++++- moonworm/crawler/networks.py | 8 ++++++++ moonworm/crawler/utils.py | 1 + moonworm/version.py | 2 +- setup.py | 2 +- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py index 3600c03..162c522 100644 --- a/moonworm/crawler/moonstream_ethereum_state_provider.py +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -17,6 +17,7 @@ PolygonLabel, PolygonTransaction, XDaiTransaction, + WyrmTransaction, yield_db_session_ctx, ) from .utils import Network @@ -82,7 +83,11 @@ def get_last_block_number(self) -> int: @staticmethod def _transform_to_w3_tx( tx_raw: Union[ - EthereumTransaction, MumbaiTransaction, PolygonTransaction, XDaiTransaction + EthereumTransaction, + MumbaiTransaction, + PolygonTransaction, + XDaiTransaction, + WyrmTransaction, ], ) -> Dict[str, Any]: tx = { diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py index 16a7bb8..e31acc7 100644 --- a/moonworm/crawler/networks.py +++ b/moonworm/crawler/networks.py @@ -16,6 +16,9 @@ XDaiBlock, XDaiLabel, XDaiTransaction, + WyrmBlock, + Wyrmlabel, + WyrmTransaction, ) except ImportError: print("this feature requires moonstreamdb which is not installed") @@ -48,4 +51,9 @@ "labels": XDaiLabel, "transactions": XDaiTransaction, }, + Network.wyrm: { + "blocks": WyrmBlock, + "labels": Wyrmlabel, + "transactions": WyrmTransaction, + }, } diff --git a/moonworm/crawler/utils.py b/moonworm/crawler/utils.py index 2fb7557..b3aade1 100644 --- a/moonworm/crawler/utils.py +++ b/moonworm/crawler/utils.py @@ -6,3 +6,4 @@ class Network(Enum): polygon = "polygon" mumbai = "mumbai" xdai = "xdai" + wyrm = "wyrm" diff --git a/moonworm/version.py b/moonworm/version.py index d568ee0..6822674 100644 --- a/moonworm/version.py +++ b/moonworm/version.py @@ -1 +1 @@ -MOONWORM_VERSION = "0.6.0" +MOONWORM_VERSION = "0.6.1" diff --git a/setup.py b/setup.py index 3a3677e..e94d061 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ], extras_require={ "dev": ["isort", "mypy", "wheel", "web3[tester] >=5.27.0"], - "moonstream": ["moonstreamdb >= 0.3.2"], + "moonstream": ["moonstreamdb >= 0.3.3"], "distribute": ["setuptools", "twine", "wheel"], }, description="moonworm: Generate a command line interface to any Ethereum smart contract", From b2d87ce655f6bb422d5fb5dd1c3b281e2d1a1b0b Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Mar 2023 14:08:29 +0200 Subject: [PATCH 2/8] Fix sorting. --- moonworm/crawler/moonstream_ethereum_state_provider.py | 2 +- moonworm/crawler/networks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py index 162c522..f0be690 100644 --- a/moonworm/crawler/moonstream_ethereum_state_provider.py +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -16,8 +16,8 @@ MumbaiTransaction, PolygonLabel, PolygonTransaction, - XDaiTransaction, WyrmTransaction, + XDaiTransaction, yield_db_session_ctx, ) from .utils import Network diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py index e31acc7..a927719 100644 --- a/moonworm/crawler/networks.py +++ b/moonworm/crawler/networks.py @@ -13,12 +13,12 @@ PolygonBlock, PolygonLabel, PolygonTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, WyrmBlock, Wyrmlabel, WyrmTransaction, + XDaiBlock, + XDaiLabel, + XDaiTransaction, ) except ImportError: print("this feature requires moonstreamdb which is not installed") From 29a3a80a28c884d21854a6e294c85375ab63034d Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Mar 2023 15:59:23 +0200 Subject: [PATCH 3/8] Move all dependency from moonstreamdb to networks. Will required upper version of moonstreamdb. --- moonworm/cli.py | 19 ++++++++--- .../moonstream_ethereum_state_provider.py | 2 +- moonworm/crawler/networks.py | 34 ++----------------- 3 files changed, 18 insertions(+), 37 deletions(-) diff --git a/moonworm/cli.py b/moonworm/cli.py index e5d6546..992e88e 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -1,9 +1,10 @@ import argparse import json import os -from multiprocessing.sharedctypes import Value from pathlib import Path from shutil import copyfile +from types import MappingProxyType +from typing import Any from web3.main import Web3 from web3.middleware import geth_poa_middleware @@ -11,8 +12,7 @@ from moonworm.crawler.ethereum_state_provider import Web3StateProvider from moonworm.watch import watch_contract -from .contracts import CU, ERC20, ERC721, CULands -from .crawler.utils import Network +from .contracts import CU, ERC20, ERC721 from .deployment import find_deployment_block from .generators.basic import ( generate_contract_cli_content, @@ -154,6 +154,9 @@ def handle_watch(args: argparse.Namespace) -> None: if args.db: if args.network is None: raise ValueError("Please specify --network") + + from .crawler.networks import Network + network = Network.__members__[args.network] from .crawler.moonstream_ethereum_state_provider import ( @@ -215,6 +218,14 @@ def generate_argument_parser() -> argparse.ArgumentParser: """ Generates the command-line argument parser for the "moonworm" command. """ + networks: MappingProxyType[Any, Any] = MappingProxyType({}) + try: + from .crawler.networks import Network + + networks = Network.__members__ + except Exception: + pass + parser = argparse.ArgumentParser(description="Moonworm: Manage your smart contract") parser.add_argument( "-v", @@ -256,7 +267,7 @@ def generate_argument_parser() -> argparse.ArgumentParser: watch_parser.add_argument( "--network", - choices=Network.__members__, + choices=networks, default=None, help="Network name that represents models from db. If --db is set, required", ) diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py index f0be690..e49c0a0 100644 --- a/moonworm/crawler/moonstream_ethereum_state_provider.py +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -19,8 +19,8 @@ WyrmTransaction, XDaiTransaction, yield_db_session_ctx, + Network, ) -from .utils import Network logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py index a927719..2cc91fc 100644 --- a/moonworm/crawler/networks.py +++ b/moonworm/crawler/networks.py @@ -14,46 +14,16 @@ PolygonLabel, PolygonTransaction, WyrmBlock, - Wyrmlabel, + WyrmLabel, WyrmTransaction, XDaiBlock, XDaiLabel, XDaiTransaction, ) + from moonstreamdb.moonworm import Network, MODELS, tx_raw_types except ImportError: print("this feature requires moonstreamdb which is not installed") print("to enable, run: `pip install moonworm[moonstream]`") raise ImportError( "moonstreamdb not installed, to install, run: `pip install moonworm[moonstream]`" ) - - -from .utils import Network - -MODELS: Dict[Network, Dict[str, Base]] = { - Network.ethereum: { - "blocks": EthereumBlock, - "labels": EthereumLabel, - "transactions": EthereumTransaction, - }, - Network.mumbai: { - "blocks": MumbaiBlock, - "labels": MumbaiLabel, - "transactions": MumbaiTransaction, - }, - Network.polygon: { - "blocks": PolygonBlock, - "labels": PolygonLabel, - "transactions": PolygonTransaction, - }, - Network.xdai: { - "blocks": XDaiBlock, - "labels": XDaiLabel, - "transactions": XDaiTransaction, - }, - Network.wyrm: { - "blocks": WyrmBlock, - "labels": Wyrmlabel, - "transactions": WyrmTransaction, - }, -} From bcb6270273b93a108217e13a9827ca8ae316750c Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Mar 2023 16:03:44 +0200 Subject: [PATCH 4/8] Remove import from moonstreamdb. --- .../moonstream_ethereum_state_provider.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py index e49c0a0..d39a921 100644 --- a/moonworm/crawler/moonstream_ethereum_state_provider.py +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -4,22 +4,13 @@ from eth_typing.evm import ChecksumAddress from hexbytes.main import HexBytes from sqlalchemy.orm import Session -from sqlalchemy.sql.base import NO_ARG from web3 import Web3 from .ethereum_state_provider import EthereumStateProvider from .networks import ( MODELS, - EthereumLabel, - EthereumTransaction, - MumbaiLabel, - MumbaiTransaction, - PolygonLabel, - PolygonTransaction, - WyrmTransaction, - XDaiTransaction, - yield_db_session_ctx, Network, + tx_raw_types, ) logging.basicConfig(level=logging.INFO) @@ -82,13 +73,7 @@ def get_last_block_number(self) -> int: @staticmethod def _transform_to_w3_tx( - tx_raw: Union[ - EthereumTransaction, - MumbaiTransaction, - PolygonTransaction, - XDaiTransaction, - WyrmTransaction, - ], + tx_raw: tx_raw_types, ) -> Dict[str, Any]: tx = { "blockNumber": tx_raw.block_number, From cc6032fad0f177eaf894ed0e29ed61ad4c84a156 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Mar 2023 18:02:42 +0200 Subject: [PATCH 5/8] isort fix --- moonworm/crawler/moonstream_ethereum_state_provider.py | 6 +----- moonworm/crawler/networks.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py index d39a921..6ad3e41 100644 --- a/moonworm/crawler/moonstream_ethereum_state_provider.py +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -7,11 +7,7 @@ from web3 import Web3 from .ethereum_state_provider import EthereumStateProvider -from .networks import ( - MODELS, - Network, - tx_raw_types, -) +from .networks import MODELS, Network, tx_raw_types logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py index 2cc91fc..77fd14d 100644 --- a/moonworm/crawler/networks.py +++ b/moonworm/crawler/networks.py @@ -20,7 +20,7 @@ XDaiLabel, XDaiTransaction, ) - from moonstreamdb.moonworm import Network, MODELS, tx_raw_types + from moonstreamdb.moonworm import MODELS, Network, tx_raw_types except ImportError: print("this feature requires moonstreamdb which is not installed") print("to enable, run: `pip install moonworm[moonstream]`") From 6ca373ce3676accec41b753a8ab35cfdc805a83b Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Mar 2023 18:09:08 +0200 Subject: [PATCH 6/8] black formating. --- moonworm/cli.py | 1 - moonworm/crawler/example.py | 1 - moonworm/crawler/log_scanner.py | 2 -- moonworm/crawler/moonstream_ethereum_state_provider.py | 1 + moonworm/generators/basic.py | 1 - moonworm/generators/brownie.py | 1 - moonworm/tests/test_tester_provider.py | 3 --- moonworm/web3_util.py | 1 - 8 files changed, 1 insertion(+), 10 deletions(-) diff --git a/moonworm/cli.py b/moonworm/cli.py index 992e88e..dfc6739 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -183,7 +183,6 @@ def handle_watch(args: argparse.Namespace) -> None: state_provider.clear_db_session() else: - watch_contract( web3=web3, state_provider=Web3StateProvider(web3), diff --git a/moonworm/crawler/example.py b/moonworm/crawler/example.py index 7479bcb..72fdfa2 100644 --- a/moonworm/crawler/example.py +++ b/moonworm/crawler/example.py @@ -43,7 +43,6 @@ ] def run(): - if len(sys.argv) < 2: print("Usage: eventscanner.py http://your-node-url") sys.exit(1) diff --git a/moonworm/crawler/log_scanner.py b/moonworm/crawler/log_scanner.py index 27b8a29..7982770 100644 --- a/moonworm/crawler/log_scanner.py +++ b/moonworm/crawler/log_scanner.py @@ -303,7 +303,6 @@ def get_block_when(block_num): all_processed = [] for event_type in self.events: - # Callable that takes care of the underlying web3 call def _fetch_events(_start_block, _end_block): return _fetch_events_chunk( @@ -385,7 +384,6 @@ def scan( all_processed = [] while current_block <= end_block: - self.state.start_chunk(current_block, chunk_size) # Print some diagnostics to logs to try to fiddle with real world JSON-RPC API performance diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py index 6ad3e41..620c047 100644 --- a/moonworm/crawler/moonstream_ethereum_state_provider.py +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -12,6 +12,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + # TODO(yhtiyar) When getting block from db, filter it by `to` address, it will be faster # also get blocks in bunch class MoonstreamEthereumStateProvider(EthereumStateProvider): diff --git a/moonworm/generators/basic.py b/moonworm/generators/basic.py index 2e8611c..1cfec03 100644 --- a/moonworm/generators/basic.py +++ b/moonworm/generators/basic.py @@ -341,7 +341,6 @@ def generate_contract_constructor_function( def generate_contract_function(func_object: Dict[str, Any]) -> cst.FunctionDef: - default_param_name = "arg" default_counter = 1 func_params = [] diff --git a/moonworm/generators/brownie.py b/moonworm/generators/brownie.py index 3d179ae..b05be9e 100644 --- a/moonworm/generators/brownie.py +++ b/moonworm/generators/brownie.py @@ -235,7 +235,6 @@ def generate_brownie_contract_function( f"return self.contract.{func_raw_name}({','.join(param_names)})" ) else: - func_params.append( cst.Param( name=cst.Name(value="block_number"), diff --git a/moonworm/tests/test_tester_provider.py b/moonworm/tests/test_tester_provider.py index 4552030..5d8bbf7 100644 --- a/moonworm/tests/test_tester_provider.py +++ b/moonworm/tests/test_tester_provider.py @@ -52,7 +52,6 @@ def check_eth_send( send_value, tx_receipt, ): - assert receiver_current_balance == receiver_previous_balance + send_value assert ( sender_current_balance @@ -60,7 +59,6 @@ def check_eth_send( ) def test_submit_transaction(self) -> None: - sender = Web3.toChecksumAddress(PK_ADDRESS) self.web3.eth.send_transaction receiver = Web3.toChecksumAddress(self.web3.eth.accounts[1]) @@ -95,7 +93,6 @@ def test_submit_transaction(self) -> None: ) def test_submit_signed_transaction(self) -> None: - sender = Web3.toChecksumAddress(PK_ADDRESS) self.web3.eth.send_transaction receiver = Web3.toChecksumAddress(self.web3.eth.accounts[1]) diff --git a/moonworm/web3_util.py b/moonworm/web3_util.py index 3494923..f777da2 100644 --- a/moonworm/web3_util.py +++ b/moonworm/web3_util.py @@ -53,7 +53,6 @@ def get_nonce(web3: Web3, address: ChecksumAddress) -> Nonce: def submit_transaction( web3: Web3, transaction: Union[TxParams, Any], signer_private_key: str ) -> HexBytes: - """ Signs and submits json transaction to blockchain from the name of signer """ From 76a9e49b728ae50327683f13b907fbac11d93446 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 7 Mar 2023 14:42:12 +0200 Subject: [PATCH 7/8] Remove import from moonsteramdb_event_state. Remove unrequired imports. --- moonworm/crawler/networks.py | 19 +++---------------- .../crawler/state/moonstream_event_state.py | 2 +- moonworm/crawler/utils.py | 9 --------- 3 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 moonworm/crawler/utils.py diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py index 77fd14d..cf1d69c 100644 --- a/moonworm/crawler/networks.py +++ b/moonworm/crawler/networks.py @@ -3,24 +3,11 @@ try: from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import ( - Base, EthereumBlock, EthereumLabel, - EthereumTransaction, - MumbaiBlock, - MumbaiLabel, - MumbaiTransaction, - PolygonBlock, - PolygonLabel, - PolygonTransaction, - WyrmBlock, - WyrmLabel, - WyrmTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, - ) - from moonstreamdb.moonworm import MODELS, Network, tx_raw_types + ) # state/moonstream_event_state dependency maybe removed in the future + from moonstreamdb.networks import MODELS, Network, tx_raw_types + except ImportError: print("this feature requires moonstreamdb which is not installed") print("to enable, run: `pip install moonworm[moonstream]`") diff --git a/moonworm/crawler/state/moonstream_event_state.py b/moonworm/crawler/state/moonstream_event_state.py index fbdcc28..a20d504 100644 --- a/moonworm/crawler/state/moonstream_event_state.py +++ b/moonworm/crawler/state/moonstream_event_state.py @@ -1,4 +1,4 @@ -from moonstreamdb.models import EthereumBlock, EthereumLabel +from ..networks import EthereumBlock, EthereumLabel from sqlalchemy.orm import Query, Session from web3 import Web3 diff --git a/moonworm/crawler/utils.py b/moonworm/crawler/utils.py deleted file mode 100644 index b3aade1..0000000 --- a/moonworm/crawler/utils.py +++ /dev/null @@ -1,9 +0,0 @@ -from enum import Enum - - -class Network(Enum): - ethereum = "ethereum" - polygon = "polygon" - mumbai = "mumbai" - xdai = "xdai" - wyrm = "wyrm" From 150b468e820dc11455872341c23c4b9fd2c4eee8 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 8 Mar 2023 19:18:45 +0200 Subject: [PATCH 8/8] isort --- moonworm/crawler/networks.py | 4 ++-- moonworm/crawler/state/moonstream_event_state.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py index cf1d69c..0c85ebd 100644 --- a/moonworm/crawler/networks.py +++ b/moonworm/crawler/networks.py @@ -2,10 +2,10 @@ try: from moonstreamdb.db import yield_db_session_ctx - from moonstreamdb.models import ( + from moonstreamdb.models import ( # state/moonstream_event_state dependency maybe removed in the future EthereumBlock, EthereumLabel, - ) # state/moonstream_event_state dependency maybe removed in the future + ) from moonstreamdb.networks import MODELS, Network, tx_raw_types except ImportError: diff --git a/moonworm/crawler/state/moonstream_event_state.py b/moonworm/crawler/state/moonstream_event_state.py index a20d504..4577f58 100644 --- a/moonworm/crawler/state/moonstream_event_state.py +++ b/moonworm/crawler/state/moonstream_event_state.py @@ -1,7 +1,7 @@ -from ..networks import EthereumBlock, EthereumLabel from sqlalchemy.orm import Query, Session from web3 import Web3 +from ..networks import EthereumBlock, EthereumLabel from .event_scanner_state import EventScannerState BLOCK_TIMESTAMP_CACHE = {}